From 689214f075d1259fa839784625a67795e56fba1c Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Wed, 9 Dec 2020 09:05:24 +0100 Subject: [PATCH 01/47] Remove Travis-specific macros --- after_failure.sh | 3 --- after_success.sh | 3 --- before_cache.sh | 3 --- common.sh | 8 -------- 4 files changed, 17 deletions(-) diff --git a/after_failure.sh b/after_failure.sh index 8258bb6..62b88a1 100755 --- a/after_failure.sh +++ b/after_failure.sh @@ -2,9 +2,6 @@ source $TRAVIS_BUILD_DIR/.travis/common.sh -# Close the after_failure fold travis has created already. -travis_fold end after_failure - $SPACER start_section "failure.tail" "${RED}Failure output...${NC}" diff --git a/after_success.sh b/after_success.sh index 4fff6c0..a7c353c 100755 --- a/after_success.sh +++ b/after_success.sh @@ -3,9 +3,6 @@ source ./.travis/common.sh set -e -# Close the after_success fold travis has created already. -travis_fold end after_success - if [[ $UPLOAD == "no-upload" ]]; then echo "Job without upload..." else diff --git a/before_cache.sh b/before_cache.sh index feff250..76c7758 100755 --- a/before_cache.sh +++ b/before_cache.sh @@ -3,9 +3,6 @@ source ./.travis/common.sh set -e -# Close the after_success.1 fold travis has created already. -travis_fold end before_cache - start_section "conda.clean.1" "${GREEN}Clean status...${NC}" #conda clean -s --dry-run end_section "conda.clean.1" diff --git a/common.sh b/common.sh index 6346dba..2348f13 100644 --- a/common.sh +++ b/common.sh @@ -9,10 +9,6 @@ NC='\033[0m' # No Color SPACER="echo -e ${GRAY} - ${NC}" -export -f travis_nanoseconds -export -f travis_fold -export -f travis_time_start -export -f travis_time_finish TRAVIS_MAX_TIME=50 @@ -88,16 +84,12 @@ if [ -z "$DATE_STR" ]; then fi function start_section() { - travis_fold start "$1" - travis_time_start echo -e "${PURPLE}${PACKAGE}${NC}: - $2${NC}" echo -e "${GRAY}-------------------------------------------------------------------${NC}" } function end_section() { echo -e "${GRAY}-------------------------------------------------------------------${NC}" - travis_time_finish - travis_fold end "$1" } # Disable this warning; From 61852f525a5dda5d1906c4f54dcc8587dfd18b61 Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Wed, 9 Dec 2020 09:13:36 +0100 Subject: [PATCH 02/47] Replace Travis' env variables with GH Actions counterparts --- after_failure.sh | 2 +- after_success.sh | 9 ++++----- before_cache.sh | 2 +- cleanup-anaconda.sh | 2 +- common.sh | 19 +++++++++---------- conda-env.sh | 19 +++++++++---------- conda-get.sh | 4 ++-- conda-meta-extra.sh | 14 +++++++------- download_sdk.sh | 2 +- fixup-git.sh | 2 +- install.sh | 12 ++++++------ master-package.sh | 8 ++++---- script.sh | 8 ++++---- 13 files changed, 50 insertions(+), 53 deletions(-) diff --git a/after_failure.sh b/after_failure.sh index 62b88a1..0ecd5ec 100755 --- a/after_failure.sh +++ b/after_failure.sh @@ -1,6 +1,6 @@ #!/bin/bash -source $TRAVIS_BUILD_DIR/.travis/common.sh +source $GITHUB_WORKSPACE/.github/scripts/common.sh $SPACER diff --git a/after_success.sh b/after_success.sh index a7c353c..312c6c0 100755 --- a/after_success.sh +++ b/after_success.sh @@ -1,6 +1,6 @@ #!/bin/bash -source ./.travis/common.sh +source $GITHUB_WORKSPACE/.github/scripts/common.sh set -e if [[ $UPLOAD == "no-upload" ]]; then @@ -9,18 +9,17 @@ else echo "Job with Conda upload..." $SPACER - # Travis will not expose the ANACONDA_TOKEN var for pull requests coming from other forks than the original one if [ x$ANACONDA_TOKEN != x ]; then # `anaconda-client` is installed in the `base` environment conda activate base start_section "package.upload" "${GREEN}Package uploading...${NC}" # Test `anaconda` with ANACONDA_TOKEN before uploading - source $TRAVIS_BUILD_DIR/.travis/test_anaconda.sh - anaconda -t $ANACONDA_TOKEN upload --no-progress --user $ANACONDA_USER --label travis-${TRAVIS_PULL_REQUEST_BRANCH:-$TRAVIS_BRANCH}-$TRAVIS_BUILD_ID $CONDA_OUT + source $GITHUB_WORKSPACE/.github/scripts/test_anaconda.sh + anaconda -t $ANACONDA_TOKEN upload --no-progress --user $ANACONDA_USER --label ci-${GITHUB_HEAD_REF:-$CI_BRANCH}-$GITHUB_RUN_ID $CONDA_OUT end_section "package.upload" else - echo "ANACONDA_TOKEN not found. Please consult README of litex-conda-ci for details on setting up Travis tests properly." + echo "ANACONDA_TOKEN not found. Please consult README of litex-conda-ci for details on setting up tests properly." echo "Packages cannot be uploaded when tests are running for cross-repository Pull Requests." fi diff --git a/before_cache.sh b/before_cache.sh index 76c7758..49edf78 100755 --- a/before_cache.sh +++ b/before_cache.sh @@ -1,6 +1,6 @@ #!/bin/bash -source ./.travis/common.sh +source $GITHUB_WORKSPACE/.github/scripts/common.sh set -e start_section "conda.clean.1" "${GREEN}Clean status...${NC}" diff --git a/cleanup-anaconda.sh b/cleanup-anaconda.sh index 21fb1e5..a701949 100644 --- a/cleanup-anaconda.sh +++ b/cleanup-anaconda.sh @@ -13,7 +13,7 @@ echo "Will remove labels older than $limit_date timestamp" for label in $(anaconda -t $ANACONDA_TOKEN label --list -o litex-hub 2>&1 | grep ' + ' | cut -f2 -d1+) do - if [[ $label != travis* ]] + if [[ $label != ci* ]] then continue fi diff --git a/common.sh b/common.sh index 2348f13..223b1a1 100644 --- a/common.sh +++ b/common.sh @@ -9,11 +9,10 @@ NC='\033[0m' # No Color SPACER="echo -e ${GRAY} - ${NC}" - -TRAVIS_MAX_TIME=50 +CI_MAX_TIME=50 # Override default travis_wait to pipe the output -travis_wait() { +ci_wait() { local timeout="${1}" if [[ "${timeout}" =~ ^[0-9]+$ ]]; then @@ -23,12 +22,12 @@ travis_wait() { fi local cmd=("${@}") - local log_file="travis_wait_${$}.log" + local log_file="ci_wait_${$}.log" "${cmd[@]}" & local cmd_pid="${!}" - travis_jigger "${!}" "${timeout}" "${cmd[@]}" & + ci_jigger "${!}" "${timeout}" "${cmd[@]}" & local jigger_pid="${!}" local result @@ -50,7 +49,7 @@ travis_wait() { } # Override default travis_jigger to print invisible character to keep build alive -travis_jigger() { +ci_jigger() { local cmd_pid="${1}" shift local timeout="${1}" @@ -70,7 +69,7 @@ travis_jigger() { kill -9 "${cmd_pid}" } -if [ $TRAVIS_OS_NAME = 'osx' ]; then +if [ $OS_NAME = 'osx' ]; then DATE_SWITCH="-r " else DATE_SWITCH="--date=@" @@ -101,13 +100,13 @@ export PYTHONWARNINGS=ignore::UserWarning:conda_build.environ export BASE_PATH="/tmp/really-long-path" mkdir -p "$BASE_PATH" -if [ $TRAVIS_OS_NAME = 'windows' ]; then +if [ $OS_NAME = 'windows' ]; then export CONDA_PATH='/c/tools/miniconda3' export PATH=$CONDA_PATH/Scripts/:$CONDA_PATH/:$PATH # It is much shorter than '$PWD/workdir/conda-env' which in the end (+conda-bld/...) # causes some build paths to exceed 255 chars (e.g. during prjtrellis building) - export CONDA_ENV='/c/Users/travis/conda-env' + export CONDA_ENV='/c/Users/runner/conda-env' if [ -d 'workdir/conda-env' ]; then mv 'workdir/conda-env' "$CONDA_ENV" fi @@ -133,7 +132,7 @@ if [ -d "workdir/recipe" ]; then export CONDA_BUILD_ARGS="$EXTRA_BUILD_ARGS workdir/recipe" export CONDA_OUT="$(conda render --output $CONDA_BUILD_ARGS | grep conda-bld | grep tar.bz2 | tail -n 1 | sed -e's/-[0-9]\+\.tar/*.tar/' -e's/-git//')" - if [ "$TRAVIS_OS_NAME" = 'windows' ]; then + if [ "$OS_NAME" = 'windows' ]; then # conda render outputs Windows-style path which may contain wildcards; # 'git bash' used by Travis works well with wildcards only in Unix-style paths export CONDA_OUT="$(cygpath -u $CONDA_OUT)" diff --git a/conda-env.sh b/conda-env.sh index 4bf59ab..6f675d9 100755 --- a/conda-env.sh +++ b/conda-env.sh @@ -22,21 +22,20 @@ if [ -z "$GITREV" ]; then echo "Setting git revision $GITREV" fi -export TRAVIS=0 export CI=0 -export TRAVIS_EVENT_TYPE="local" -echo "TRAVIS_EVENT_TYPE='${TRAVIS_EVENT_TYPE}'" +export CI_EVENT_TYPE="local" +echo "CI_EVENT_TYPE='${CI_EVENT_TYPE}'" -export TRAVIS_BRANCH="$(git rev-parse --abbrev-ref HEAD)" -echo "TRAVIS_BRANCH='${TRAVIS_BRANCH}'" +export CI_BRANCH="$(git rev-parse --abbrev-ref HEAD)" +echo "CI_BRANCH='${CI_BRANCH}'" -export TRAVIS_COMMIT="$(git rev-parse HEAD)" -echo "TRAVIS_COMMIT='${TRAVIS_COMMIT}'" +export CI_COMMIT="$(git rev-parse HEAD)" +echo "CI_COMMIT='${CI_COMMIT}'" -export TRAVIS_REPO_SLUG="$(git rev-parse --abbrev-ref --symbolic-full-name @{u})" -echo "TRAVIS_REPO_SLUG='${TRAVIS_REPO_SLUG}'" +export CI_REPO_SLUG="$(git rev-parse --abbrev-ref --symbolic-full-name @{u})" +echo "CI_REPO_SLUG='${CI_REPO_SLUG}'" -./.travis/conda-meta-extra.sh +$GITHUB_WORKSPACE/.github/scripts/conda-meta-extra.sh echo conda $@ conda $@ diff --git a/conda-get.sh b/conda-get.sh index 16b5ed3..94f1652 100755 --- a/conda-get.sh +++ b/conda-get.sh @@ -4,7 +4,7 @@ set -x set -e CONDA_PATH=${1:-~/conda} -if [ $TRAVIS_OS_NAME = 'windows' ]; then +if [ $OS_NAME = 'windows' ]; then if [ ! -d $CONDA_PATH -o ! -z "$CI" ]; then choco install openssl.light choco install miniconda3 --params="/AddToPath:1" @@ -12,7 +12,7 @@ if [ $TRAVIS_OS_NAME = 'windows' ]; then export CONDA_PATH='/c/tools/miniconda3' export PATH=$CONDA_PATH/bin/:$CONDA_PATH/Scripts/:$PATH else - if [ $TRAVIS_OS_NAME = 'linux' ]; then + if [ $OS_NAME = 'linux' ]; then sys_name=Linux else sys_name=MacOSX diff --git a/conda-meta-extra.sh b/conda-meta-extra.sh index b729d18..d3c61cf 100755 --- a/conda-meta-extra.sh +++ b/conda-meta-extra.sh @@ -12,14 +12,14 @@ extra: maintainers: - Tim 'mithro' Ansell - HDMI2USB Project - https://hdmi2usb.tv - travis: - job_id: $TRAVIS_JOB_ID - job_num: $TRAVIS_JOB_NUMBER - type: $TRAVIS_EVENT_TYPE + ci: + job_id: $GITHUB_RUN_ID + job_num: $GITHUB_RUN_NUMBER + type: $CI_EVENT_TYPE recipe: - repo: 'https://github.com/$TRAVIS_REPO_SLUG' - branch: $TRAVIS_BRANCH - commit: $TRAVIS_COMMIT + repo: 'https://github.com/$CI_REPO_SLUG' + branch: $CI_BRANCH + commit: $CI_COMMIT describe: $GITREV date: $DATESTR EOF diff --git a/download_sdk.sh b/download_sdk.sh index 74ee0f3..97f6d79 100755 --- a/download_sdk.sh +++ b/download_sdk.sh @@ -1,4 +1,4 @@ -if [ $TRAVIS_OS_NAME = 'osx' ]; then +if [ $OS_NAME = 'osx' ]; then if [[ ! -d $HOME/sdk/MacOSX10.9.sdk ]]; then git clone https://github.com/phracker/MacOSX-SDKs $HOME/sdk fi diff --git a/fixup-git.sh b/fixup-git.sh index f8423d1..a3e0630 100755 --- a/fixup-git.sh +++ b/fixup-git.sh @@ -1,6 +1,6 @@ #!/bin/bash -source $TRAVIS_BUILD_DIR/.travis/common.sh +source $GITHUB_WORKSPACE/.github/scripts/common.sh set -e # Git repo fixup diff --git a/install.sh b/install.sh index 4dc56e7..250a73f 100755 --- a/install.sh +++ b/install.sh @@ -1,14 +1,14 @@ #!/bin/bash -source $TRAVIS_BUILD_DIR/.travis/common.sh +source $GITHUB_WORKSPACE/.github/scripts/common.sh set -e # Getting the conda environment start_section "environment.conda" "Setting up basic ${YELLOW}conda environment${NC}" -branch=${TRAVIS_PULL_REQUEST_BRANCH:-$TRAVIS_BRANCH} +branch="$(git rev-parse --abbrev-ref HEAD)" mkdir -p $BASE_PATH -./.travis/conda-get.sh $CONDA_PATH +$GITHUB_WORKSPACE/.github/scripts/conda-get.sh $CONDA_PATH hash -r if [ x$PACKAGE = x"" ]; then @@ -17,13 +17,13 @@ if [ x$PACKAGE = x"" ]; then fi # Add '.travis' build variants to the recipe dir (appended keys win in case of any conflict) -cat "$TRAVIS_BUILD_DIR/.travis/conda_build_config.yaml" >> "$PACKAGE/conda_build_config.yaml" +cat "$GITHUB_WORKSPACE/.github/scripts/conda_build_config.yaml" >> "$PACKAGE/conda_build_config.yaml" # Install conda-build-prepare python -m pip install git+https://github.com/litex-hub/conda-build-prepare@v0.1#egg=conda-build-prepare # The last channel will be on top of the environment's channel list -ADDITIONAL_CHANNELS="litex-hub $(echo $TRAVIS_REPO_SLUG | sed -e's@/.*$@@') litex-hub/label/travis-$branch-$TRAVIS_BUILD_ID $(echo $TRAVIS_REPO_SLUG | sed -e's@/.*$@@')/label/travis-$branch-$TRAVIS_BUILD_ID" +ADDITIONAL_CHANNELS="litex-hub $(echo $CI_REPO_SLUG | sed -e's@/.*$@@') litex-hub/label/ci-$branch-$GITHUB_RUN_ID $(echo $CI_REPO_SLUG | sed -e's@/.*$@@')/label/ci-$branch-$CI_BUILD_ID" ADDITIONAL_PACKAGES="conda-build=3.20.3 conda-verify jinja2 pexpect python=3.7" if [[ "$TRAVIS_OS_NAME" != 'windows' ]]; then @@ -34,7 +34,7 @@ fi python -m conda_build_prepare --channels $ADDITIONAL_CHANNELS --packages $ADDITIONAL_PACKAGES --dir workdir $PACKAGE # Freshly created conda environment will be activated by the common.sh -source $TRAVIS_BUILD_DIR/.travis/common.sh +source $GITHUB_WORKSPACE/.github/scripts/common.sh end_section "environment.conda" diff --git a/master-package.sh b/master-package.sh index 9a8d6cb..469441a 100644 --- a/master-package.sh +++ b/master-package.sh @@ -1,11 +1,11 @@ #!/bin/bash # `for ... in $(anaconda ...` fails silently if there's any problem with anaconda -source $TRAVIS_BUILD_DIR/.travis/test_anaconda.sh +source $GITHUB_WORKSPACE/.github/scripts/test_anaconda.sh -branch=${TRAVIS_PULL_REQUEST_BRANCH:-$TRAVIS_BRANCH} +branch=${GITHUB_HEAD_REF:-$CI_BRANCH} # Move all packages from the current label to the main label -for package in $(anaconda -t $ANACONDA_TOKEN label --show travis-$branch-$TRAVIS_BUILD_ID 2>&1 | grep + | cut -f2 -d+) +for package in $(anaconda -t $ANACONDA_TOKEN label --show ci-$branch-$GITHUB_RUN_ID 2>&1 | grep + | cut -f2 -d+) do - anaconda -t $ANACONDA_TOKEN move --from-label travis-$branch-$TRAVIS_BUILD_ID --to-label main $package + anaconda -t $ANACONDA_TOKEN move --from-label ci-$branch-$GITHUB_RUN_ID --to-label main $package done diff --git a/script.sh b/script.sh index 8f8579c..9a020cd 100755 --- a/script.sh +++ b/script.sh @@ -1,6 +1,6 @@ #!/bin/bash -source ./.travis/common.sh +source $GITHUB_WORKSPACE/.github/scripts/common.sh set -e $SPACER @@ -18,11 +18,11 @@ end_section "conda.check" $SPACER start_section "conda.build" "${GREEN}Building..${NC}" -if [[ $TRAVIS_OS_NAME != 'windows' ]]; then +if [[ $OS_NAME != 'windows' ]]; then if [[ $KEEP_ALIVE = 'true' ]]; then - travis_wait $TRAVIS_MAX_TIME python $TRAVIS_BUILD_DIR/.travis/.travis-output.py /tmp/output.log conda build $CONDA_BUILD_ARGS + ci_wait $CI_MAX_TIME $CONDA_PATH/bin/python $GITHUB_WORKSPACE/.github/scripts/.ci-output.py /tmp/output.log conda build $CONDA_BUILD_ARGS else - python $TRAVIS_BUILD_DIR/.travis/.travis-output.py /tmp/output.log conda build $CONDA_BUILD_ARGS + python $GITHUB_WORKSPACE/.github/scripts/.ci-output.py /tmp/output.log conda build $CONDA_BUILD_ARGS fi else # Work-around: prevent console output being mangled From 71c0d6af8e4889f652a8af616b95bcbb595c5c2e Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Wed, 9 Dec 2020 09:14:35 +0100 Subject: [PATCH 03/47] Remove Travis-specific comments --- common.sh | 8 -------- conda-env.sh | 5 ----- install.sh | 2 +- 3 files changed, 1 insertion(+), 14 deletions(-) diff --git a/common.sh b/common.sh index 223b1a1..61c1737 100644 --- a/common.sh +++ b/common.sh @@ -11,7 +11,6 @@ SPACER="echo -e ${GRAY} - ${NC}" CI_MAX_TIME=50 -# Override default travis_wait to pipe the output ci_wait() { local timeout="${1}" @@ -48,7 +47,6 @@ ci_wait() { return "${result}" } -# Override default travis_jigger to print invisible character to keep build alive ci_jigger() { local cmd_pid="${1}" shift @@ -91,11 +89,6 @@ function end_section() { echo -e "${GRAY}-------------------------------------------------------------------${NC}" } -# Disable this warning; -# xxxx/conda_build/environ.py:377: UserWarning: The environment variable -# 'TRAVIS' is being passed through with value 0. If you are splitting -# build and test phases with --no-test, please ensure that this value is -# also set similarly at test time. export PYTHONWARNINGS=ignore::UserWarning:conda_build.environ export BASE_PATH="/tmp/really-long-path" @@ -134,7 +127,6 @@ if [ -d "workdir/recipe" ]; then if [ "$OS_NAME" = 'windows' ]; then # conda render outputs Windows-style path which may contain wildcards; - # 'git bash' used by Travis works well with wildcards only in Unix-style paths export CONDA_OUT="$(cygpath -u $CONDA_OUT)" fi fi diff --git a/conda-env.sh b/conda-env.sh index 6f675d9..4108d04 100755 --- a/conda-env.sh +++ b/conda-env.sh @@ -4,11 +4,6 @@ if ! which conda; then export PATH=~/conda/bin:$PATH fi -# Disable this warning; -# xxxx/conda_build/environ.py:377: UserWarning: The environment variable -# 'TRAVIS' is being passed through with value 0. If you are splitting -# build and test phases with --no-test, please ensure that this value is -# also set similarly at test time. export PYTHONWARNINGS=ignore::UserWarning:conda_build.environ if [ -z "$DATE_STR" ]; then diff --git a/install.sh b/install.sh index 250a73f..06f2ce2 100755 --- a/install.sh +++ b/install.sh @@ -16,7 +16,7 @@ if [ x$PACKAGE = x"" ]; then exit 0 fi -# Add '.travis' build variants to the recipe dir (appended keys win in case of any conflict) +# Add build variants to the recipe dir (appended keys win in case of any conflict) cat "$GITHUB_WORKSPACE/.github/scripts/conda_build_config.yaml" >> "$PACKAGE/conda_build_config.yaml" # Install conda-build-prepare From b4f32465f6b997726cfcc130189556c86d106371 Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Wed, 9 Dec 2020 09:17:17 +0100 Subject: [PATCH 04/47] windows: dump use of winpty Winpty causes assertion error in CI: https://github.com/antmicro/litex-conda-misc-1/runs/1510103359?check_suite_focus=true Disabling it fixed Windows builds. --- script.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script.sh b/script.sh index 9a020cd..2009946 100755 --- a/script.sh +++ b/script.sh @@ -26,7 +26,7 @@ if [[ $OS_NAME != 'windows' ]]; then fi else # Work-around: prevent console output being mangled - winpty.exe -Xallow-non-tty -Xplain conda build $CONDA_BUILD_ARGS 2>&1 | tee /tmp/output.log + conda build $CONDA_BUILD_ARGS | tee /tmp/output.log fi end_section "conda.build" From 911fefc43f6afef6045a2688eea1bcdfed4b386c Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Wed, 9 Dec 2020 09:26:42 +0100 Subject: [PATCH 05/47] fixup: git reset at desired revision, not checkout There were problems with these delete/checkout operations https://github.com/antmicro/litex-conda-misc-1/runs/1479196864?check_suite_focus=true --- fixup-git.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fixup-git.sh b/fixup-git.sh index a3e0630..1757cc2 100755 --- a/fixup-git.sh +++ b/fixup-git.sh @@ -13,9 +13,8 @@ git submodule foreach git submodule update --recursive --init $SPACER git remote -v git branch -v -git branch -D $TRAVIS_BRANCH CURRENT_GITREV="$(git rev-parse HEAD)" -git checkout -b $TRAVIS_BRANCH $CURRENT_GITREV +git reset $CURRENT_GITREV git tag -l git status -v git describe --long From 1721ed810adb2e1a00a043bc217f750563921eaa Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Wed, 9 Dec 2020 10:10:58 +0100 Subject: [PATCH 06/47] Rename .travis-output.py to .ci-output.py --- .travis-output.py => .ci-output.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .travis-output.py => .ci-output.py (100%) diff --git a/.travis-output.py b/.ci-output.py similarity index 100% rename from .travis-output.py rename to .ci-output.py From 1c94eb1bde011f7e94fd7fe3b910cdfb349c71bb Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Wed, 9 Dec 2020 10:24:08 +0100 Subject: [PATCH 07/47] Update macOS SDK location --- conda_build_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_build_config.yaml b/conda_build_config.yaml index 13aa574..46eb426 100644 --- a/conda_build_config.yaml +++ b/conda_build_config.yaml @@ -1,3 +1,3 @@ CONDA_BUILD_SYSROOT: - - /Users/travis/sdk/MacOSX10.9.sdk # [osx] + - /Users/runner/sdk/MacOSX10.9.sdk # [osx] From f170a0240c862580a298620c68c103049b8f34e3 Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Wed, 9 Dec 2020 11:38:58 +0100 Subject: [PATCH 08/47] install.sh: limit additional channels --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 06f2ce2..8b1e0a3 100755 --- a/install.sh +++ b/install.sh @@ -23,7 +23,7 @@ cat "$GITHUB_WORKSPACE/.github/scripts/conda_build_config.yaml" >> "$PACKAGE/con python -m pip install git+https://github.com/litex-hub/conda-build-prepare@v0.1#egg=conda-build-prepare # The last channel will be on top of the environment's channel list -ADDITIONAL_CHANNELS="litex-hub $(echo $CI_REPO_SLUG | sed -e's@/.*$@@') litex-hub/label/ci-$branch-$GITHUB_RUN_ID $(echo $CI_REPO_SLUG | sed -e's@/.*$@@')/label/ci-$branch-$CI_BUILD_ID" +ADDITIONAL_CHANNELS="litex-hub litex-hub/label/ci-$branch-$GITHUB_RUN_ID" ADDITIONAL_PACKAGES="conda-build=3.20.3 conda-verify jinja2 pexpect python=3.7" if [[ "$TRAVIS_OS_NAME" != 'windows' ]]; then From 0a4376fb72a47c26de25a32fe42f77557f6cc266 Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Wed, 9 Dec 2020 12:32:31 +0100 Subject: [PATCH 09/47] Create directory before generating conda env in it --- common.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/common.sh b/common.sh index 61c1737..636c453 100644 --- a/common.sh +++ b/common.sh @@ -99,7 +99,12 @@ if [ $OS_NAME = 'windows' ]; then # It is much shorter than '$PWD/workdir/conda-env' which in the end (+conda-bld/...) # causes some build paths to exceed 255 chars (e.g. during prjtrellis building) - export CONDA_ENV='/c/Users/runner/conda-env' + RUNNER_DIR='/c/Users/runner/' + if [ ! -d "$RUNNER_DIR" ]; then + mkdir "$RUNNER_DIR" + fi + + export CONDA_ENV="$RUNNER_DIR/conda-env" if [ -d 'workdir/conda-env' ]; then mv 'workdir/conda-env' "$CONDA_ENV" fi From 92458e38222f9cab760b8439d7283ca9506ffe75 Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Tue, 15 Dec 2020 12:15:53 +0100 Subject: [PATCH 10/47] Update README.md --- README.md | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/README.md b/README.md index d4bd706..1a94fb4 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,5 @@ # litex-conda-ci -Travis scripts for Litex-Hub/litex-conda-(compilers|eda|misc|prog) . +Scripts for Litex-Hub/litex-conda-(compilers|eda|misc|prog) Continuous Integration. This repository is not meant to be used on its own, but as submodule for above repositories. - -## Setting up Travis for your own fork - -To use any of the litex-conda-* repositories in your own fork with Travis you must meet the following requirements: - -* enable Travis for your repository -* create an account on https://anaconda.org/ -* create an access token on https://anaconda.org/{your-account}/settings/access -* visit https://travis-ci.com/github/{your-account}/litex-conda-{suffix}/settings -* add two environment variables: - * `ANACONDA_USER` - set to your anaconda username - * `ANACONDA_TOKEN` - set to the value of the token you created -* remember to **disable** the `Display value in build log` option for `ANACONDA_TOKEN` -* create a daily or weekly Cron Job on the same settings page, to start on the master branch - * this will ensure removal of temporary labels on your Anaconda account (they are created during the build) From 754718669748077c4a6d66951fdad56f91072db0 Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Wed, 16 Dec 2020 10:45:19 +0100 Subject: [PATCH 11/47] Install pexpect --- script.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/script.sh b/script.sh index 2009946..9132dfe 100755 --- a/script.sh +++ b/script.sh @@ -17,6 +17,11 @@ end_section "conda.check" $SPACER +echo "Install pexpect" +if [ -f $CONDA_PATH/bin/python ]; then + $CONDA_PATH/bin/python -m pip install pexpect +fi + start_section "conda.build" "${GREEN}Building..${NC}" if [[ $OS_NAME != 'windows' ]]; then if [[ $KEEP_ALIVE = 'true' ]]; then From 5158c1d090a35b13d490f574405ccb3faa9937dc Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Mon, 21 Dec 2020 12:45:49 +0100 Subject: [PATCH 12/47] Add Github action definition --- action.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 action.yml diff --git a/action.yml b/action.yml new file mode 100644 index 0000000..f22c4a3 --- /dev/null +++ b/action.yml @@ -0,0 +1,25 @@ +name: 'Package build' +description: 'Build a package' +runs: + using: "composite" + steps: + - name: build + run: | + if [ $SKIP ]; then + exit 0; + fi + export PATH="$PATH:/c/ProgramData/Chocolatey/bin/" + export PATH="$PATH:/c/Program Files/Git/usr/bin/" + source .github/scripts/common.sh + bash .github/scripts/fixup-git.sh + bash .github/scripts/download_sdk.sh + source .github/scripts/common.sh + bash .github/scripts/install.sh + set -x + bash .github/scripts/script.sh + if [ $? -eq 0 ]; then + source .github/scripts/after_success.sh + else + source .github/scripts/after_failure.sh + fi + shell: bash From 865f4c58297fc78d07ddc124c6e785fe8a243445 Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Tue, 22 Dec 2020 11:24:45 +0100 Subject: [PATCH 13/47] Initialize environment only when conda-meta exists --- common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common.sh b/common.sh index 636c453..f607e1b 100644 --- a/common.sh +++ b/common.sh @@ -114,7 +114,7 @@ else export CONDA_ENV='workdir/conda-env' fi -if [ -d "$CONDA_ENV" ] && which conda &>/dev/null; then +if [ -d "$CONDA_ENV/conda-meta" ] && which conda &>/dev/null; then # >>> conda initialize >>> eval "$('conda' 'shell.bash' 'hook' 2> /dev/null)" # <<< conda initialize <<< From 48d51ac579eac12caff3ff1e09ea7e7d151a8733 Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Wed, 23 Dec 2020 12:21:51 +0100 Subject: [PATCH 14/47] Revert "install.sh: limit additional channels" This reverts commit f170a0240c862580a298620c68c103049b8f34e3. --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 8b1e0a3..06f2ce2 100755 --- a/install.sh +++ b/install.sh @@ -23,7 +23,7 @@ cat "$GITHUB_WORKSPACE/.github/scripts/conda_build_config.yaml" >> "$PACKAGE/con python -m pip install git+https://github.com/litex-hub/conda-build-prepare@v0.1#egg=conda-build-prepare # The last channel will be on top of the environment's channel list -ADDITIONAL_CHANNELS="litex-hub litex-hub/label/ci-$branch-$GITHUB_RUN_ID" +ADDITIONAL_CHANNELS="litex-hub $(echo $CI_REPO_SLUG | sed -e's@/.*$@@') litex-hub/label/ci-$branch-$GITHUB_RUN_ID $(echo $CI_REPO_SLUG | sed -e's@/.*$@@')/label/ci-$branch-$CI_BUILD_ID" ADDITIONAL_PACKAGES="conda-build=3.20.3 conda-verify jinja2 pexpect python=3.7" if [[ "$TRAVIS_OS_NAME" != 'windows' ]]; then From 767d83d82dc2d0eb3776dc952e33c034f4c74770 Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Wed, 23 Dec 2020 12:22:49 +0100 Subject: [PATCH 15/47] Use GITHUB_REPOSITORY to extract owner name --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 06f2ce2..c97eb29 100755 --- a/install.sh +++ b/install.sh @@ -23,7 +23,7 @@ cat "$GITHUB_WORKSPACE/.github/scripts/conda_build_config.yaml" >> "$PACKAGE/con python -m pip install git+https://github.com/litex-hub/conda-build-prepare@v0.1#egg=conda-build-prepare # The last channel will be on top of the environment's channel list -ADDITIONAL_CHANNELS="litex-hub $(echo $CI_REPO_SLUG | sed -e's@/.*$@@') litex-hub/label/ci-$branch-$GITHUB_RUN_ID $(echo $CI_REPO_SLUG | sed -e's@/.*$@@')/label/ci-$branch-$CI_BUILD_ID" +ADDITIONAL_CHANNELS="litex-hub $(echo $GITHUB_REPOSITORY | sed -e's@/.*$@@') litex-hub/label/ci-$branch-$GITHUB_RUN_ID $(echo $GITHUB_REPOSITORY | sed -e's@/.*$@@')/label/ci-$branch-$CI_BUILD_ID" ADDITIONAL_PACKAGES="conda-build=3.20.3 conda-verify jinja2 pexpect python=3.7" if [[ "$TRAVIS_OS_NAME" != 'windows' ]]; then From 1b3dc51bec855a2f6e63218d068d9450c81513e2 Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Wed, 23 Dec 2020 13:29:15 +0100 Subject: [PATCH 16/47] Correct channel labels --- after_success.sh | 3 ++- install.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/after_success.sh b/after_success.sh index 312c6c0..4cf2dbf 100755 --- a/after_success.sh +++ b/after_success.sh @@ -16,7 +16,8 @@ else start_section "package.upload" "${GREEN}Package uploading...${NC}" # Test `anaconda` with ANACONDA_TOKEN before uploading source $GITHUB_WORKSPACE/.github/scripts/test_anaconda.sh - anaconda -t $ANACONDA_TOKEN upload --no-progress --user $ANACONDA_USER --label ci-${GITHUB_HEAD_REF:-$CI_BRANCH}-$GITHUB_RUN_ID $CONDA_OUT + branch="$(git rev-parse --abbrev-ref HEAD)" + anaconda -t $ANACONDA_TOKEN upload --no-progress --user $ANACONDA_USER --label ci-$branch-$GITHUB_RUN_ID $CONDA_OUT end_section "package.upload" else echo "ANACONDA_TOKEN not found. Please consult README of litex-conda-ci for details on setting up tests properly." diff --git a/install.sh b/install.sh index c97eb29..5ecdc7c 100755 --- a/install.sh +++ b/install.sh @@ -23,7 +23,7 @@ cat "$GITHUB_WORKSPACE/.github/scripts/conda_build_config.yaml" >> "$PACKAGE/con python -m pip install git+https://github.com/litex-hub/conda-build-prepare@v0.1#egg=conda-build-prepare # The last channel will be on top of the environment's channel list -ADDITIONAL_CHANNELS="litex-hub $(echo $GITHUB_REPOSITORY | sed -e's@/.*$@@') litex-hub/label/ci-$branch-$GITHUB_RUN_ID $(echo $GITHUB_REPOSITORY | sed -e's@/.*$@@')/label/ci-$branch-$CI_BUILD_ID" +ADDITIONAL_CHANNELS="litex-hub $(echo $GITHUB_REPOSITORY | sed -e's@/.*$@@') litex-hub/label/ci-$branch-$GITHUB_RUN_ID $(echo $GITHUB_REPOSITORY | sed -e's@/.*$@@')/label/ci-$branch-$GITHUB_RUN_ID" ADDITIONAL_PACKAGES="conda-build=3.20.3 conda-verify jinja2 pexpect python=3.7" if [[ "$TRAVIS_OS_NAME" != 'windows' ]]; then From d3ebc64959eccf65e2c19b408b514ce4d080604b Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Mon, 28 Dec 2020 11:42:57 +0100 Subject: [PATCH 17/47] Force upload to temporary label Upload resulted in an error when there was already package in this version in channel https://github.com/antmicro/litex-conda-misc-1/runs/1616885674?check_suite_focus=true --- after_success.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/after_success.sh b/after_success.sh index 4cf2dbf..9579d3a 100755 --- a/after_success.sh +++ b/after_success.sh @@ -17,7 +17,7 @@ else # Test `anaconda` with ANACONDA_TOKEN before uploading source $GITHUB_WORKSPACE/.github/scripts/test_anaconda.sh branch="$(git rev-parse --abbrev-ref HEAD)" - anaconda -t $ANACONDA_TOKEN upload --no-progress --user $ANACONDA_USER --label ci-$branch-$GITHUB_RUN_ID $CONDA_OUT + anaconda -t $ANACONDA_TOKEN upload --force --no-progress --user $ANACONDA_USER --label ci-$branch-$GITHUB_RUN_ID $CONDA_OUT end_section "package.upload" else echo "ANACONDA_TOKEN not found. Please consult README of litex-conda-ci for details on setting up tests properly." From cc37d7a19c3d2dead61507bdc44d464f57e44cdd Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Mon, 28 Dec 2020 13:22:08 +0100 Subject: [PATCH 18/47] Remove fixup-git.sh --- action.yml | 1 - fixup-git.sh | 22 ---------------------- 2 files changed, 23 deletions(-) delete mode 100755 fixup-git.sh diff --git a/action.yml b/action.yml index f22c4a3..c6a1909 100644 --- a/action.yml +++ b/action.yml @@ -11,7 +11,6 @@ runs: export PATH="$PATH:/c/ProgramData/Chocolatey/bin/" export PATH="$PATH:/c/Program Files/Git/usr/bin/" source .github/scripts/common.sh - bash .github/scripts/fixup-git.sh bash .github/scripts/download_sdk.sh source .github/scripts/common.sh bash .github/scripts/install.sh diff --git a/fixup-git.sh b/fixup-git.sh deleted file mode 100755 index 1757cc2..0000000 --- a/fixup-git.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -source $GITHUB_WORKSPACE/.github/scripts/common.sh -set -e - -# Git repo fixup -start_section "environment.git" "Setting up ${YELLOW}git checkout${NC}" -set -x -git fetch --unshallow || true -git fetch --tags -git submodule update --recursive --init -git submodule foreach git submodule update --recursive --init -$SPACER -git remote -v -git branch -v -CURRENT_GITREV="$(git rev-parse HEAD)" -git reset $CURRENT_GITREV -git tag -l -git status -v -git describe --long -set +x -end_section "environment.git" From 45056ab4974dff605afad6431f85bb1eb4bea94a Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Mon, 28 Dec 2020 13:22:22 +0100 Subject: [PATCH 19/47] action.yml: little cleanup --- action.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/action.yml b/action.yml index c6a1909..2a89eab 100644 --- a/action.yml +++ b/action.yml @@ -12,9 +12,7 @@ runs: export PATH="$PATH:/c/Program Files/Git/usr/bin/" source .github/scripts/common.sh bash .github/scripts/download_sdk.sh - source .github/scripts/common.sh bash .github/scripts/install.sh - set -x bash .github/scripts/script.sh if [ $? -eq 0 ]; then source .github/scripts/after_success.sh From 5642fc755001954b48a046e51fdc361dd6088a98 Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Mon, 28 Dec 2020 15:16:35 +0100 Subject: [PATCH 20/47] Add master-package script handler --- master-package.sh | 0 wait-for-statuses.py | 24 ++++++++++++++++++++++++ 2 files changed, 24 insertions(+) mode change 100644 => 100755 master-package.sh create mode 100644 wait-for-statuses.py diff --git a/master-package.sh b/master-package.sh old mode 100644 new mode 100755 diff --git a/wait-for-statuses.py b/wait-for-statuses.py new file mode 100644 index 0000000..e450ea8 --- /dev/null +++ b/wait-for-statuses.py @@ -0,0 +1,24 @@ +import urllib.request +import json +import subprocess +import time +import os + +status_url = "https://api.github.com/repos/" \ + + os.environ['GITHUB_REPOSITORY'] \ + + "/actions/runs/" \ + + os.environ['GITHUB_RUN_ID'] \ + + "/jobs" + +while(True): + time.sleep(60) + success = True + with urllib.request.urlopen(status_url) as url: + data = json.loads(url.read().decode()) + for j in data["jobs"]: + if(j["status"] != "completed" and j["name"] != "master-package"): + success = False + if(success): + break + +subprocess.call(os.environ['GITHUB_WORKSPACE'] + "/.github/scripts/master-package.sh") From 70b0e372a48bae5bd4b194d1715c4769b8cf9a73 Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Tue, 29 Dec 2020 08:52:00 +0100 Subject: [PATCH 21/47] master-package: source common.sh We need Conda environment set when script is executed. --- master-package.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/master-package.sh b/master-package.sh index 469441a..488eee9 100755 --- a/master-package.sh +++ b/master-package.sh @@ -1,5 +1,6 @@ #!/bin/bash +source $GITHUB_WORKSPACE/.github/scripts/common.sh # `for ... in $(anaconda ...` fails silently if there's any problem with anaconda source $GITHUB_WORKSPACE/.github/scripts/test_anaconda.sh From b07835999b1e60e037631b4c4d3bd5e59c2c1c02 Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Tue, 29 Dec 2020 09:29:44 +0100 Subject: [PATCH 22/47] Correct remaining Travis' env variables --- cleanup-anaconda.sh | 3 ++- install.sh | 2 +- master-package.sh | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cleanup-anaconda.sh b/cleanup-anaconda.sh index a701949..c72e725 100644 --- a/cleanup-anaconda.sh +++ b/cleanup-anaconda.sh @@ -1,7 +1,8 @@ #!/bin/bash +source $GITHUB_WORKSPACE/.github/scripts/common.sh # `for ... in $(anaconda ...` fails silently if there's any problem with anaconda -source $TRAVIS_BUILD_DIR/.travis/test_anaconda.sh +source $GITHUB_WORKSPACE/.github/scripts/test_anaconda.sh #if the timestamp is older than one week, remove the whole label ago="7 days ago" diff --git a/install.sh b/install.sh index 5ecdc7c..fe27db4 100755 --- a/install.sh +++ b/install.sh @@ -26,7 +26,7 @@ python -m pip install git+https://github.com/litex-hub/conda-build-prepare@v0.1# ADDITIONAL_CHANNELS="litex-hub $(echo $GITHUB_REPOSITORY | sed -e's@/.*$@@') litex-hub/label/ci-$branch-$GITHUB_RUN_ID $(echo $GITHUB_REPOSITORY | sed -e's@/.*$@@')/label/ci-$branch-$GITHUB_RUN_ID" ADDITIONAL_PACKAGES="conda-build=3.20.3 conda-verify jinja2 pexpect python=3.7" -if [[ "$TRAVIS_OS_NAME" != 'windows' ]]; then +if [[ "$OS_NAME" != 'windows' ]]; then ADDITIONAL_PACKAGES="$ADDITIONAL_PACKAGES ripgrep" fi diff --git a/master-package.sh b/master-package.sh index 488eee9..879e8f4 100755 --- a/master-package.sh +++ b/master-package.sh @@ -4,7 +4,7 @@ source $GITHUB_WORKSPACE/.github/scripts/common.sh # `for ... in $(anaconda ...` fails silently if there's any problem with anaconda source $GITHUB_WORKSPACE/.github/scripts/test_anaconda.sh -branch=${GITHUB_HEAD_REF:-$CI_BRANCH} +branch="$(git rev-parse --abbrev-ref HEAD)" # Move all packages from the current label to the main label for package in $(anaconda -t $ANACONDA_TOKEN label --show ci-$branch-$GITHUB_RUN_ID 2>&1 | grep + | cut -f2 -d+) do From 9e4a7875caeeae330dc756f7e6ed7dac41ef2e8b Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Tue, 29 Dec 2020 09:30:31 +0100 Subject: [PATCH 23/47] Run cleanup-anaconda after master-package --- cleanup-anaconda.sh | 0 wait-for-statuses.py | 2 ++ 2 files changed, 2 insertions(+) mode change 100644 => 100755 cleanup-anaconda.sh diff --git a/cleanup-anaconda.sh b/cleanup-anaconda.sh old mode 100644 new mode 100755 diff --git a/wait-for-statuses.py b/wait-for-statuses.py index e450ea8..1d1a283 100644 --- a/wait-for-statuses.py +++ b/wait-for-statuses.py @@ -22,3 +22,5 @@ break subprocess.call(os.environ['GITHUB_WORKSPACE'] + "/.github/scripts/master-package.sh") +subprocess.call(os.environ['GITHUB_WORKSPACE'] + "/.github/scripts/cleanup-anaconda.sh") + From fcfb19e215b07be58f37c1f8dff096d52aec32f2 Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Tue, 29 Dec 2020 11:09:20 +0100 Subject: [PATCH 24/47] cleanup-anaconda: set missing variable --- cleanup-anaconda.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cleanup-anaconda.sh b/cleanup-anaconda.sh index c72e725..b9e0482 100755 --- a/cleanup-anaconda.sh +++ b/cleanup-anaconda.sh @@ -7,6 +7,12 @@ source $GITHUB_WORKSPACE/.github/scripts/test_anaconda.sh #if the timestamp is older than one week, remove the whole label ago="7 days ago" +if [ $OS_NAME = 'osx' ]; then + DATE_SWITCH="-r " +else + DATE_SWITCH="--date=@" +fi + #extract date in milliseconds limit_date=$(date $DATE_SWITCH "$ago" +'%s%N' | cut -b1-13) From ca3f9efc7db8929b6bda677f0586aadb95bf2fc1 Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Tue, 29 Dec 2020 12:14:57 +0100 Subject: [PATCH 25/47] Execute inline build script if it was defined in workflow --- action.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/action.yml b/action.yml index 2a89eab..bd51ba2 100644 --- a/action.yml +++ b/action.yml @@ -13,7 +13,12 @@ runs: source .github/scripts/common.sh bash .github/scripts/download_sdk.sh bash .github/scripts/install.sh - bash .github/scripts/script.sh + set -x + if [ $SCRIPT ]; then + bash $SCRIPT + else + bash .github/scripts/script.sh + fi if [ $? -eq 0 ]; then source .github/scripts/after_success.sh else From 8c7d40fd6c31cc8cbf1ea2881295aee4ffd56956 Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Thu, 31 Dec 2020 10:04:21 +0100 Subject: [PATCH 26/47] Ensure that all the other jobs were completed before We use this simple counter condition to determine if all jobs were queued and completed. Not queued jobs aren't returned in API call. --- wait-for-statuses.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/wait-for-statuses.py b/wait-for-statuses.py index 1d1a283..cecb030 100644 --- a/wait-for-statuses.py +++ b/wait-for-statuses.py @@ -10,15 +10,20 @@ + os.environ['GITHUB_RUN_ID'] \ + "/jobs" +numOfJobs = int(os.environ['NUM_OF_JOBS']) + while(True): time.sleep(60) - success = True + countCompleted = 0 + with urllib.request.urlopen(status_url) as url: data = json.loads(url.read().decode()) for j in data["jobs"]: - if(j["status"] != "completed" and j["name"] != "master-package"): - success = False - if(success): + if(j["status"] == "completed"): + countCompleted += 1 + + print("Completed jobs:" + str(countCompleted) + ". Jobs overall: " + str(numOfJobs)) + if(countCompleted >= numOfJobs): break subprocess.call(os.environ['GITHUB_WORKSPACE'] + "/.github/scripts/master-package.sh") From e118870b0f573eeddd157ef77ba443282fc28968 Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Thu, 31 Dec 2020 14:25:59 +0100 Subject: [PATCH 27/47] Export windows related paths conditionally --- action.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/action.yml b/action.yml index bd51ba2..779497e 100644 --- a/action.yml +++ b/action.yml @@ -8,8 +8,10 @@ runs: if [ $SKIP ]; then exit 0; fi - export PATH="$PATH:/c/ProgramData/Chocolatey/bin/" - export PATH="$PATH:/c/Program Files/Git/usr/bin/" + if [ $OS_NAME = "windows" ]; then + export PATH="$PATH:/c/ProgramData/Chocolatey/bin/" + export PATH="$PATH:/c/Program Files/Git/usr/bin/" + fi source .github/scripts/common.sh bash .github/scripts/download_sdk.sh bash .github/scripts/install.sh From fc2991a56b644d1f527c7cc958385b25b0698aa2 Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Thu, 31 Dec 2020 14:47:46 +0100 Subject: [PATCH 28/47] cleanup-anaconda: increase verbosity --- cleanup-anaconda.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/cleanup-anaconda.sh b/cleanup-anaconda.sh index b9e0482..cf90761 100755 --- a/cleanup-anaconda.sh +++ b/cleanup-anaconda.sh @@ -3,6 +3,7 @@ source $GITHUB_WORKSPACE/.github/scripts/common.sh # `for ... in $(anaconda ...` fails silently if there's any problem with anaconda source $GITHUB_WORKSPACE/.github/scripts/test_anaconda.sh +set -x #if the timestamp is older than one week, remove the whole label ago="7 days ago" From 8f0016546b812ec711c097c487cdbdb72455e103 Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Thu, 31 Dec 2020 15:00:48 +0100 Subject: [PATCH 29/47] cleanup-anaconda: correct date switch --- cleanup-anaconda.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cleanup-anaconda.sh b/cleanup-anaconda.sh index cf90761..60cdde3 100755 --- a/cleanup-anaconda.sh +++ b/cleanup-anaconda.sh @@ -11,7 +11,7 @@ ago="7 days ago" if [ $OS_NAME = 'osx' ]; then DATE_SWITCH="-r " else - DATE_SWITCH="--date=@" + DATE_SWITCH="--date " fi #extract date in milliseconds From 7d89aff885c11a42fa0d755615dca40f3355e4e1 Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Thu, 31 Dec 2020 15:11:28 +0100 Subject: [PATCH 30/47] cleanup-anaconda: correct cut delimiter --- cleanup-anaconda.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cleanup-anaconda.sh b/cleanup-anaconda.sh index 60cdde3..e33d498 100755 --- a/cleanup-anaconda.sh +++ b/cleanup-anaconda.sh @@ -19,7 +19,7 @@ limit_date=$(date $DATE_SWITCH "$ago" +'%s%N' | cut -b1-13) echo "Will remove labels older than $limit_date timestamp" -for label in $(anaconda -t $ANACONDA_TOKEN label --list -o litex-hub 2>&1 | grep ' + ' | cut -f2 -d1+) +for label in $(anaconda -t $ANACONDA_TOKEN label --list -o litex-hub 2>&1 | grep ' + ' | cut -f2 -d+) do if [[ $label != ci* ]] then From 1035db3b10a4160d6a13029a17443f10280973f2 Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Mon, 4 Jan 2021 16:11:55 +0100 Subject: [PATCH 31/47] ghactions-api: raise limit of jobs per page in API call Becuase limit is by default too low (28 records) we didn't get the whole list of jobs in this call. --- wait-for-statuses.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/wait-for-statuses.py b/wait-for-statuses.py index cecb030..1f173a5 100644 --- a/wait-for-statuses.py +++ b/wait-for-statuses.py @@ -4,11 +4,16 @@ import time import os +# We're limited to this number by GH Actions API +# https://docs.github.com/en/free-pro-team@latest/rest/reference/actions#list-jobs-for-a-workflow-run +max_jobs=100 + status_url = "https://api.github.com/repos/" \ + os.environ['GITHUB_REPOSITORY'] \ + "/actions/runs/" \ + os.environ['GITHUB_RUN_ID'] \ - + "/jobs" + + "/jobs" \ + + "?per_page=" + str(max_jobs) numOfJobs = int(os.environ['NUM_OF_JOBS']) From 23289cf9c0b8a497bd92691b381d242e77c22cdc Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Tue, 5 Jan 2021 09:58:18 +0100 Subject: [PATCH 32/47] Return error when anaconda/token doesn't work --- test_anaconda.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test_anaconda.sh b/test_anaconda.sh index 5556038..5e38474 100644 --- a/test_anaconda.sh +++ b/test_anaconda.sh @@ -3,18 +3,22 @@ echo if [ x$ANACONDA_TOKEN != x ]; then if ! anaconda -h &>/dev/null; then - echo 'WARNING: Missing `anaconda-client` package!' + echo 'ERROR: Missing `anaconda-client` package!' echo 'This Conda environment contains the following packages:' conda list else echo 'Testing ANACONDA_TOKEN with a simple `anaconda` call...' if anaconda -t $ANACONDA_TOKEN label --show main &>/dev/null; then echo 'ANACONDA_TOKEN works!' + exit 0 else - echo 'WARNING: Invalid ANACONDA_TOKEN!' + echo 'ERROR: Invalid ANACONDA_TOKEN!' fi fi else - echo 'WARNING: ANACONDA_TOKEN not set!' + echo 'ERROR: ANACONDA_TOKEN not set!' fi echo + +# We only get here when anaconda/token doesn't work +exit 1 From ce25d236f590d242de5cb62ae9b1f26c5613f40e Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Tue, 5 Jan 2021 10:22:40 +0100 Subject: [PATCH 33/47] test_anaconda: don't exit when token works This caused upload scripts to exit prematurely https://github.com/litex-hub/litex-conda-misc/runs/1648904572?check_suite_focus=true --- test_anaconda.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test_anaconda.sh b/test_anaconda.sh index 5e38474..4568711 100644 --- a/test_anaconda.sh +++ b/test_anaconda.sh @@ -6,19 +6,18 @@ if [ x$ANACONDA_TOKEN != x ]; then echo 'ERROR: Missing `anaconda-client` package!' echo 'This Conda environment contains the following packages:' conda list + exit 1 else echo 'Testing ANACONDA_TOKEN with a simple `anaconda` call...' if anaconda -t $ANACONDA_TOKEN label --show main &>/dev/null; then echo 'ANACONDA_TOKEN works!' - exit 0 else echo 'ERROR: Invalid ANACONDA_TOKEN!' + exit 1 fi fi else echo 'ERROR: ANACONDA_TOKEN not set!' + exit 1 fi echo - -# We only get here when anaconda/token doesn't work -exit 1 From 6aed4b1b81d7d932cac86b3705c1abe7f83037d5 Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Thu, 7 Jan 2021 09:25:09 +0100 Subject: [PATCH 34/47] Remove unused scripts --- before_cache.sh | 16 ---------------- conda-env.sh | 36 ------------------------------------ conda-meta-extra.sh | 35 ----------------------------------- 3 files changed, 87 deletions(-) delete mode 100755 before_cache.sh delete mode 100755 conda-env.sh delete mode 100755 conda-meta-extra.sh diff --git a/before_cache.sh b/before_cache.sh deleted file mode 100755 index 49edf78..0000000 --- a/before_cache.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -source $GITHUB_WORKSPACE/.github/scripts/common.sh -set -e - -start_section "conda.clean.1" "${GREEN}Clean status...${NC}" -#conda clean -s --dry-run -end_section "conda.clean.1" - -start_section "conda.clean.2" "${GREEN}Cleaning...${NC}" -conda build purge -end_section "conda.clean.2" - -start_section "conda.clean.3" "${GREEN}Clean status...${NC}" -#conda clean -s --dry-run -end_section "conda.clean.3" diff --git a/conda-env.sh b/conda-env.sh deleted file mode 100755 index 4108d04..0000000 --- a/conda-env.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -if ! which conda; then - export PATH=~/conda/bin:$PATH -fi - -export PYTHONWARNINGS=ignore::UserWarning:conda_build.environ - -if [ -z "$DATE_STR" ]; then - export DATE_NUM="$(date -u +%Y%m%d%H%M%S)" - export DATE_STR="$(date -u +%Y%m%d_%H%M%S)" - echo "Setting date number to $DATE_NUM" - echo "Setting date string to $DATE_STR" -fi -if [ -z "$GITREV" ]; then - export GITREV="$(git describe --long)" - echo "Setting git revision $GITREV" -fi - -export CI=0 - -export CI_EVENT_TYPE="local" -echo "CI_EVENT_TYPE='${CI_EVENT_TYPE}'" - -export CI_BRANCH="$(git rev-parse --abbrev-ref HEAD)" -echo "CI_BRANCH='${CI_BRANCH}'" - -export CI_COMMIT="$(git rev-parse HEAD)" -echo "CI_COMMIT='${CI_COMMIT}'" - -export CI_REPO_SLUG="$(git rev-parse --abbrev-ref --symbolic-full-name @{u})" -echo "CI_REPO_SLUG='${CI_REPO_SLUG}'" - -$GITHUB_WORKSPACE/.github/scripts/conda-meta-extra.sh -echo conda $@ -conda $@ diff --git a/conda-meta-extra.sh b/conda-meta-extra.sh deleted file mode 100755 index d3c61cf..0000000 --- a/conda-meta-extra.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -PACKAGE=${1:-PACKAGE} -if [ x$PACKAGE = x"" ]; then - echo "\$PACKAGE not set!" - exit 1 -fi - -rm -f $PACKAGE/recipe_append.yaml -cat > $PACKAGE/recipe_append.yaml < - - HDMI2USB Project - https://hdmi2usb.tv - ci: - job_id: $GITHUB_RUN_ID - job_num: $GITHUB_RUN_NUMBER - type: $CI_EVENT_TYPE - recipe: - repo: 'https://github.com/$CI_REPO_SLUG' - branch: $CI_BRANCH - commit: $CI_COMMIT - describe: $GITREV - date: $DATESTR -EOF -if [ ! -z "${TOOLCHAIN_ARCH}" ]; then - cat >> recipe_append.yaml <> $PACKAGE/recipe_append.yaml - cat $PACKAGE_CONDARC | sed -e's/^/ /' >> $PACKAGE/recipe_append.yaml -fi From b24e4814dc40b89f30a3afc625076b1561d0c6c8 Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Thu, 7 Jan 2021 09:25:30 +0100 Subject: [PATCH 35/47] wip: check if we are inside correct environment Since we are sourcing common.sh we should already be inside Conda environment with pexpect installed --- script.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/script.sh b/script.sh index 9132dfe..49e4e2e 100755 --- a/script.sh +++ b/script.sh @@ -17,15 +17,10 @@ end_section "conda.check" $SPACER -echo "Install pexpect" -if [ -f $CONDA_PATH/bin/python ]; then - $CONDA_PATH/bin/python -m pip install pexpect -fi - start_section "conda.build" "${GREEN}Building..${NC}" if [[ $OS_NAME != 'windows' ]]; then if [[ $KEEP_ALIVE = 'true' ]]; then - ci_wait $CI_MAX_TIME $CONDA_PATH/bin/python $GITHUB_WORKSPACE/.github/scripts/.ci-output.py /tmp/output.log conda build $CONDA_BUILD_ARGS + ci_wait $CI_MAX_TIME python $GITHUB_WORKSPACE/.github/scripts/.ci-output.py /tmp/output.log conda build $CONDA_BUILD_ARGS else python $GITHUB_WORKSPACE/.github/scripts/.ci-output.py /tmp/output.log conda build $CONDA_BUILD_ARGS fi From ced63477472bbb32dd62fa8be42a202e89181965 Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Thu, 7 Jan 2021 09:29:08 +0100 Subject: [PATCH 36/47] Simplify path used to check Conda env existance --- common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common.sh b/common.sh index f607e1b..636c453 100644 --- a/common.sh +++ b/common.sh @@ -114,7 +114,7 @@ else export CONDA_ENV='workdir/conda-env' fi -if [ -d "$CONDA_ENV/conda-meta" ] && which conda &>/dev/null; then +if [ -d "$CONDA_ENV" ] && which conda &>/dev/null; then # >>> conda initialize >>> eval "$('conda' 'shell.bash' 'hook' 2> /dev/null)" # <<< conda initialize <<< From 173f2df65afa362b944170abc453618592e0d48a Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Thu, 7 Jan 2021 09:30:51 +0100 Subject: [PATCH 37/47] common.sh: remove code repetition --- cleanup-anaconda.sh | 6 ------ common.sh | 4 ++-- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/cleanup-anaconda.sh b/cleanup-anaconda.sh index e33d498..5155752 100755 --- a/cleanup-anaconda.sh +++ b/cleanup-anaconda.sh @@ -8,12 +8,6 @@ set -x #if the timestamp is older than one week, remove the whole label ago="7 days ago" -if [ $OS_NAME = 'osx' ]; then - DATE_SWITCH="-r " -else - DATE_SWITCH="--date " -fi - #extract date in milliseconds limit_date=$(date $DATE_SWITCH "$ago" +'%s%N' | cut -b1-13) diff --git a/common.sh b/common.sh index 636c453..52a6445 100644 --- a/common.sh +++ b/common.sh @@ -68,9 +68,9 @@ ci_jigger() { } if [ $OS_NAME = 'osx' ]; then - DATE_SWITCH="-r " + export DATE_SWITCH="-r " else - DATE_SWITCH="--date=@" + export DATE_SWITCH="--date=@" fi if [ -z "$DATE_STR" ]; then export DATE_TS="$(git log --format=%ct -n1)" From b3da438e21e6528be261746a1b7ead0a7443a10c Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Thu, 7 Jan 2021 09:32:32 +0100 Subject: [PATCH 38/47] Update README.md --- README.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1a94fb4..f00a5f0 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,15 @@ # litex-conda-ci -Scripts for Litex-Hub/litex-conda-(compilers|eda|misc|prog) Continuous Integration. +Travis scripts for Litex-Hub/litex-conda-(compilers|eda|misc|prog) . This repository is not meant to be used on its own, but as submodule for above repositories. + +## Setting up Github Actions for your own fork + +To use any of the litex-conda-* repositories in Github Actions you must meet the following requirements: + +* create an account on https://anaconda.org/ +* create an access token on https://anaconda.org/{your-account}/settings/access +* follow https://docs.github.com/en/free-pro-team@latest/actions/reference/encrypted-secrets#creating-encrypted-secrets-for-an-environment to add two encrypted secrets: + * `ANACONDA_USER` - set to your anaconda username + * `ANACONDA_TOKEN` - set to the value of the token you created From 25ff50839e50a46b4e973acf0a6ae28472a71473 Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Thu, 7 Jan 2021 09:47:08 +0100 Subject: [PATCH 39/47] Return an error when number of jobs exceeds max_jobs ghactions API call we're using limits number of jobs returned in one call. If jobs exceed this number they are grouped into "pages". Page handling code shall be added when we exceed this number. --- wait-for-statuses.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/wait-for-statuses.py b/wait-for-statuses.py index 1f173a5..cf6f69a 100644 --- a/wait-for-statuses.py +++ b/wait-for-statuses.py @@ -3,6 +3,7 @@ import subprocess import time import os +import sys # We're limited to this number by GH Actions API # https://docs.github.com/en/free-pro-team@latest/rest/reference/actions#list-jobs-for-a-workflow-run @@ -17,6 +18,10 @@ numOfJobs = int(os.environ['NUM_OF_JOBS']) + +if(numOfJobs > max_jobs): + sys.exit("ERROR: number of jobs exceeded max_jobs: " + str(max_jobs)) + while(True): time.sleep(60) countCompleted = 0 From e35959c316289885a95a1b51372f322ee4b1f62e Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Thu, 7 Jan 2021 09:56:49 +0100 Subject: [PATCH 40/47] Use ghactions ::group:: mechanism wherever travis_fold was used These functions split the output logs into foldable groups --- after_failure.sh | 3 +++ after_success.sh | 3 +++ common.sh | 2 ++ 3 files changed, 8 insertions(+) diff --git a/after_failure.sh b/after_failure.sh index 0ecd5ec..8064894 100755 --- a/after_failure.sh +++ b/after_failure.sh @@ -2,6 +2,9 @@ source $GITHUB_WORKSPACE/.github/scripts/common.sh +# Close the after_failure fold +echo ::endgroup:: + $SPACER start_section "failure.tail" "${RED}Failure output...${NC}" diff --git a/after_success.sh b/after_success.sh index 9579d3a..9ea1e0c 100755 --- a/after_success.sh +++ b/after_success.sh @@ -3,6 +3,9 @@ source $GITHUB_WORKSPACE/.github/scripts/common.sh set -e +# Close the after_success fold +echo ::endgroup:: + if [[ $UPLOAD == "no-upload" ]]; then echo "Job without upload..." else diff --git a/common.sh b/common.sh index 52a6445..cf81475 100644 --- a/common.sh +++ b/common.sh @@ -81,12 +81,14 @@ if [ -z "$DATE_STR" ]; then fi function start_section() { + echo "::group::$1" echo -e "${PURPLE}${PACKAGE}${NC}: - $2${NC}" echo -e "${GRAY}-------------------------------------------------------------------${NC}" } function end_section() { echo -e "${GRAY}-------------------------------------------------------------------${NC}" + echo ::endgroup:: } export PYTHONWARNINGS=ignore::UserWarning:conda_build.environ From dc0ee40b475f806fe1d2af9d9a188e914375035c Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Thu, 7 Jan 2021 09:59:00 +0100 Subject: [PATCH 41/47] after_failure: remove commented out code --- after_failure.sh | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/after_failure.sh b/after_failure.sh index 8064894..4fc0337 100755 --- a/after_failure.sh +++ b/after_failure.sh @@ -16,16 +16,6 @@ end_section "failure.tail" $SPACER -#COUNT=0 -#for i in $(find $BASE_PATH -name config.log); do -# start_section "failure.log.$COUNT" "${RED}Failure output $i...${NC}" -# cat $i -# end_section "failure.log.$COUNT" -# COUNT=$((COUNT+1)) -#done -# -#$SPACER - start_section "failure.log.full" "${RED}Failure output.log...${NC}" cat /tmp/output.log end_section "failure.log.full" From 3d0779dfd618741aa4a2b9b32d5356ab01f65f4f Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Thu, 7 Jan 2021 10:07:10 +0100 Subject: [PATCH 42/47] Look for failure statuses in workflow jobs and act accordingly --- wait-for-statuses.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/wait-for-statuses.py b/wait-for-statuses.py index cf6f69a..c17e80c 100644 --- a/wait-for-statuses.py +++ b/wait-for-statuses.py @@ -22,6 +22,8 @@ if(numOfJobs > max_jobs): sys.exit("ERROR: number of jobs exceeded max_jobs: " + str(max_jobs)) +jobFailure = False + while(True): time.sleep(60) countCompleted = 0 @@ -31,11 +33,17 @@ for j in data["jobs"]: if(j["status"] == "completed"): countCompleted += 1 + if(j["conclusion"] == "failure"): + jobFailure = True print("Completed jobs:" + str(countCompleted) + ". Jobs overall: " + str(numOfJobs)) if(countCompleted >= numOfJobs): break -subprocess.call(os.environ['GITHUB_WORKSPACE'] + "/.github/scripts/master-package.sh") +# Upload packages only when whole build succeeded +if(not jobFailure): + subprocess.call(os.environ['GITHUB_WORKSPACE'] + "/.github/scripts/master-package.sh") + +# Always clean up subprocess.call(os.environ['GITHUB_WORKSPACE'] + "/.github/scripts/cleanup-anaconda.sh") From c404f2068577ad1cfa03b67aef3bf8ec245b69d1 Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Thu, 7 Jan 2021 10:09:32 +0100 Subject: [PATCH 43/47] wait-for-statuses: apply more pythonic path joining --- wait-for-statuses.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/wait-for-statuses.py b/wait-for-statuses.py index c17e80c..a8b94b2 100644 --- a/wait-for-statuses.py +++ b/wait-for-statuses.py @@ -18,7 +18,6 @@ numOfJobs = int(os.environ['NUM_OF_JOBS']) - if(numOfJobs > max_jobs): sys.exit("ERROR: number of jobs exceeded max_jobs: " + str(max_jobs)) @@ -42,8 +41,9 @@ # Upload packages only when whole build succeeded if(not jobFailure): - subprocess.call(os.environ['GITHUB_WORKSPACE'] + "/.github/scripts/master-package.sh") + subprocess.call(os.path.join(os.environ['GITHUB_WORKSPACE'], + ".github/scripts/master-package.sh")) # Always clean up -subprocess.call(os.environ['GITHUB_WORKSPACE'] + "/.github/scripts/cleanup-anaconda.sh") - +subprocess.call(os.path.join(os.environ['GITHUB_WORKSPACE'], + ".github/scripts/cleanup-anaconda.sh")) From 99e8cd399d983f108ef3d019e198ecc802d57d8a Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Thu, 7 Jan 2021 10:10:21 +0100 Subject: [PATCH 44/47] wait-for-statuses: fix a typo --- wait-for-statuses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wait-for-statuses.py b/wait-for-statuses.py index a8b94b2..eaa245b 100644 --- a/wait-for-statuses.py +++ b/wait-for-statuses.py @@ -35,7 +35,7 @@ if(j["conclusion"] == "failure"): jobFailure = True - print("Completed jobs:" + str(countCompleted) + ". Jobs overall: " + str(numOfJobs)) + print("Completed jobs: " + str(countCompleted) + ". Jobs overall: " + str(numOfJobs)) if(countCompleted >= numOfJobs): break From 735fdba827c0367a5fe0ebe148f211754be50efc Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Thu, 7 Jan 2021 14:03:02 +0100 Subject: [PATCH 45/47] wait-for-statuses: add missing failure handling --- wait-for-statuses.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/wait-for-statuses.py b/wait-for-statuses.py index eaa245b..b15c477 100644 --- a/wait-for-statuses.py +++ b/wait-for-statuses.py @@ -47,3 +47,6 @@ # Always clean up subprocess.call(os.path.join(os.environ['GITHUB_WORKSPACE'], ".github/scripts/cleanup-anaconda.sh")) + +if(jobFailure): + sys.exit("ERROR: some jobs failed") From d188f6875ac2bc0adfa22c23ea57ddbae6fdb7ce Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Thu, 7 Jan 2021 14:08:52 +0100 Subject: [PATCH 46/47] wait-for-statuses: treat everything but success as failure --- wait-for-statuses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wait-for-statuses.py b/wait-for-statuses.py index b15c477..d5277e4 100644 --- a/wait-for-statuses.py +++ b/wait-for-statuses.py @@ -32,7 +32,7 @@ for j in data["jobs"]: if(j["status"] == "completed"): countCompleted += 1 - if(j["conclusion"] == "failure"): + if(j["conclusion"] != "success"): jobFailure = True print("Completed jobs: " + str(countCompleted) + ". Jobs overall: " + str(numOfJobs)) From 76961316a7195c3f2a09053afb35f405aa0754f9 Mon Sep 17 00:00:00 2001 From: Tomasz Jurtsch Date: Thu, 7 Jan 2021 16:07:54 +0100 Subject: [PATCH 47/47] wait-for-statuses: look for failing jobs when all jobs completed --- wait-for-statuses.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/wait-for-statuses.py b/wait-for-statuses.py index d5277e4..969bac0 100644 --- a/wait-for-statuses.py +++ b/wait-for-statuses.py @@ -21,7 +21,6 @@ if(numOfJobs > max_jobs): sys.exit("ERROR: number of jobs exceeded max_jobs: " + str(max_jobs)) -jobFailure = False while(True): time.sleep(60) @@ -32,13 +31,21 @@ for j in data["jobs"]: if(j["status"] == "completed"): countCompleted += 1 - if(j["conclusion"] != "success"): - jobFailure = True print("Completed jobs: " + str(countCompleted) + ". Jobs overall: " + str(numOfJobs)) if(countCompleted >= numOfJobs): break +# Check if all jobs succeeded +jobFailure = False +with urllib.request.urlopen(status_url) as url: + data = json.loads(url.read().decode()) + for j in data["jobs"]: + # THIS is master-package job, still in progress (not concluded) + if(j["conclusion"] != "success" and j["name"] != "master-package"): + jobFailure = True + break + # Upload packages only when whole build succeeded if(not jobFailure): subprocess.call(os.path.join(os.environ['GITHUB_WORKSPACE'],