From 8b7a794f1db8287189caec6b1f1aad83efe796ed Mon Sep 17 00:00:00 2001 From: Alex Hornby Date: Mon, 6 Jan 2025 11:16:30 -0800 Subject: [PATCH] skip unnecessary github actions steps Summary: X-link: https://github.com/facebookincubator/zstrong/pull/1100 Update generated github actions to only run the fetch and and build steps when there are sources expected for a manifest For local github actions testing using `act` this speeds up the test runs, and in real github CI it makes it clearer which steps are actually doing something on the given runner (we don't know exactly what it has installed beforehand) Also set the windows git config the same as on internal CI Reviewed By: bigfootjon Differential Revision: D67839708 fbshipit-source-id: 0a60c6fc89e8c6abb2464f879459aa23d5aec969 --- .github/workflows/getdeps_linux.yml | 59 ++++++++++++- .github/workflows/getdeps_mac.yml | 51 +++++++++++ .github/workflows/getdeps_windows.yml | 56 +++++++++++- build/fbcode_builder/getdeps.py | 111 ++++++++++++++++-------- build/fbcode_builder/getdeps/fetcher.py | 4 +- 5 files changed, 240 insertions(+), 41 deletions(-) diff --git a/.github/workflows/getdeps_linux.yml b/.github/workflows/getdeps_linux.yml index f0d010ba7..230c307c3 100644 --- a/.github/workflows/getdeps_linux.yml +++ b/.github/workflows/getdeps_linux.yml @@ -21,112 +21,165 @@ jobs: - name: Update system package info run: sudo apt-get update - name: Install system deps - run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive mvfst - - name: Install packaging system deps - run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive patchelf + run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive mvfst && sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive patchelf + - id: paths + name: Query paths + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages query-paths --recursive --src-dir=. mvfst >> "$GITHUB_OUTPUT" - name: Fetch ninja + if: ${{ steps.paths.outputs.ninja_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests ninja - name: Fetch cmake + if: ${{ steps.paths.outputs.cmake_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests cmake - name: Fetch zlib + if: ${{ steps.paths.outputs.zlib_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests zlib - name: Fetch zstd + if: ${{ steps.paths.outputs.zstd_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests zstd - name: Fetch boost + if: ${{ steps.paths.outputs.boost_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests boost - name: Fetch double-conversion + if: ${{ steps.paths.outputs.double-conversion_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests double-conversion - name: Fetch fast_float + if: ${{ steps.paths.outputs.fast_float_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fast_float - name: Fetch fmt + if: ${{ steps.paths.outputs.fmt_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fmt - name: Fetch gflags + if: ${{ steps.paths.outputs.gflags_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests gflags - name: Fetch glog + if: ${{ steps.paths.outputs.glog_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests glog - name: Fetch googletest + if: ${{ steps.paths.outputs.googletest_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests googletest - name: Fetch libdwarf + if: ${{ steps.paths.outputs.libdwarf_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libdwarf - name: Fetch libevent + if: ${{ steps.paths.outputs.libevent_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libevent - name: Fetch lz4 + if: ${{ steps.paths.outputs.lz4_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests lz4 - name: Fetch snappy + if: ${{ steps.paths.outputs.snappy_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests snappy - name: Fetch openssl + if: ${{ steps.paths.outputs.openssl_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests openssl - name: Fetch liboqs + if: ${{ steps.paths.outputs.liboqs_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests liboqs - name: Fetch autoconf + if: ${{ steps.paths.outputs.autoconf_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests autoconf - name: Fetch automake + if: ${{ steps.paths.outputs.automake_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests automake - name: Fetch libtool + if: ${{ steps.paths.outputs.libtool_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libtool - name: Fetch libsodium + if: ${{ steps.paths.outputs.libsodium_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libsodium - name: Fetch libiberty + if: ${{ steps.paths.outputs.libiberty_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libiberty - name: Fetch libunwind + if: ${{ steps.paths.outputs.libunwind_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libunwind - name: Fetch xz + if: ${{ steps.paths.outputs.xz_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests xz - name: Fetch folly + if: ${{ steps.paths.outputs.folly_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests folly - name: Fetch fizz + if: ${{ steps.paths.outputs.fizz_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fizz - name: Build ninja + if: ${{ steps.paths.outputs.ninja_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests ninja - name: Build cmake + if: ${{ steps.paths.outputs.cmake_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests cmake - name: Build zlib + if: ${{ steps.paths.outputs.zlib_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests zlib - name: Build zstd + if: ${{ steps.paths.outputs.zstd_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests zstd - name: Build boost + if: ${{ steps.paths.outputs.boost_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests boost - name: Build double-conversion + if: ${{ steps.paths.outputs.double-conversion_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests double-conversion - name: Build fast_float + if: ${{ steps.paths.outputs.fast_float_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fast_float - name: Build fmt + if: ${{ steps.paths.outputs.fmt_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fmt - name: Build gflags + if: ${{ steps.paths.outputs.gflags_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests gflags - name: Build glog + if: ${{ steps.paths.outputs.glog_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests glog - name: Build googletest + if: ${{ steps.paths.outputs.googletest_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests googletest - name: Build libdwarf + if: ${{ steps.paths.outputs.libdwarf_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libdwarf - name: Build libevent + if: ${{ steps.paths.outputs.libevent_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libevent - name: Build lz4 + if: ${{ steps.paths.outputs.lz4_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests lz4 - name: Build snappy + if: ${{ steps.paths.outputs.snappy_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests snappy - name: Build openssl + if: ${{ steps.paths.outputs.openssl_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests openssl - name: Build liboqs + if: ${{ steps.paths.outputs.liboqs_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests liboqs - name: Build autoconf + if: ${{ steps.paths.outputs.autoconf_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests autoconf - name: Build automake + if: ${{ steps.paths.outputs.automake_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests automake - name: Build libtool + if: ${{ steps.paths.outputs.libtool_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libtool - name: Build libsodium + if: ${{ steps.paths.outputs.libsodium_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libsodium - name: Build libiberty + if: ${{ steps.paths.outputs.libiberty_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libiberty - name: Build libunwind + if: ${{ steps.paths.outputs.libunwind_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libunwind - name: Build xz + if: ${{ steps.paths.outputs.xz_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests xz - name: Build folly + if: ${{ steps.paths.outputs.folly_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests folly - name: Build fizz + if: ${{ steps.paths.outputs.fizz_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fizz - name: Build mvfst run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --src-dir=. mvfst --project-install-prefix mvfst:/usr/local diff --git a/.github/workflows/getdeps_mac.yml b/.github/workflows/getdeps_mac.yml index a5462f1ad..0a2959192 100644 --- a/.github/workflows/getdeps_mac.yml +++ b/.github/workflows/getdeps_mac.yml @@ -20,101 +20,152 @@ jobs: - uses: actions/checkout@v4 - name: Install system deps run: python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive mvfst + - id: paths + name: Query paths + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages query-paths --recursive --src-dir=. mvfst >> "$GITHUB_OUTPUT" - name: Fetch ninja + if: ${{ steps.paths.outputs.ninja_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests ninja - name: Fetch cmake + if: ${{ steps.paths.outputs.cmake_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests cmake - name: Fetch zlib + if: ${{ steps.paths.outputs.zlib_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests zlib - name: Fetch zstd + if: ${{ steps.paths.outputs.zstd_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests zstd - name: Fetch boost + if: ${{ steps.paths.outputs.boost_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests boost - name: Fetch double-conversion + if: ${{ steps.paths.outputs.double-conversion_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests double-conversion - name: Fetch fast_float + if: ${{ steps.paths.outputs.fast_float_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fast_float - name: Fetch fmt + if: ${{ steps.paths.outputs.fmt_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fmt - name: Fetch gflags + if: ${{ steps.paths.outputs.gflags_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests gflags - name: Fetch glog + if: ${{ steps.paths.outputs.glog_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests glog - name: Fetch googletest + if: ${{ steps.paths.outputs.googletest_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests googletest - name: Fetch libdwarf + if: ${{ steps.paths.outputs.libdwarf_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libdwarf - name: Fetch lz4 + if: ${{ steps.paths.outputs.lz4_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests lz4 - name: Fetch openssl + if: ${{ steps.paths.outputs.openssl_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests openssl - name: Fetch snappy + if: ${{ steps.paths.outputs.snappy_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests snappy - name: Fetch libevent + if: ${{ steps.paths.outputs.libevent_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libevent - name: Fetch liboqs + if: ${{ steps.paths.outputs.liboqs_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests liboqs - name: Fetch autoconf + if: ${{ steps.paths.outputs.autoconf_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests autoconf - name: Fetch automake + if: ${{ steps.paths.outputs.automake_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests automake - name: Fetch libtool + if: ${{ steps.paths.outputs.libtool_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libtool - name: Fetch libsodium + if: ${{ steps.paths.outputs.libsodium_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libsodium - name: Fetch xz + if: ${{ steps.paths.outputs.xz_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests xz - name: Fetch folly + if: ${{ steps.paths.outputs.folly_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests folly - name: Fetch fizz + if: ${{ steps.paths.outputs.fizz_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fizz - name: Build ninja + if: ${{ steps.paths.outputs.ninja_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests ninja - name: Build cmake + if: ${{ steps.paths.outputs.cmake_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests cmake - name: Build zlib + if: ${{ steps.paths.outputs.zlib_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests zlib - name: Build zstd + if: ${{ steps.paths.outputs.zstd_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests zstd - name: Build boost + if: ${{ steps.paths.outputs.boost_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests boost - name: Build double-conversion + if: ${{ steps.paths.outputs.double-conversion_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests double-conversion - name: Build fast_float + if: ${{ steps.paths.outputs.fast_float_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fast_float - name: Build fmt + if: ${{ steps.paths.outputs.fmt_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fmt - name: Build gflags + if: ${{ steps.paths.outputs.gflags_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests gflags - name: Build glog + if: ${{ steps.paths.outputs.glog_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests glog - name: Build googletest + if: ${{ steps.paths.outputs.googletest_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests googletest - name: Build libdwarf + if: ${{ steps.paths.outputs.libdwarf_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libdwarf - name: Build lz4 + if: ${{ steps.paths.outputs.lz4_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests lz4 - name: Build openssl + if: ${{ steps.paths.outputs.openssl_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests openssl - name: Build snappy + if: ${{ steps.paths.outputs.snappy_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests snappy - name: Build libevent + if: ${{ steps.paths.outputs.libevent_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libevent - name: Build liboqs + if: ${{ steps.paths.outputs.liboqs_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests liboqs - name: Build autoconf + if: ${{ steps.paths.outputs.autoconf_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests autoconf - name: Build automake + if: ${{ steps.paths.outputs.automake_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests automake - name: Build libtool + if: ${{ steps.paths.outputs.libtool_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libtool - name: Build libsodium + if: ${{ steps.paths.outputs.libsodium_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libsodium - name: Build xz + if: ${{ steps.paths.outputs.xz_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests xz - name: Build folly + if: ${{ steps.paths.outputs.folly_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests folly - name: Build fizz + if: ${{ steps.paths.outputs.fizz_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fizz - name: Build mvfst run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --src-dir=. mvfst --project-install-prefix mvfst:/usr/local diff --git a/.github/workflows/getdeps_windows.yml b/.github/workflows/getdeps_windows.yml index 215e787d2..33b7227fa 100644 --- a/.github/workflows/getdeps_windows.yml +++ b/.github/workflows/getdeps_windows.yml @@ -21,97 +21,147 @@ jobs: run: "echo BOOST_ROOT=%BOOST_ROOT_1_83_0% >> %GITHUB_ENV%" shell: cmd - name: Fix Git config - run: git config --system core.longpaths true - - name: Disable autocrlf - run: git config --system core.autocrlf false + run: > + git config --system core.longpaths true && + git config --system core.autocrlf false && + git config --system core.symlinks true + shell: cmd - uses: actions/checkout@v4 + - id: paths + name: Query paths + run: python build/fbcode_builder/getdeps.py query-paths --recursive --src-dir=. mvfst >> $env:GITHUB_OUTPUT + shell: pwsh - name: Fetch ninja + if: ${{ steps.paths.outputs.ninja_SOURCE }} run: python build/fbcode_builder/getdeps.py fetch --no-tests ninja - name: Fetch cmake + if: ${{ steps.paths.outputs.cmake_SOURCE }} run: python build/fbcode_builder/getdeps.py fetch --no-tests cmake - name: Fetch libsodium + if: ${{ steps.paths.outputs.libsodium_SOURCE }} run: python build/fbcode_builder/getdeps.py fetch --no-tests libsodium - name: Fetch zlib + if: ${{ steps.paths.outputs.zlib_SOURCE }} run: python build/fbcode_builder/getdeps.py fetch --no-tests zlib - name: Fetch zstd + if: ${{ steps.paths.outputs.zstd_SOURCE }} run: python build/fbcode_builder/getdeps.py fetch --no-tests zstd - name: Fetch boost + if: ${{ steps.paths.outputs.boost_SOURCE }} run: python build/fbcode_builder/getdeps.py fetch --no-tests boost - name: Fetch double-conversion + if: ${{ steps.paths.outputs.double-conversion_SOURCE }} run: python build/fbcode_builder/getdeps.py fetch --no-tests double-conversion - name: Fetch fast_float + if: ${{ steps.paths.outputs.fast_float_SOURCE }} run: python build/fbcode_builder/getdeps.py fetch --no-tests fast_float - name: Fetch fmt + if: ${{ steps.paths.outputs.fmt_SOURCE }} run: python build/fbcode_builder/getdeps.py fetch --no-tests fmt - name: Fetch gflags + if: ${{ steps.paths.outputs.gflags_SOURCE }} run: python build/fbcode_builder/getdeps.py fetch --no-tests gflags - name: Fetch glog + if: ${{ steps.paths.outputs.glog_SOURCE }} run: python build/fbcode_builder/getdeps.py fetch --no-tests glog - name: Fetch googletest + if: ${{ steps.paths.outputs.googletest_SOURCE }} run: python build/fbcode_builder/getdeps.py fetch --no-tests googletest - name: Fetch libdwarf + if: ${{ steps.paths.outputs.libdwarf_SOURCE }} run: python build/fbcode_builder/getdeps.py fetch --no-tests libdwarf - name: Fetch lz4 + if: ${{ steps.paths.outputs.lz4_SOURCE }} run: python build/fbcode_builder/getdeps.py fetch --no-tests lz4 - name: Fetch snappy + if: ${{ steps.paths.outputs.snappy_SOURCE }} run: python build/fbcode_builder/getdeps.py fetch --no-tests snappy - name: Fetch jom + if: ${{ steps.paths.outputs.jom_SOURCE }} run: python build/fbcode_builder/getdeps.py fetch --no-tests jom - name: Fetch perl + if: ${{ steps.paths.outputs.perl_SOURCE }} run: python build/fbcode_builder/getdeps.py fetch --no-tests perl - name: Fetch openssl + if: ${{ steps.paths.outputs.openssl_SOURCE }} run: python build/fbcode_builder/getdeps.py fetch --no-tests openssl - name: Fetch libevent + if: ${{ steps.paths.outputs.libevent_SOURCE }} run: python build/fbcode_builder/getdeps.py fetch --no-tests libevent - name: Fetch folly + if: ${{ steps.paths.outputs.folly_SOURCE }} run: python build/fbcode_builder/getdeps.py fetch --no-tests folly - name: Fetch liboqs + if: ${{ steps.paths.outputs.liboqs_SOURCE }} run: python build/fbcode_builder/getdeps.py fetch --no-tests liboqs - name: Fetch fizz + if: ${{ steps.paths.outputs.fizz_SOURCE }} run: python build/fbcode_builder/getdeps.py fetch --no-tests fizz - name: Build ninja + if: ${{ steps.paths.outputs.ninja_SOURCE }} run: python build/fbcode_builder/getdeps.py build --no-tests ninja - name: Build cmake + if: ${{ steps.paths.outputs.cmake_SOURCE }} run: python build/fbcode_builder/getdeps.py build --no-tests cmake - name: Build libsodium + if: ${{ steps.paths.outputs.libsodium_SOURCE }} run: python build/fbcode_builder/getdeps.py build --no-tests libsodium - name: Build zlib + if: ${{ steps.paths.outputs.zlib_SOURCE }} run: python build/fbcode_builder/getdeps.py build --no-tests zlib - name: Build zstd + if: ${{ steps.paths.outputs.zstd_SOURCE }} run: python build/fbcode_builder/getdeps.py build --no-tests zstd - name: Build boost + if: ${{ steps.paths.outputs.boost_SOURCE }} run: python build/fbcode_builder/getdeps.py build --no-tests boost - name: Build double-conversion + if: ${{ steps.paths.outputs.double-conversion_SOURCE }} run: python build/fbcode_builder/getdeps.py build --no-tests double-conversion - name: Build fast_float + if: ${{ steps.paths.outputs.fast_float_SOURCE }} run: python build/fbcode_builder/getdeps.py build --no-tests fast_float - name: Build fmt + if: ${{ steps.paths.outputs.fmt_SOURCE }} run: python build/fbcode_builder/getdeps.py build --no-tests fmt - name: Build gflags + if: ${{ steps.paths.outputs.gflags_SOURCE }} run: python build/fbcode_builder/getdeps.py build --no-tests gflags - name: Build glog + if: ${{ steps.paths.outputs.glog_SOURCE }} run: python build/fbcode_builder/getdeps.py build --no-tests glog - name: Build googletest + if: ${{ steps.paths.outputs.googletest_SOURCE }} run: python build/fbcode_builder/getdeps.py build --no-tests googletest - name: Build libdwarf + if: ${{ steps.paths.outputs.libdwarf_SOURCE }} run: python build/fbcode_builder/getdeps.py build --no-tests libdwarf - name: Build lz4 + if: ${{ steps.paths.outputs.lz4_SOURCE }} run: python build/fbcode_builder/getdeps.py build --no-tests lz4 - name: Build snappy + if: ${{ steps.paths.outputs.snappy_SOURCE }} run: python build/fbcode_builder/getdeps.py build --no-tests snappy - name: Build jom + if: ${{ steps.paths.outputs.jom_SOURCE }} run: python build/fbcode_builder/getdeps.py build --no-tests jom - name: Build perl + if: ${{ steps.paths.outputs.perl_SOURCE }} run: python build/fbcode_builder/getdeps.py build --no-tests perl - name: Build openssl + if: ${{ steps.paths.outputs.openssl_SOURCE }} run: python build/fbcode_builder/getdeps.py build --no-tests openssl - name: Build libevent + if: ${{ steps.paths.outputs.libevent_SOURCE }} run: python build/fbcode_builder/getdeps.py build --no-tests libevent - name: Build folly + if: ${{ steps.paths.outputs.folly_SOURCE }} run: python build/fbcode_builder/getdeps.py build --no-tests folly - name: Build liboqs + if: ${{ steps.paths.outputs.liboqs_SOURCE }} run: python build/fbcode_builder/getdeps.py build --no-tests liboqs - name: Build fizz + if: ${{ steps.paths.outputs.fizz_SOURCE }} run: python build/fbcode_builder/getdeps.py build --no-tests fizz - name: Build mvfst run: python build/fbcode_builder/getdeps.py build --src-dir=. mvfst diff --git a/build/fbcode_builder/getdeps.py b/build/fbcode_builder/getdeps.py index 03a5f3c04..138e729e7 100755 --- a/build/fbcode_builder/getdeps.py +++ b/build/fbcode_builder/getdeps.py @@ -552,6 +552,33 @@ def setup_project_cmd_parser(self, parser): ) +@cmd("query-paths", "print the paths for tooling to use") +class QueryPathsCmd(ProjectCmdBase): + def run_project_cmd(self, args, loader, manifest): + if args.recursive: + manifests = loader.manifests_in_dependency_order() + else: + manifests = [manifest] + + for m in manifests: + fetcher = loader.create_fetcher(m) + if isinstance(fetcher, SystemPackageFetcher): + # We are guaranteed that if the fetcher is set to + # SystemPackageFetcher then this item is completely + # satisfied by the appropriate system packages + continue + src_dir = fetcher.get_src_dir() + print(f"{m.name}_SOURCE={src_dir}") + + def setup_project_cmd_parser(self, parser): + parser.add_argument( + "--recursive", + help="print the transitive deps also", + action="store_true", + default=False, + ) + + @cmd("show-source-dir", "print the source dir for a given project") class ShowSourceDirCmd(ProjectCmdBase): def run_project_cmd(self, args, loader, manifest): @@ -1001,6 +1028,10 @@ def write_job_for_platform(self, platform, args): # noqa: C901 manifest_ctx.set("test", "on") run_on = self.get_run_on(args) + tests_arg = "--no-tests " + if run_tests: + tests_arg = "" + # Some projects don't do anything "useful" as a leaf project, only # as a dep for a leaf project. Check for those here; we don't want # to waste the effort scheduling them on CI. @@ -1086,12 +1117,14 @@ def write_job_for_platform(self, platform, args): # noqa: C901 ) out.write(" shell: cmd\n") - # The git installation may not like long filenames, so tell it - # that we want it to use them! out.write(" - name: Fix Git config\n") - out.write(" run: git config --system core.longpaths true\n") - out.write(" - name: Disable autocrlf\n") - out.write(" run: git config --system core.autocrlf false\n") + out.write(" run: >\n") + out.write(" git config --system core.longpaths true &&\n") + out.write(" git config --system core.autocrlf false &&\n") + # cxx crate needs symlinks enabled + out.write(" git config --system core.symlinks true\n") + # && is not supported on default windows powershell, so use cmd + out.write(" shell: cmd\n") out.write(" - uses: actions/checkout@v4\n") @@ -1127,17 +1160,12 @@ def write_job_for_platform(self, platform, args): # noqa: C901 if build_opts.is_darwin(): # brew is installed as regular user sudo_arg = "" - tests_arg = "--no-tests " - if run_tests: - tests_arg = "" - out.write( - f" run: {sudo_arg}python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps {tests_arg}--recursive {manifest.name}\n" - ) + + system_deps_cmd = f"{sudo_arg}{getdepscmd}{allow_sys_arg} install-system-deps {tests_arg}--recursive {manifest.name}" if build_opts.is_linux() or build_opts.is_freebsd(): - out.write(" - name: Install packaging system deps\n") - out.write( - f" run: {sudo_arg}python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps {tests_arg}--recursive patchelf\n" - ) + system_deps_cmd += f" && {sudo_arg}{getdepscmd}{allow_sys_arg} install-system-deps {tests_arg}--recursive patchelf" + out.write(f" run: {system_deps_cmd}\n") + required_locales = manifest.get( "github.actions", "required_locales", ctx=manifest_ctx ) @@ -1152,6 +1180,18 @@ def write_job_for_platform(self, platform, args): # noqa: C901 out.write(f" - name: Ensure {loc} locale present\n") out.write(f" run: {sudo_arg}locale-gen {loc}\n") + out.write(" - id: paths\n") + out.write(" name: Query paths\n") + if build_opts.is_windows(): + out.write( + f" run: {getdepscmd}{allow_sys_arg} query-paths {tests_arg}--recursive --src-dir=. {manifest.name} >> $env:GITHUB_OUTPUT\n" + ) + out.write(" shell: pwsh\n") + else: + out.write( + f' run: {getdepscmd}{allow_sys_arg} query-paths {tests_arg}--recursive --src-dir=. {manifest.name} >> "$GITHUB_OUTPUT"\n' + ) + projects = loader.manifests_in_dependency_order() main_repo_url = manifest.get_repo_url(manifest_ctx) @@ -1178,25 +1218,32 @@ def write_job_for_platform(self, platform, args): # noqa: C901 ctx = loader.ctx_gen.get_context(m.name) if m.get_repo_url(ctx) != main_repo_url: out.write(" - name: Fetch %s\n" % m.name) + out.write( + f" if: ${{{{ steps.paths.outputs.{m.name}_SOURCE }}}}\n" + ) out.write( f" run: {getdepscmd}{allow_sys_arg} fetch --no-tests {m.name}\n" ) for m in projects: - if m != manifest: - if m.name == "rust": - continue - else: - src_dir_arg = "" - ctx = loader.ctx_gen.get_context(m.name) - if main_repo_url and m.get_repo_url(ctx) == main_repo_url: - # Its in the same repo, so src-dir is also . - src_dir_arg = "--src-dir=. " - has_same_repo_dep = True - out.write(" - name: Build %s\n" % m.name) - out.write( - f" run: {getdepscmd}{allow_sys_arg} build {build_type_arg}{src_dir_arg}{free_up_disk}--no-tests {m.name}\n" - ) + if m == manifest or m.name == "rust": + continue + src_dir_arg = "" + ctx = loader.ctx_gen.get_context(m.name) + if main_repo_url and m.get_repo_url(ctx) == main_repo_url: + # Its in the same repo, so src-dir is also . + src_dir_arg = "--src-dir=. " + has_same_repo_dep = True + + out.write(" - name: Build %s\n" % m.name) + if not src_dir_arg: + # only run the step if needed + out.write( + f" if: ${{{{ steps.paths.outputs.{m.name}_SOURCE }}}}\n" + ) + out.write( + f" run: {getdepscmd}{allow_sys_arg} build {build_type_arg}{src_dir_arg}{free_up_disk}--no-tests {m.name}\n" + ) out.write(" - name: Build %s\n" % manifest.name) @@ -1213,12 +1260,8 @@ def write_job_for_platform(self, platform, args): # noqa: C901 if has_same_repo_dep: no_deps_arg = "--no-deps " - no_tests_arg = "" - if not run_tests: - no_tests_arg = "--no-tests " - out.write( - f" run: {getdepscmd}{allow_sys_arg} build {build_type_arg}{no_tests_arg}{no_deps_arg}--src-dir=. {manifest.name} {project_prefix}\n" + f" run: {getdepscmd}{allow_sys_arg} build {build_type_arg}{tests_arg}{no_deps_arg}--src-dir=. {manifest.name} {project_prefix}\n" ) out.write(" - name: Copy artifacts\n") diff --git a/build/fbcode_builder/getdeps/fetcher.py b/build/fbcode_builder/getdeps/fetcher.py index c769c118d..a1737089b 100644 --- a/build/fbcode_builder/getdeps/fetcher.py +++ b/build/fbcode_builder/getdeps/fetcher.py @@ -244,7 +244,9 @@ def __init__(self, build_options, manifest, repo_url, rev, depth) -> None: if not m: raise Exception("Failed to parse rev from %s" % hash_file) rev = m.group(1) - print("Using pinned rev %s for %s" % (rev, repo_url)) + print( + "Using pinned rev %s for %s" % (rev, repo_url), file=sys.stderr + ) self.rev = rev or "main" self.origin_repo = repo_url