diff --git a/.github/workflows/edenfs_linux.yml b/.github/workflows/edenfs_linux.yml index 58daeeae14203..98978c25d3544 100644 --- a/.github/workflows/edenfs_linux.yml +++ b/.github/workflows/edenfs_linux.yml @@ -15,6 +15,12 @@ jobs: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 + - name: Show disk space at start + run: df -h + - name: Free up disk space + run: sudo rm -rf /usr/local/lib/android + - name: Show disk space after freeing up + run: df -h - name: Update system package info run: sudo apt-get update - name: Install system deps @@ -25,14 +31,16 @@ jobs: run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests ninja - name: Fetch cmake run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests cmake + - name: Fetch blake3 + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests blake3 - name: Fetch cpptoml run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests cpptoml + - name: Fetch fmt + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fmt - name: Fetch gflags run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests gflags - name: Fetch glog run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests glog - - name: Fetch fmt - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fmt - name: Fetch googletest run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests googletest - name: Fetch python-six @@ -93,6 +101,8 @@ jobs: run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests folly - name: Fetch fizz run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fizz + - name: Fetch mvfst + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests mvfst - name: Fetch wangle run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests wangle - name: Fetch fbthrift @@ -101,99 +111,101 @@ jobs: run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fb303 - name: Fetch rust-shed run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests rust-shed - - name: Fetch sapling - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests sapling - name: Fetch edencommon run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests edencommon - name: Build ninja - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests ninja + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests ninja - name: Build cmake - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests cmake + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests cmake + - name: Build blake3 + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests blake3 - name: Build cpptoml - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests cpptoml + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests cpptoml + - name: Build fmt + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests fmt - name: Build gflags - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests gflags + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests gflags - name: Build glog - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests glog - - name: Build fmt - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fmt + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests glog - name: Build googletest - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests googletest + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests googletest - name: Build python-six - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests python-six + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests python-six - name: Build zstd - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests zstd + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests zstd - name: Build boost - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests boost + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests boost - name: Build double-conversion - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests double-conversion + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests double-conversion - name: Build libevent - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libevent + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests libevent - name: Build lz4 - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests lz4 + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests lz4 - name: Build snappy - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests snappy + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests snappy - name: Build libgit2 - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libgit2 + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests libgit2 - name: Build python-ptyprocess - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests python-ptyprocess + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests python-ptyprocess - name: Build pexpect - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests pexpect + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests pexpect - name: Build bz2 - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests bz2 + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests bz2 - name: Build python-filelock - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests python-filelock + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests python-filelock - name: Build python-toml - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests python-toml + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests python-toml - name: Build re2 - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests re2 + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests re2 - name: Build rocksdb - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests rocksdb + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests rocksdb - name: Build sqlite3 - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests sqlite3 + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests sqlite3 - name: Build zlib - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests zlib + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests zlib - name: Build autoconf - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests autoconf + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests autoconf - name: Build automake - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests automake + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests automake - name: Build libtool - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libtool + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests libtool - name: Build nghttp2 - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests nghttp2 + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests nghttp2 - name: Build libcurl - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libcurl + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests libcurl - name: Build libffi - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libffi + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests libffi - name: Build ncurses - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests ncurses + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests ncurses - name: Build python - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests python + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests python - name: Build libsodium - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libsodium + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests libsodium - name: Build xz - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests xz + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests xz - name: Build folly - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests folly + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests folly - name: Build fizz - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fizz + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests fizz + - name: Build mvfst + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests mvfst - name: Build wangle - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests wangle + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests wangle - name: Build fbthrift - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fbthrift + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests fbthrift - name: Build fb303 - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fb303 + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests fb303 - name: Build rust-shed - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests rust-shed - - name: Build sapling - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests sapling + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests rust-shed - name: Build edencommon - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests edencommon + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests edencommon - name: Build eden - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --src-dir=. eden --project-install-prefix eden:/usr/local + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests --src-dir=. eden --project-install-prefix eden:/usr/local - name: Copy artifacts run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fixup-dyn-deps --strip --src-dir=. eden _artifacts/linux --project-install-prefix eden:/usr/local --final-install-prefix /usr/local - uses: actions/upload-artifact@v2 with: name: eden path: _artifacts + - name: Show disk space at end + run: df -h diff --git a/build/fbcode_builder/getdeps.py b/build/fbcode_builder/getdeps.py index f7baaf8e231ab..8a722a9470037 100755 --- a/build/fbcode_builder/getdeps.py +++ b/build/fbcode_builder/getdeps.py @@ -939,7 +939,8 @@ def write_job_for_platform(self, platform, args): # noqa: C901 # We do this by looking at the builder type in the manifest file # rather than creating a builder and checking its type because we # don't know enough to create the full builder instance here. - if manifest.get("build", "builder", ctx=manifest_ctx) == "nop": + builder_name = manifest.get("build", "builder", ctx=manifest_ctx) + if builder_name == "nop": return None # We want to be sure that we're running things with python 3 @@ -1058,18 +1059,30 @@ def write_job_for_platform(self, platform, args): # noqa: C901 main_repo_url = manifest.get_repo_url(manifest_ctx) has_same_repo_dep = False + # Add the rust dep which doesn't have a manifest for m in projects: - if m != manifest: - if m.name == "rust": - out.write(" - name: Install Rust Stable\n") - out.write(" uses: dtolnay/rust-toolchain@stable\n") - else: - ctx = loader.ctx_gen.get_context(m.name) - if m.get_repo_url(ctx) != main_repo_url: - out.write(" - name: Fetch %s\n" % m.name) - out.write( - f" run: {getdepscmd}{allow_sys_arg} fetch --no-tests {m.name}\n" - ) + if m == manifest: + continue + mbuilder_name = m.get("build", "builder", ctx=manifest_ctx) + if ( + m.name == "rust" + or builder_name == "cargo" + or mbuilder_name == "cargo" + ): + out.write(" - name: Install Rust Stable\n") + out.write(" uses: dtolnay/rust-toolchain@stable\n") + break + + # Normal deps that have manifests + for m in projects: + if m == manifest or m.name == "rust": + continue + ctx = loader.ctx_gen.get_context(m.name) + if m.get_repo_url(ctx) != main_repo_url: + out.write(" - name: Fetch %s\n" % m.name) + out.write( + f" run: {getdepscmd}{allow_sys_arg} fetch --no-tests {m.name}\n" + ) for m in projects: if m != manifest: @@ -1100,8 +1113,12 @@ def write_job_for_platform(self, platform, args): # noqa: C901 if has_same_repo_dep: no_deps_arg = "--no-deps " + no_tests_arg = "" + if not args.enable_tests: + no_tests_arg = "--no-tests " + out.write( - f" run: {getdepscmd}{allow_sys_arg} build {no_deps_arg}--src-dir=. {manifest.name} {project_prefix}\n" + f" run: {getdepscmd}{allow_sys_arg} build {no_tests_arg}{no_deps_arg}--src-dir=. {manifest.name} {project_prefix}\n" ) out.write(" - name: Copy artifacts\n") @@ -1125,7 +1142,11 @@ def write_job_for_platform(self, platform, args): # noqa: C901 out.write(" name: %s\n" % manifest.name) out.write(" path: _artifacts\n") - if manifest.get("github.actions", "run_tests", ctx=manifest_ctx) != "off": + if ( + args.enable_tests + and manifest.get("github.actions", "run_tests", ctx=manifest_ctx) + != "off" + ): out.write(" - name: Test %s\n" % manifest.name) out.write( f" run: {getdepscmd}{allow_sys_arg} test --src-dir=. {manifest.name} {project_prefix}\n" diff --git a/build/fbcode_builder/manifests/eden b/build/fbcode_builder/manifests/eden index 5261fd955b5d9..4c32bf698a1c0 100644 --- a/build/fbcode_builder/manifests/eden +++ b/build/fbcode_builder/manifests/eden @@ -5,7 +5,7 @@ shipit_project = eden shipit_fbcode_builder = true [git] -repo_url = https://github.com/facebookexperimental/eden.git +repo_url = https://github.com/facebook/sapling.git [github.actions] run_tests = off @@ -30,6 +30,7 @@ pexpect python-toml python-filelock edencommon +rust-shed [dependencies.fbsource=on] rust