fix getdeps actions generation for rust toolchain (#682)

Summary:
X-link: facebookincubator/velox#6924

fix getdeps actions generation for rust toolchain

Generated actions files created from getdeps.py were missing rust toolchain which meant annoying manual edits

This change also makes the github actions generation honor the --no-tests argument, which is useful  when regenerating the edenfs actions

Pull Request resolved: #682

Test Plan:
To test we add the missing undeclared rust-shed dependency to the eden manifest and regenerate.

Generate updated actions yaml with:
```
./build/fbcode_builder/getdeps.py --allow-system-packages generate-github-actions --no-tests --free-up-disk --os-type=linux --src-dir=. --output-dir=.github/workflows --job-name "EdenFS " --job-file-prefix=edenfs_ eden
```

Before, was missing the rust toolchain in the auto regenerated eden actions file.  After, toolchain is present

Before, was missing the --no-tests in the "Build eden" step of the  auto-regenerated eden actions file.  After, has it

Eden stil needs some other fixes to build, which follow in this stqck

 ---
Stack created with [Sapling](https://sapling-scm.com). Best reviewed with [ReviewStack](https://reviewstack.dev/facebook/sapling/pull/682).
* #693
* #696
* #692
* #691
* __->__ #682
* #689
* #697
* #706
* #730

Reviewed By: sggutier

Differential Revision: D49875258

Pulled By: genevievehelsel

fbshipit-source-id: 173f86083ba92bab4063813d3e392df428b9ffe4

.github/workflows/edenfs_linux.yml

@@ -15,6 +15,12 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
     - uses: actions/checkout@v2
+    - name: Show disk space at start
+      run: df -h
+    - name: Free up disk space
+      run: sudo rm -rf /usr/local/lib/android
+    - name: Show disk space after freeing up
+      run: df -h
     - name: Update system package info
       run: sudo apt-get update
     - name: Install system deps
@@ -25,14 +31,16 @@ jobs:
       run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests ninja
     - name: Fetch cmake
       run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests cmake
+    - name: Fetch blake3
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests blake3
     - name: Fetch cpptoml
       run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests cpptoml
+    - name: Fetch fmt
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fmt
     - name: Fetch gflags
       run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests gflags
     - name: Fetch glog
       run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests glog
-    - name: Fetch fmt
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fmt
     - name: Fetch googletest
       run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests googletest
     - name: Fetch python-six
@@ -93,6 +101,8 @@ jobs:
       run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests folly
     - name: Fetch fizz
       run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fizz
+    - name: Fetch mvfst
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests mvfst
     - name: Fetch wangle
       run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests wangle
     - name: Fetch fbthrift
@@ -101,99 +111,101 @@ jobs:
       run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fb303
     - name: Fetch rust-shed
       run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests rust-shed
-    - name: Fetch sapling
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests sapling
     - name: Fetch edencommon
       run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests edencommon
     - name: Build ninja
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests ninja
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests ninja
     - name: Build cmake
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests cmake
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests cmake
+    - name: Build blake3
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests blake3
     - name: Build cpptoml
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests cpptoml
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests cpptoml
+    - name: Build fmt
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests fmt
     - name: Build gflags
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests gflags
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests gflags
     - name: Build glog
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests glog
-    - name: Build fmt
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fmt
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests glog
     - name: Build googletest
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests googletest
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests googletest
     - name: Build python-six
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests python-six
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests python-six
     - name: Build zstd
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests zstd
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests zstd
     - name: Build boost
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests boost
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests boost
     - name: Build double-conversion
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests double-conversion
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests double-conversion
     - name: Build libevent
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libevent
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests libevent
     - name: Build lz4
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests lz4
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests lz4
     - name: Build snappy
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests snappy
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests snappy
     - name: Build libgit2
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libgit2
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests libgit2
     - name: Build python-ptyprocess
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests python-ptyprocess
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests python-ptyprocess
     - name: Build pexpect
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests pexpect
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests pexpect
     - name: Build bz2
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests bz2
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests bz2
     - name: Build python-filelock
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests python-filelock
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests python-filelock
     - name: Build python-toml
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests python-toml
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests python-toml
     - name: Build re2
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests re2
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests re2
     - name: Build rocksdb
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests rocksdb
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests rocksdb
     - name: Build sqlite3
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests sqlite3
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests sqlite3
     - name: Build zlib
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests zlib
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests zlib
     - name: Build autoconf
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests autoconf
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests autoconf
     - name: Build automake
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests automake
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests automake
     - name: Build libtool
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libtool
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests libtool
     - name: Build nghttp2
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests nghttp2
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests nghttp2
     - name: Build libcurl
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libcurl
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests libcurl
     - name: Build libffi
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libffi
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests libffi
     - name: Build ncurses
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests ncurses
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests ncurses
     - name: Build python
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests python
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests python
     - name: Build libsodium
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libsodium
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests libsodium
     - name: Build xz
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests xz
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests xz
     - name: Build folly
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests folly
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests folly
     - name: Build fizz
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fizz
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests fizz
+    - name: Build mvfst
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests mvfst
     - name: Build wangle
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests wangle
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests wangle
     - name: Build fbthrift
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fbthrift
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests fbthrift
     - name: Build fb303
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fb303
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests fb303
     - name: Build rust-shed
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests rust-shed
-    - name: Build sapling
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests sapling
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests rust-shed
     - name: Build edencommon
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests edencommon
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests edencommon
     - name: Build eden
-      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --src-dir=. eden  --project-install-prefix eden:/usr/local
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests --src-dir=. eden  --project-install-prefix eden:/usr/local
     - name: Copy artifacts
       run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fixup-dyn-deps --strip --src-dir=. eden _artifacts/linux  --project-install-prefix eden:/usr/local --final-install-prefix /usr/local
     - uses: actions/upload-artifact@v2
       with:
         name: eden
         path: _artifacts
+    - name: Show disk space at end
+      run: df -h

build/fbcode_builder/getdeps.py

@@ -939,7 +939,8 @@ def write_job_for_platform(self, platform, args):  # noqa: C901
         # We do this by looking at the builder type in the manifest file
         # rather than creating a builder and checking its type because we
         # don't know enough to create the full builder instance here.
-        if manifest.get("build", "builder", ctx=manifest_ctx) == "nop":
+        builder_name = manifest.get("build", "builder", ctx=manifest_ctx)
+        if builder_name == "nop":
             return None
 
         # We want to be sure that we're running things with python 3
@@ -1058,18 +1059,30 @@ def write_job_for_platform(self, platform, args):  # noqa: C901
             main_repo_url = manifest.get_repo_url(manifest_ctx)
             has_same_repo_dep = False
 
+            # Add the rust dep which doesn't have a manifest
             for m in projects:
-                if m != manifest:
-                    if m.name == "rust":
-                        out.write("    - name: Install Rust Stable\n")
-                        out.write("      uses: dtolnay/rust-toolchain@stable\n")
-                    else:
-                        ctx = loader.ctx_gen.get_context(m.name)
-                        if m.get_repo_url(ctx) != main_repo_url:
-                            out.write("    - name: Fetch %s\n" % m.name)
-                            out.write(
-                                f"      run: {getdepscmd}{allow_sys_arg} fetch --no-tests {m.name}\n"
-                            )
+                if m == manifest:
+                    continue
+                mbuilder_name = m.get("build", "builder", ctx=manifest_ctx)
+                if (
+                    m.name == "rust"
+                    or builder_name == "cargo"
+                    or mbuilder_name == "cargo"
+                ):
+                    out.write("    - name: Install Rust Stable\n")
+                    out.write("      uses: dtolnay/rust-toolchain@stable\n")
+                    break
+
+            # Normal deps that have manifests
+            for m in projects:
+                if m == manifest or m.name == "rust":
+                    continue
+                ctx = loader.ctx_gen.get_context(m.name)
+                if m.get_repo_url(ctx) != main_repo_url:
+                    out.write("    - name: Fetch %s\n" % m.name)
+                    out.write(
+                        f"      run: {getdepscmd}{allow_sys_arg} fetch --no-tests {m.name}\n"
+                    )
 
             for m in projects:
                 if m != manifest:
@@ -1100,8 +1113,12 @@ def write_job_for_platform(self, platform, args):  # noqa: C901
             if has_same_repo_dep:
                 no_deps_arg = "--no-deps "
 
+            no_tests_arg = ""
+            if not args.enable_tests:
+                no_tests_arg = "--no-tests "
+
             out.write(
-                f"      run: {getdepscmd}{allow_sys_arg} build {no_deps_arg}--src-dir=. {manifest.name} {project_prefix}\n"
+                f"      run: {getdepscmd}{allow_sys_arg} build {no_tests_arg}{no_deps_arg}--src-dir=. {manifest.name} {project_prefix}\n"
             )
 
             out.write("    - name: Copy artifacts\n")
@@ -1125,7 +1142,11 @@ def write_job_for_platform(self, platform, args):  # noqa: C901
             out.write("        name: %s\n" % manifest.name)
             out.write("        path: _artifacts\n")
 
-            if manifest.get("github.actions", "run_tests", ctx=manifest_ctx) != "off":
+            if (
+                args.enable_tests
+                and manifest.get("github.actions", "run_tests", ctx=manifest_ctx)
+                != "off"
+            ):
                 out.write("    - name: Test %s\n" % manifest.name)
                 out.write(
                     f"      run: {getdepscmd}{allow_sys_arg} test --src-dir=. {manifest.name} {project_prefix}\n"

build/fbcode_builder/manifests/eden

@@ -5,7 +5,7 @@ shipit_project = eden
 shipit_fbcode_builder = true
 
 [git]
-repo_url = https://github.com/facebookexperimental/eden.git
+repo_url = https://github.com/facebook/sapling.git
 
 [github.actions]
 run_tests = off
@@ -30,6 +30,7 @@ pexpect
 python-toml
 python-filelock
 edencommon
+rust-shed
 
 [dependencies.fbsource=on]
 rust