From 4d0b5a7e6c667282f3d45b7ff0dec29225829baa Mon Sep 17 00:00:00 2001
From: Cody Ohlsen <codyohl@meta.com>
Date: Wed, 17 Apr 2024 17:24:31 -0700
Subject: [PATCH] getdeps: support GETDEPS_WGET_ARGS in wget version, support
 skipping lfs/upload steps (#229)

Summary:
Pull Request resolved: https://github.com/facebookincubator/katran/pull/229

X-link: https://github.com/facebookexperimental/edencommon/pull/18

X-link: https://github.com/facebook/sapling/pull/878

X-link: https://github.com/facebook/openr/pull/154

X-link: https://github.com/facebook/folly/pull/2177

X-link: https://github.com/facebookincubator/zstrong/pull/748

I found it useful to be able to set `GETDEPS_WGET_ARGS` to change some of the flags to `wget` while it's in that fetch mode :)
I also need to skip the lfs upload part in my environment

Reviewed By: mackorone

Differential Revision: D56263907

fbshipit-source-id: ae45c31ebb10123e0358544d294fe2f2979dd59a
---
 build/fbcode_builder/getdeps.py         | 27 ++++++++++++++++++++++++-
 build/fbcode_builder/getdeps/fetcher.py |  7 +++++--
 2 files changed, 31 insertions(+), 3 deletions(-)

diff --git a/build/fbcode_builder/getdeps.py b/build/fbcode_builder/getdeps.py
index aeae6bf63..4ab21537f 100755
--- a/build/fbcode_builder/getdeps.py
+++ b/build/fbcode_builder/getdeps.py
@@ -561,6 +561,10 @@ def run_project_cmd(self, args, loader, manifest):
         for m in projects:
             fetcher = loader.create_fetcher(m)
 
+            if args.build_skip_lfs_download and hasattr(fetcher, "skip_lfs_download"):
+                print("skipping lfs download for %s" % m.name)
+                fetcher.skip_lfs_download()
+
             if isinstance(fetcher, SystemPackageFetcher):
                 # We are guaranteed that if the fetcher is set to
                 # SystemPackageFetcher then this item is completely
@@ -652,7 +656,11 @@ def run_project_cmd(self, args, loader, manifest):
 
                     # Only populate the cache from continuous build runs, and
                     # only if we have a built_marker.
-                    if args.schedule_type == "continuous" and has_built_marker:
+                    if (
+                        not args.skip_upload
+                        and args.schedule_type == "continuous"
+                        and has_built_marker
+                    ):
                         cached_project.upload()
                 elif args.verbose:
                     print("found good %s" % built_marker)
@@ -1322,11 +1330,28 @@ def add_common_arg(*args, **kwargs):
         action="store_true",
         default=False,
     )
+    add_common_arg(
+        "-su",
+        "--skip-upload",
+        help="skip upload steps",
+        action="store_true",
+        default=False,
+    )
     add_common_arg(
         "--lfs-path",
         help="Provide a parent directory for lfs when fbsource is unavailable",
         default=None,
     )
+    add_common_arg(
+        "--build-skip-lfs-download",
+        action="store_true",
+        default=False,
+        help=(
+            "Download from the URL, rather than LFS. This is useful "
+            "in cases where the upstream project has uploaded a new "
+            "version of the archive with a different hash"
+        ),
+    )
 
     ap = argparse.ArgumentParser(
         description="Get and build dependencies and projects", parents=[common_args]
diff --git a/build/fbcode_builder/getdeps/fetcher.py b/build/fbcode_builder/getdeps/fetcher.py
index b1c113f46..f02dd447b 100644
--- a/build/fbcode_builder/getdeps/fetcher.py
+++ b/build/fbcode_builder/getdeps/fetcher.py
@@ -683,15 +683,18 @@ def progress_pycurl(self, total, amount, _uploadtotal, _uploadamount):
     start = time.time()
     try:
         if os.environ.get("GETDEPS_USE_WGET") is not None:
-            subprocess.run(
+            procargs = (
                 [
                     "wget",
+                ]
+                + os.environ.get("GETDEPS_WGET_ARGS", "").split()
+                + [
                     "-O",
                     file_name,
                     url,
                 ]
             )
-
+            subprocess.run(procargs, capture_output=True)
             headers = None
 
         elif os.environ.get("GETDEPS_USE_LIBCURL") is not None: