Fix TestStacksGenerateRemote (#3865)

* Updated compression level

* Setting compression 0

* Add depth url

* Config update

* git commands debug

* space check test

* Updated pack threads

* Window memory update

* global update

* Updated git clone timeout

* Git ssh path

* Memory limit update

* Usage of https for repo clone

* Post buffer update

* HTTP config update

* Config cleanup

* Add http low speed

* Clone update

* Config update

* Disable GC

* Size limit update

* https git clone

* Switch to go getter v2

* Usage of go-getter v2, added remote review

* Git config cleanup

* Cleanup

* stack generate improvements

Author: denis256

.circleci/config.yml

@@ -78,6 +78,9 @@ setup_test_environment: &setup_test_environment
     # Import test / dev key for SOPS
     gpg --import --no-tty --batch --yes ./test/fixtures/sops/test_pgp_key.asc
     mkdir -p logs
+    # configure git to avoid periodic failures
+    git config --global core.compression 0
+    git config --global gc.auto 0
   no_output_timeout: 30m
 
 run_integration_test: &run_integration_test

_ci/install-golang.ps1

@@ -4,6 +4,10 @@ choco install golang --version 1.23.1 -y
 Get-Command go
 go version
 
+# configure git compression
+git config --global core.compression 0
+git config --global gc.auto 0
+
 try {
     # Enable Developer Mode
     reg add "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\CurrentVersion\AppModelUnlock" /t REG_DWORD /f /v "AllowDevelopmentWithoutDevLicense" /d "1"

cli/commands/stack/generate.go

@@ -3,19 +3,23 @@ package stack
 import (
 	"context"
 	"fmt"
-	"io"
-	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
 
+	"github.com/gruntwork-io/terragrunt/util"
+
 	"github.com/gruntwork-io/terragrunt/config"
-	"github.com/hashicorp/go-getter"
+	"github.com/hashicorp/go-getter/v2"
 
 	"github.com/gruntwork-io/terragrunt/internal/errors"
 	"github.com/gruntwork-io/terragrunt/options"
 )
 
+const (
+	StackManifestName = ".terragrunt-stack-manifest"
+)
+
 func generateStack(ctx context.Context, opts *options.TerragruntOptions) error {
 	opts.TerragruntStackConfigPath = filepath.Join(opts.WorkingDir, defaultStackFile)
 	opts.Logger.Infof("Generating stack from %s", opts.TerragruntStackConfigPath)
@@ -48,172 +52,58 @@ func processStackFile(ctx context.Context, opts *options.TerragruntOptions, stac
 			return errors.New(fmt.Errorf("failed to get absolute path for destination '%s': %w", dest, err))
 		}
 
-		client := &getter.Client{
-			Dst:             dest,
-			Mode:            getter.ClientModeAny,
-			Dir:             true,
-			DisableSymlinks: true,
-			Options: []getter.ClientOption{
-				getter.WithContext(ctx),
-			},
-		}
-
-		// setting custom getters
-		client.Getters = map[string]getter.Getter{}
-
-		for getterName, getterValue := range getter.Getters {
-			// setting custom getter for file to not use symlinks
-			if getterName == "file" {
-				client.Getters[getterName] = &stacksFileProvider{}
-			} else {
-				client.Getters[getterName] = getterValue
-			}
-		}
-
-		// fetching unit source
 		src := unit.Source
+		opts.Logger.Debugf("Processing unit: %s (%s) to %s", unit.Name, src, dest)
 
-		// set absolute path for source if it's not an absolute path or URL
-		if !filepath.IsAbs(unit.Source) && !isURL(client, unit.Source) {
+		if isLocal(opts, src) {
 			src = filepath.Join(opts.WorkingDir, unit.Source)
 			src, err = filepath.Abs(src)
 
 			if err != nil {
 				opts.Logger.Warnf("failed to get absolute path for source '%s': %v", unit.Source, err)
 				src = unit.Source
 			}
-		}
 
-		opts.Logger.Debugf("Processing unit: %s (%s) to %s", unit.Name, src, dest)
-
-		client.Src = src
+			if err := util.CopyFolderContentsWithFilter(opts.Logger, src, dest, StackManifestName, func(absolutePath string) bool {
+				return true
+			}); err != nil {
+				return errors.New(err)
+			}
+		} else {
+			if err := os.MkdirAll(dest, dirPerm); err != nil {
+				return errors.New(err)
+			}
 
-		if err := client.Get(); err != nil {
-			return errors.New(err)
+			if _, err := getter.GetAny(ctx, dest, src); err != nil {
+				return errors.New(err)
+			}
 		}
 	}
 
 	return nil
 }
 
-func isURL(client *getter.Client, str string) bool {
-	value, err := getter.Detect(str, client.Dst, getter.Detectors)
-	if err != nil {
-		return false
-	}
-	// check if starts with file://
-	if strings.HasPrefix(value, "file://") {
-		return false
-	}
-
-	return true
-}
-
-// stacksFileProvider is a custom getter for file:// protocol.
-type stacksFileProvider struct {
-	client *getter.Client
-}
-
-// Get implements downloading functionality.
-func (p *stacksFileProvider) Get(dst string, u *url.URL) error {
-	src := u.Path
-	file, err := os.Stat(src)
-
-	if err != nil {
-		return errors.New(fmt.Errorf("source path error: %w", err))
-	}
-
-	if file.IsDir() {
-		return p.copyDir(src, dst)
-	}
-
-	return p.copyFile(src, dst)
-}
-
-// GetFile implements single file download.
-func (p *stacksFileProvider) GetFile(dst string, u *url.URL) error {
-	return p.copyFile(u.Path, dst)
-}
-
-// ClientMode determines if we're getting a directory or single file.
-func (p *stacksFileProvider) ClientMode(u *url.URL) (getter.ClientMode, error) {
-	fi, err := os.Stat(u.Path)
-	if err != nil {
-		return getter.ClientModeInvalid, errors.New(err)
-	}
-
-	if fi.IsDir() {
-		return getter.ClientModeDir, nil
-	}
-
-	return getter.ClientModeFile, nil
-}
-
-// SetClient sets the client for this provider.
-func (p *stacksFileProvider) SetClient(c *getter.Client) {
-	p.client = c
-}
-
-func (p *stacksFileProvider) copyFile(src, dst string) error {
-	if err := os.MkdirAll(filepath.Dir(dst), dirPerm); err != nil {
-		return errors.New(err)
-	}
-
-	srcFile, err := os.Open(src)
-	if err != nil {
-		return errors.New(err)
-	}
-	defer srcFile.Close()
-
-	srcInfo, err := srcFile.Stat()
-	if err != nil {
-		return errors.New(err)
-	}
-
-	dstFile, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, srcInfo.Mode())
-	if err != nil {
-		return errors.New(err)
-	}
-	defer dstFile.Close()
-
-	if _, err := io.Copy(dstFile, srcFile); err != nil {
-		return errors.New(err)
-	}
-
-	return nil
-}
-
-func (p *stacksFileProvider) copyDir(src, dst string) error {
-	srcInfo, err := os.Stat(src)
-	if err != nil {
-		return errors.New(err)
-	}
-
-	if err := os.MkdirAll(dst, srcInfo.Mode()); err != nil {
-		return errors.New(err)
+func isLocal(opts *options.TerragruntOptions, src string) bool {
+	// check initially if the source is a local file
+	src = filepath.Join(opts.WorkingDir, src)
+	if util.FileExists(src) {
+		return true
 	}
-
-	entries, err := os.ReadDir(src)
-	if err != nil {
-		return errors.New(err)
+	// check path through getters
+	req := &getter.Request{
+		Src: src,
 	}
-
-	for _, entry := range entries {
-		srcPath := filepath.Join(src, entry.Name())
-		dstPath := filepath.Join(dst, entry.Name())
-
-		if entry.IsDir() {
-			if err := p.copyDir(srcPath, dstPath); err != nil {
-				return errors.New(err)
-			}
-
+	for _, g := range getter.Getters {
+		recognized, err := getter.Detect(req, g)
+		if err != nil {
+			opts.Logger.Debugf("Error detecting getter for %s: %v", src, err)
 			continue
 		}
 
-		if err := p.copyFile(srcPath, dstPath); err != nil {
-			return errors.New(err)
+		if recognized {
+			break
 		}
 	}
 
-	return nil
+	return strings.HasPrefix(req.Src, "file://")
 }

test/fixtures/stacks/remote/terragrunt.stack.hcl

@@ -3,12 +3,12 @@ locals {
 }
 
 unit "app1" {
-	source = "github.com/gruntwork-io/terragrunt.git//test/fixtures/stacks/basic/units/chick?ref=${local.version}"
+	source = "git::https://github.com/gruntwork-io/terragrunt.git//test/fixtures/stacks/basic/units/chick?ref=${local.version}&depth=1"
 	path   = "app1"
 }
 
 unit "app2" {
-	source = "github.com/gruntwork-io/terragrunt.git//test/fixtures/stacks/basic/units/chick?ref=${local.version}"
+	source = "git::https://github.com/gruntwork-io/terragrunt.git//test/fixtures/stacks/basic/units/chick?ref=${local.version}&depth=1"
 	path   = "app2"
 }
 

test/integration_stacks_test.go

@@ -143,6 +143,22 @@ func TestStacksApply(t *testing.T) {
 	assert.Contains(t, stdout, "local_file.file: Creation complete")
 }
 
+func TestStacksApplyRemote(t *testing.T) {
+	t.Parallel()
+
+	helpers.CleanupTerraformFolder(t, testFixtureStacksRemote)
+	tmpEnvPath := helpers.CopyEnvironment(t, testFixtureStacksRemote)
+	rootPath := util.JoinPath(tmpEnvPath, testFixtureStacksRemote)
+
+	stdout, _, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt stack run apply --experiment stacks --terragrunt-non-interactive --terragrunt-working-dir "+rootPath)
+	require.NoError(t, err)
+
+	assert.Contains(t, stdout, "Apply complete! Resources: 1 added, 0 changed, 0 destroyed")
+	assert.Contains(t, stdout, "local_file.file: Creation complete")
+	path := util.JoinPath(rootPath, ".terragrunt-stack")
+	validateStackDir(t, path)
+}
+
 func TestStacksDestroy(t *testing.T) {
 	t.Parallel()