Enhance zip-deploy during build for Python/Node

cli/azd/.vscode/cspell-azd-dictionary.txt

@@ -85,6 +85,7 @@ pflag
 preinit
 pulumi
 pyapp
+pyvenv
 restoreapp
 retriable
 rzip

cli/azd/internal/repository/initializer.go

@@ -81,7 +81,7 @@ func (i *Initializer) Initialize(
 
 	options := copy.Options{}
 	if skipStagingFiles != nil {
-		options.Skip = func(src string) (bool, error) {
+		options.Skip = func(fileInfo os.FileInfo, src, dest string) (bool, error) {
 			if _, shouldSkip := skipStagingFiles[src]; shouldSkip {
 				return true, nil
 			}

cli/azd/pkg/project/framework_service_npm.go

@@ -14,7 +14,6 @@ import (
 	"github.com/azure/azure-dev/cli/azd/pkg/exec"
 	"github.com/azure/azure-dev/cli/azd/pkg/tools"
 	"github.com/azure/azure-dev/cli/azd/pkg/tools/npm"
-	"github.com/otiai10/copy"
 )
 
 type npmProject struct {
@@ -90,11 +89,15 @@ func (np *npmProject) Build(
 			}
 
 			task.SetProgress(NewServiceProgress("Copying deployment package"))
-			if err := copy.Copy(
+
+			if err := buildForZip(
 				publishSource,
 				publishRoot,
-				skipPatterns(
-					filepath.Join(publishSource, "node_modules"), filepath.Join(publishSource, ".azure"))); err != nil {
+				buildForZipOptions{
+					excludeConditions: []excludeDirEntryCondition{
+						excludeNodeModules,
+					},
+				}); err != nil {
 				task.SetError(fmt.Errorf("publishing for %s: %w", serviceConfig.Name, err))
 				return
 			}
@@ -106,3 +109,9 @@ func (np *npmProject) Build(
 		},
 	)
 }
+
+const cNodeModulesName = "node_modules"
+
+func excludeNodeModules(path string, file os.FileInfo) bool {
+	return !file.IsDir() && file.Name() == cNodeModulesName
+}

cli/azd/pkg/project/framework_service_python.go

@@ -16,7 +16,6 @@ import (
 	"github.com/azure/azure-dev/cli/azd/pkg/exec"
 	"github.com/azure/azure-dev/cli/azd/pkg/tools"
 	"github.com/azure/azure-dev/cli/azd/pkg/tools/python"
-	"github.com/otiai10/copy"
 )
 
 type pythonProject struct {
@@ -107,12 +106,15 @@ func (pp *pythonProject) Build(
 
 			task.SetProgress(NewServiceProgress("Copying deployment package"))
 
-			if err := copy.Copy(
+			if err := buildForZip(
 				publishSource,
 				publishRoot,
-				skipPatterns(
-					filepath.Join(publishSource, "__pycache__"), filepath.Join(publishSource, ".venv"),
-					filepath.Join(publishSource, ".azure"))); err != nil {
+				buildForZipOptions{
+					excludeConditions: []excludeDirEntryCondition{
+						excludeVirtualEnv,
+						excludePyCache,
+					},
+				}); err != nil {
 				task.SetError(fmt.Errorf("publishing for %s: %w", serviceConfig.Name, err))
 				return
 			}
@@ -125,6 +127,29 @@ func (pp *pythonProject) Build(
 	)
 }
 
+const cVenvConfigFileName = "pyvenv.cfg"
+
+func excludeVirtualEnv(path string, file os.FileInfo) bool {
+	if !file.IsDir() {
+		return false
+	}
+
+	// check if `pyvenv.cfg` is within the folder
+	if _, err := os.Stat(filepath.Join(path, cVenvConfigFileName)); err == nil {
+		return true
+	}
+	return false
+}
+
+func excludePyCache(path string, file os.FileInfo) bool {
+	if !file.IsDir() {
+		return false
+	}
+
+	folderName := strings.ToLower(file.Name())
+	return folderName == "__pycache__"
+}
+
 func (pp *pythonProject) getVenvName(serviceConfig *ServiceConfig) string {
 	trimmedPath := strings.TrimSpace(serviceConfig.Path())
 	if len(trimmedPath) > 0 && trimmedPath[len(trimmedPath)-1] == os.PathSeparator {
@@ -133,23 +158,3 @@ func (pp *pythonProject) getVenvName(serviceConfig *ServiceConfig) string {
 	_, projectDir := filepath.Split(trimmedPath)
 	return projectDir + "_env"
 }
-
-// skipPatterns returns a `copy.Options` which will skip any files
-// that match a given pattern. Matching is done with `filepath.Match`.
-func skipPatterns(patterns ...string) copy.Options {
-	return copy.Options{
-		Skip: func(src string) (bool, error) {
-			for _, pattern := range patterns {
-				skip, err := filepath.Match(pattern, src)
-				switch {
-				case err != nil:
-					return false, fmt.Errorf("error matching pattern %s: %w", pattern, err)
-				case skip:
-					return true, nil
-				}
-			}
-
-			return false, nil
-		},
-	}
-}

cli/azd/pkg/project/project_utils.go

@@ -0,0 +1,70 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+package project
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/azure/azure-dev/cli/azd/pkg/rzip"
+	"github.com/otiai10/copy"
+)
+
+// CreateDeployableZip creates a zip file of a folder, recursively.
+// Returns the path to the created zip file or an error if it fails.
+func createDeployableZip(appName string, path string) (string, error) {
+	// TODO: should probably avoid picking up files that weren't meant to be published (ie, local .env files, etc..)
+	zipFile, err := os.CreateTemp("", "azddeploy*.zip")
+	if err != nil {
+		return "", fmt.Errorf("failed when creating zip package to deploy %s: %w", appName, err)
+	}
+
+	if err := rzip.CreateFromDirectory(path, zipFile); err != nil {
+		// if we fail here just do our best to close things out and cleanup
+		zipFile.Close()
+		os.Remove(zipFile.Name())
+		return "", err
+	}
+
+	if err := zipFile.Close(); err != nil {
+		// may fail but, again, we'll do our best to cleanup here.
+		os.Remove(zipFile.Name())
+		return "", err
+	}
+
+	return zipFile.Name(), nil
+}
+
+// excludeDirEntryCondition resolves when a file or directory should be considered or not as part of build, when build is a
+// copy-paste source strategy. Return true to exclude the directory entry.
+type excludeDirEntryCondition func(path string, file os.FileInfo) bool
+
+// buildForZipOptions provides a set of options for doing build for zip
+type buildForZipOptions struct {
+	excludeConditions []excludeDirEntryCondition
+}
+
+// buildForZip is use by projects which build strategy is to only copy the source code into a folder which is later
+// zipped for packaging. For example Python and Node framework languages. buildForZipOptions provides the specific
+// details for each language which should not be ever copied.
+func buildForZip(src, dst string, options buildForZipOptions) error {
+
+	// these exclude conditions applies to all projects
+	options.excludeConditions = append(options.excludeConditions, globalExcludeAzdFolder)
+
+	return copy.Copy(src, dst, copy.Options{
+		Skip: func(srcInfo os.FileInfo, src, dest string) (bool, error) {
+			for _, checkExclude := range options.excludeConditions {
+				if checkExclude(src, srcInfo) {
+					return true, nil
+				}
+			}
+			return false, nil
+		},
+	})
+}
+
+func globalExcludeAzdFolder(path string, file os.FileInfo) bool {
+	return file.IsDir() && file.Name() == ".azure"
+}

cli/azd/pkg/project/service_target_appservice.go

@@ -13,7 +13,6 @@ import (
 	"github.com/azure/azure-dev/cli/azd/pkg/azure"
 	"github.com/azure/azure-dev/cli/azd/pkg/environment"
 	"github.com/azure/azure-dev/cli/azd/pkg/infra"
-	"github.com/azure/azure-dev/cli/azd/pkg/project/internal"
 	"github.com/azure/azure-dev/cli/azd/pkg/tools"
 	"github.com/azure/azure-dev/cli/azd/pkg/tools/azcli"
 )
@@ -54,7 +53,7 @@ func (st *appServiceTarget) Package(
 	return async.RunTaskWithProgress(
 		func(task *async.TaskContextWithProgress[*ServicePackageResult, ServiceProgress]) {
 			task.SetProgress(NewServiceProgress("Compressing deployment artifacts"))
-			zipFilePath, err := internal.CreateDeployableZip(serviceConfig.Name, buildOutput.BuildOutputPath)
+			zipFilePath, err := createDeployableZip(serviceConfig.Name, buildOutput.BuildOutputPath)
 			if err != nil {
 				task.SetError(err)
 				return

cli/azd/pkg/project/service_target_functionapp.go

@@ -13,7 +13,6 @@ import (
 	"github.com/azure/azure-dev/cli/azd/pkg/azure"
 	"github.com/azure/azure-dev/cli/azd/pkg/environment"
 	"github.com/azure/azure-dev/cli/azd/pkg/infra"
-	"github.com/azure/azure-dev/cli/azd/pkg/project/internal"
 	"github.com/azure/azure-dev/cli/azd/pkg/tools"
 	"github.com/azure/azure-dev/cli/azd/pkg/tools/azcli"
 )
@@ -55,7 +54,7 @@ func (f *functionAppTarget) Package(
 	return async.RunTaskWithProgress(
 		func(task *async.TaskContextWithProgress[*ServicePackageResult, ServiceProgress]) {
 			task.SetProgress(NewServiceProgress("Compressing deployment artifacts"))
-			zipFilePath, err := internal.CreateDeployableZip(serviceConfig.Name, buildOutput.BuildOutputPath)
+			zipFilePath, err := createDeployableZip(serviceConfig.Name, buildOutput.BuildOutputPath)
 			if err != nil {
 				task.SetError(err)
 				return

go.mod

@@ -32,7 +32,7 @@ require (
 	github.com/mattn/go-isatty v0.0.14
 	github.com/microsoft/ApplicationInsights-Go v0.4.4
 	github.com/microsoft/azure-devops-go-api/azuredevops v1.0.0-b5
-	github.com/otiai10/copy v1.7.0
+	github.com/otiai10/copy v1.9.0
 	github.com/sethvargo/go-retry v0.2.3
 	github.com/spf13/cobra v1.3.0
 	github.com/spf13/pflag v1.0.5

go.sum

@@ -138,7 +138,6 @@ github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWH
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
-github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -331,9 +330,8 @@ github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.4 h1:5Myjjh3JY/NaAi4IsUbHADytDyl1VE1Y9PXDlL+P/VQ=
 github.com/kr/pty v1.1.4/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
-github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w=
@@ -383,13 +381,13 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLA
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/otiai10/copy v1.7.0 h1:hVoPiN+t+7d2nzzwMiDHPSOogsWAStewq3TwU05+clE=
-github.com/otiai10/copy v1.7.0/go.mod h1:rmRl6QPdJj6EiUqXQ/4Nn2lLXoNQjFCQbbNrxgc/t3U=
+github.com/otiai10/copy v1.9.0 h1:7KFNiCgZ91Ru4qW4CWPf/7jqtxLagGRmIxWldPP9VY4=
+github.com/otiai10/copy v1.9.0/go.mod h1:hsfX19wcn0UWIHUQ3/4fHuehhk2UyArQ9dVFAn3FczI=
 github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
 github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs=
 github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
-github.com/otiai10/mint v1.3.3 h1:7JgpsBaN0uMkyju4tbYHu0mnM55hNKVYLsXmwr15NQI=
-github.com/otiai10/mint v1.3.3/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc=
+github.com/otiai10/mint v1.4.0 h1:umwcf7gbpEwf7WFzqmWwSv0CzbeMsae2u9ZvpP8j2q4=
+github.com/otiai10/mint v1.4.0/go.mod h1:gifjb2MYOoULtKLqUAEILUG/9KONW6f7YsJ6vQLTlFI=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
@@ -684,6 +682,7 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=