Transform source files

Given a representation of a file from a source repository, transform
it into its destination in a target directory.  This includes
substituting variables in the file path, in any text file and
not-substituting unknown variables.

Signed-off-by: Aidan Delaney <adelaney21@bloomberg.net>

pkg/internal/init_test.go

@@ -7,7 +7,11 @@ import (
 	"github.com/sclevine/spec/report"
 )
 
-func TestInternal(t *testing.T) {
+func TestIternal(t *testing.T) {
 	spec.Run(t, "ReadPrompt", testReadPrompt, spec.Report(report.Terminal{}))
+	spec.Run(t, "Apply", testApply, spec.Report(report.Terminal{}))
 	spec.Run(t, "AskPrompts", testAskPrompts, spec.Report(report.Terminal{}))
+	spec.Run(t, "NoArgument", testApplyNoArgument, spec.Report(report.Terminal{}))
+	spec.Run(t, "Replace", testReplace, spec.Report(report.Terminal{}))
+	spec.Run(t, "Transform", testTransform, spec.Report(report.Terminal{}))
 }

pkg/internal/source_file.go

@@ -0,0 +1,90 @@
+package internal
+
+import (
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	t "github.com/coveooss/gotemplate/v3/template"
+)
+
+type SourceFile struct {
+	FilePath    string
+	FileContent string
+	FileMode    fs.FileMode
+}
+
+func (s SourceFile) Transform(inputDir string, outputDir string, vars map[string]string) error {
+	outputFile, err := s.Replace(vars)
+	if err != nil {
+		return err
+	}
+
+	dstDir := filepath.Join(outputDir, filepath.Dir(outputFile.FilePath))
+	mkdirErr := os.MkdirAll(dstDir, 0744)
+	if mkdirErr != nil {
+		return fmt.Errorf("failed to create target directory %s", dstDir)
+	}
+
+	outputPath := filepath.Join(outputDir, outputFile.FilePath)
+	if outputFile.FileContent == "" {
+		inputPath := filepath.Join(inputDir, s.FilePath)
+		mvErr := os.Rename(inputPath, outputPath)
+		if mvErr != nil {
+			return fmt.Errorf("failed to rename %s to %s", s.FilePath, outputFile.FilePath)
+		}
+	} else {
+		os.WriteFile(outputPath, []byte(outputFile.FileContent), outputFile.FileMode|0600)
+	}
+	return nil
+}
+
+func replaceUnknownVars(vars map[string]string, content string) string {
+	regex := regexp.MustCompile(`{{[ \t]*\.\w+`)
+	transformed := content
+	for _, token := range regex.FindAllString(content, -1) {
+		candidate := strings.Split(token, ".")[1]
+		if _, exists := vars[candidate]; !exists {
+			// replace "{{\s*.candidate" with "{&{&\s*.candidate"
+			replacement := strings.Replace(token, "{{", ReplacementDelimiter, 1)
+			transformed = strings.ReplaceAll(transformed, token, replacement)
+		}
+	}
+	return transformed
+}
+
+func (s SourceFile) Replace(vars map[string]string) (SourceFile, error) {
+	opts := t.DefaultOptions().
+		Set(t.Overwrite, t.Sprig, t.StrictErrorCheck, t.AcceptNoValue).
+		Unset(t.Razor)
+	template, err := t.NewTemplate(
+		"",
+		vars,
+		"",
+		opts)
+	if err != nil {
+		return SourceFile{}, err
+	}
+
+	filePath := replaceUnknownVars(vars, s.FilePath)
+	transformedFilePath, err := template.ProcessContent(filePath, "")
+	if err != nil {
+		return SourceFile{}, err
+	}
+	transformedFilePath = strings.ReplaceAll(transformedFilePath, ReplacementDelimiter, "{{")
+
+	transformedFileContent := ""
+	if s.FileContent != "" {
+		fileContent := replaceUnknownVars(vars, s.FileContent)
+		transformedFileContent, err = template.ProcessContent(fileContent, "")
+		if err != nil {
+			return SourceFile{}, err
+		}
+		transformedFileContent = strings.ReplaceAll(transformedFileContent, ReplacementDelimiter, "{{")
+	}
+
+	return SourceFile{FilePath: transformedFilePath, FileContent: transformedFileContent}, nil
+}

pkg/internal/source_file_test.go

@@ -0,0 +1,97 @@
+package internal_test
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	h "github.com/buildpacks/pack/testhelpers"
+	"github.com/sclevine/spec"
+
+	"github.com/buildpacks/scafall/pkg/internal"
+)
+
+func testReplace(t *testing.T, when spec.G, it spec.S) {
+	type TestCase struct {
+		file         internal.SourceFile
+		vars         map[string]string
+		expectedName string
+	}
+
+	testCases := []TestCase{
+		{
+			internal.SourceFile{FilePath: "{{.Foo}}", FileContent: ""},
+			map[string]string{"Foo": "Bar"},
+			"Bar",
+		},
+		{
+			internal.SourceFile{FilePath: "{{.Foo}}"},
+			map[string]string{"Bar": "Bar"},
+			"{{.Foo}}",
+		},
+	}
+	for _, testCase := range testCases {
+		current := testCase
+		when("variable replacement is called", func() {
+			it("correctly replaces tokens", func() {
+				output, err := current.file.Replace(current.vars)
+				h.AssertNil(t, err)
+				h.AssertEq(t, output.FilePath, current.expectedName)
+			})
+		})
+	}
+}
+
+func testTransform(t *testing.T, when spec.G, it spec.S) {
+	type TestCase struct {
+		file            internal.SourceFile
+		vars            map[string]string
+		expectedName    string
+		expectedContent string
+	}
+	testCases := []TestCase{
+		{
+			internal.SourceFile{FilePath: "{{.Foo}}", FileContent: "{{.Foo}}"},
+			map[string]string{"Foo": "Bar"},
+			"Bar",
+			"Bar",
+		},
+		{
+			internal.SourceFile{FilePath: "{{.Foo}}"},
+			map[string]string{"Bar": "Bar"},
+			"{{.Foo}}",
+			"",
+		},
+	}
+	for _, testCase := range testCases {
+		testCase := testCase
+		when("variable replacement is called", func() {
+			var (
+				inputDir  string
+				outputDir string
+			)
+			it.Before(func() {
+				var err error
+				inputDir, err = os.MkdirTemp("", "scafall")
+				h.AssertNil(t, err)
+				err = os.WriteFile(filepath.Join(inputDir, testCase.file.FilePath), []byte(testCase.file.FileContent), 0400)
+				h.AssertNil(t, err)
+				outputDir, err = os.MkdirTemp("", "scafall")
+				h.AssertNil(t, err)
+			})
+			it.After(func() {
+				_ = os.RemoveAll(inputDir)
+				_ = os.RemoveAll(outputDir)
+			})
+
+			it("correctly replaces tokens", func() {
+				err := testCase.file.Transform(inputDir, outputDir, testCase.vars)
+				h.AssertNil(t, err)
+
+				contents, err := os.ReadFile(filepath.Join(outputDir, testCase.expectedName))
+				h.AssertNil(t, err)
+				h.AssertEq(t, string(contents), testCase.expectedContent)
+			})
+		})
+	}
+}

pkg/internal/transform.go

@@ -0,0 +1,116 @@
+package internal
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/BurntSushi/toml"
+	"github.com/pkg/errors"
+
+	"github.com/gabriel-vasile/mimetype"
+
+	"github.com/buildpacks/scafall/pkg/internal/util"
+)
+
+const (
+	OverrideFile         string = ".override.toml"
+	ReplacementDelimiter string = "{&{&"
+)
+
+var (
+	IgnoredNames       = []string{PromptFile, OverrideFile}
+	IgnoredDirectories = []string{".git", "node_modules"}
+)
+
+func ReadFile(path string) (string, error) {
+	buf, err := os.ReadFile(path)
+	if err != nil {
+		return "", fmt.Errorf("cannot read file %s", path)
+	}
+	return string(buf), nil
+}
+
+func ReadOverrides(overrideFile string) (map[string]string, error) {
+	var overrides map[string]string
+	// if no override file
+	if _, err := os.Stat(overrideFile); err != nil {
+		return nil, nil
+	}
+
+	overrideData, err := ReadFile(overrideFile)
+	if err != nil {
+		return nil, err
+	}
+
+	if _, err := toml.Decode(overrideData, &overrides); err != nil {
+		return nil, errors.Wrap(err, fmt.Sprintf("%s file does not match required format", overrideFile))
+	}
+
+	return overrides, nil
+}
+
+func Apply(inputDir string, vars map[string]string, outputDir string) error {
+	if vars == nil {
+		vars = map[string]string{}
+	}
+	files, err := findTransformableFiles(inputDir)
+	if err != nil {
+		return fmt.Errorf("failed to find files in input folder: %s %s", inputDir, err)
+	}
+
+	for _, file := range files {
+		err := file.Transform(inputDir, outputDir, vars)
+		if err != nil {
+			return errors.Wrap(err, fmt.Sprintf("failed to transform %s", file.FilePath))
+		}
+	}
+
+	return err
+}
+
+func findTransformableFiles(dir string) ([]SourceFile, error) {
+	files := []SourceFile{}
+	err := filepath.WalkDir(dir, func(path string, info os.DirEntry, err error) error {
+		if info.IsDir() && util.Contains(IgnoredDirectories, info.Name()) {
+			return filepath.SkipDir
+		}
+
+		if !info.IsDir() {
+			// Ignore all prompts.toml files and any top-level README.md
+			rootReadme := filepath.Join(dir, "README")
+			if util.Contains(IgnoredNames, info.Name()) || strings.HasPrefix(path, rootReadme) {
+				return nil
+			}
+
+			relPath := strings.TrimPrefix(path, dir+"/")
+			if isTextfile(path) {
+				fileContent, err := ReadFile(path)
+				if err != nil {
+					return err
+				}
+				fileMode := info.Type().Perm()
+				files = append(files, SourceFile{FilePath: relPath, FileContent: fileContent, FileMode: fileMode})
+			} else {
+				files = append(files, SourceFile{FilePath: relPath, FileContent: ""})
+			}
+		}
+		return nil
+	})
+
+	return files, err
+}
+
+func isTextfile(path string) bool {
+	fd, err := os.Open(path)
+	if err != nil {
+		return false
+	}
+	mtype, err := mimetype.DetectReader(fd)
+	if err != nil {
+		return false
+	}
+
+	return strings.HasPrefix(mtype.String(), "text")
+}