From 6286d0f7bf7839a9ab3771b7948cb6dfc6641796 Mon Sep 17 00:00:00 2001 From: Aidan Delaney Date: Wed, 28 Dec 2022 12:03:01 +0000 Subject: [PATCH] Transform source files Given a representation of a file from a source repository, transform it into its destination in a target directory. This includes substituting variables in the file path, in any text file and not-substituting unknown variables. Signed-off-by: Aidan Delaney --- pkg/internal/init_test.go | 6 +- pkg/internal/source_file.go | 90 ++++++++++++++++++++++++ pkg/internal/source_file_test.go | 97 ++++++++++++++++++++++++++ pkg/internal/transform.go | 116 +++++++++++++++++++++++++++++++ pkg/internal/transform_test.go | 93 +++++++++++++++++++++++++ pkg/internal/util/util.go | 10 +++ 6 files changed, 411 insertions(+), 1 deletion(-) create mode 100644 pkg/internal/source_file.go create mode 100644 pkg/internal/source_file_test.go create mode 100644 pkg/internal/transform.go create mode 100644 pkg/internal/transform_test.go create mode 100644 pkg/internal/util/util.go diff --git a/pkg/internal/init_test.go b/pkg/internal/init_test.go index 7e56a8a3..a46cc793 100644 --- a/pkg/internal/init_test.go +++ b/pkg/internal/init_test.go @@ -7,7 +7,11 @@ import ( "github.com/sclevine/spec/report" ) -func TestInternal(t *testing.T) { +func TestIternal(t *testing.T) { spec.Run(t, "ReadPrompt", testReadPrompt, spec.Report(report.Terminal{})) + spec.Run(t, "Apply", testApply, spec.Report(report.Terminal{})) spec.Run(t, "AskPrompts", testAskPrompts, spec.Report(report.Terminal{})) + spec.Run(t, "NoArgument", testApplyNoArgument, spec.Report(report.Terminal{})) + spec.Run(t, "Replace", testReplace, spec.Report(report.Terminal{})) + spec.Run(t, "Transform", testTransform, spec.Report(report.Terminal{})) } diff --git a/pkg/internal/source_file.go b/pkg/internal/source_file.go new file mode 100644 index 00000000..6cbb4a06 --- /dev/null +++ b/pkg/internal/source_file.go @@ -0,0 +1,90 @@ +package internal + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + "regexp" + "strings" + + t "github.com/coveooss/gotemplate/v3/template" +) + +type SourceFile struct { + FilePath string + FileContent string + FileMode fs.FileMode +} + +func (s SourceFile) Transform(inputDir string, outputDir string, vars map[string]string) error { + outputFile, err := s.Replace(vars) + if err != nil { + return err + } + + dstDir := filepath.Join(outputDir, filepath.Dir(outputFile.FilePath)) + mkdirErr := os.MkdirAll(dstDir, 0744) + if mkdirErr != nil { + return fmt.Errorf("failed to create target directory %s", dstDir) + } + + outputPath := filepath.Join(outputDir, outputFile.FilePath) + if outputFile.FileContent == "" { + inputPath := filepath.Join(inputDir, s.FilePath) + mvErr := os.Rename(inputPath, outputPath) + if mvErr != nil { + return fmt.Errorf("failed to rename %s to %s", s.FilePath, outputFile.FilePath) + } + } else { + os.WriteFile(outputPath, []byte(outputFile.FileContent), outputFile.FileMode|0600) + } + return nil +} + +func replaceUnknownVars(vars map[string]string, content string) string { + regex := regexp.MustCompile(`{{[ \t]*\.\w+`) + transformed := content + for _, token := range regex.FindAllString(content, -1) { + candidate := strings.Split(token, ".")[1] + if _, exists := vars[candidate]; !exists { + // replace "{{\s*.candidate" with "{&{&\s*.candidate" + replacement := strings.Replace(token, "{{", ReplacementDelimiter, 1) + transformed = strings.ReplaceAll(transformed, token, replacement) + } + } + return transformed +} + +func (s SourceFile) Replace(vars map[string]string) (SourceFile, error) { + opts := t.DefaultOptions(). + Set(t.Overwrite, t.Sprig, t.StrictErrorCheck, t.AcceptNoValue). + Unset(t.Razor) + template, err := t.NewTemplate( + "", + vars, + "", + opts) + if err != nil { + return SourceFile{}, err + } + + filePath := replaceUnknownVars(vars, s.FilePath) + transformedFilePath, err := template.ProcessContent(filePath, "") + if err != nil { + return SourceFile{}, err + } + transformedFilePath = strings.ReplaceAll(transformedFilePath, ReplacementDelimiter, "{{") + + transformedFileContent := "" + if s.FileContent != "" { + fileContent := replaceUnknownVars(vars, s.FileContent) + transformedFileContent, err = template.ProcessContent(fileContent, "") + if err != nil { + return SourceFile{}, err + } + transformedFileContent = strings.ReplaceAll(transformedFileContent, ReplacementDelimiter, "{{") + } + + return SourceFile{FilePath: transformedFilePath, FileContent: transformedFileContent}, nil +} diff --git a/pkg/internal/source_file_test.go b/pkg/internal/source_file_test.go new file mode 100644 index 00000000..c1039f41 --- /dev/null +++ b/pkg/internal/source_file_test.go @@ -0,0 +1,97 @@ +package internal_test + +import ( + "os" + "path/filepath" + "testing" + + h "github.com/buildpacks/pack/testhelpers" + "github.com/sclevine/spec" + + "github.com/buildpacks/scafall/pkg/internal" +) + +func testReplace(t *testing.T, when spec.G, it spec.S) { + type TestCase struct { + file internal.SourceFile + vars map[string]string + expectedName string + } + + testCases := []TestCase{ + { + internal.SourceFile{FilePath: "{{.Foo}}", FileContent: ""}, + map[string]string{"Foo": "Bar"}, + "Bar", + }, + { + internal.SourceFile{FilePath: "{{.Foo}}"}, + map[string]string{"Bar": "Bar"}, + "{{.Foo}}", + }, + } + for _, testCase := range testCases { + current := testCase + when("variable replacement is called", func() { + it("correctly replaces tokens", func() { + output, err := current.file.Replace(current.vars) + h.AssertNil(t, err) + h.AssertEq(t, output.FilePath, current.expectedName) + }) + }) + } +} + +func testTransform(t *testing.T, when spec.G, it spec.S) { + type TestCase struct { + file internal.SourceFile + vars map[string]string + expectedName string + expectedContent string + } + testCases := []TestCase{ + { + internal.SourceFile{FilePath: "{{.Foo}}", FileContent: "{{.Foo}}"}, + map[string]string{"Foo": "Bar"}, + "Bar", + "Bar", + }, + { + internal.SourceFile{FilePath: "{{.Foo}}"}, + map[string]string{"Bar": "Bar"}, + "{{.Foo}}", + "", + }, + } + for _, testCase := range testCases { + testCase := testCase + when("variable replacement is called", func() { + var ( + inputDir string + outputDir string + ) + it.Before(func() { + var err error + inputDir, err = os.MkdirTemp("", "scafall") + h.AssertNil(t, err) + err = os.WriteFile(filepath.Join(inputDir, testCase.file.FilePath), []byte(testCase.file.FileContent), 0400) + h.AssertNil(t, err) + outputDir, err = os.MkdirTemp("", "scafall") + h.AssertNil(t, err) + }) + it.After(func() { + _ = os.RemoveAll(inputDir) + _ = os.RemoveAll(outputDir) + }) + + it("correctly replaces tokens", func() { + err := testCase.file.Transform(inputDir, outputDir, testCase.vars) + h.AssertNil(t, err) + + contents, err := os.ReadFile(filepath.Join(outputDir, testCase.expectedName)) + h.AssertNil(t, err) + h.AssertEq(t, string(contents), testCase.expectedContent) + }) + }) + } +} diff --git a/pkg/internal/transform.go b/pkg/internal/transform.go new file mode 100644 index 00000000..611e1ec2 --- /dev/null +++ b/pkg/internal/transform.go @@ -0,0 +1,116 @@ +package internal + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/BurntSushi/toml" + "github.com/pkg/errors" + + "github.com/gabriel-vasile/mimetype" + + "github.com/buildpacks/scafall/pkg/internal/util" +) + +const ( + OverrideFile string = ".override.toml" + ReplacementDelimiter string = "{&{&" +) + +var ( + IgnoredNames = []string{PromptFile, OverrideFile} + IgnoredDirectories = []string{".git", "node_modules"} +) + +func ReadFile(path string) (string, error) { + buf, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("cannot read file %s", path) + } + return string(buf), nil +} + +func ReadOverrides(overrideFile string) (map[string]string, error) { + var overrides map[string]string + // if no override file + if _, err := os.Stat(overrideFile); err != nil { + return nil, nil + } + + overrideData, err := ReadFile(overrideFile) + if err != nil { + return nil, err + } + + if _, err := toml.Decode(overrideData, &overrides); err != nil { + return nil, errors.Wrap(err, fmt.Sprintf("%s file does not match required format", overrideFile)) + } + + return overrides, nil +} + +func Apply(inputDir string, vars map[string]string, outputDir string) error { + if vars == nil { + vars = map[string]string{} + } + files, err := findTransformableFiles(inputDir) + if err != nil { + return fmt.Errorf("failed to find files in input folder: %s %s", inputDir, err) + } + + for _, file := range files { + err := file.Transform(inputDir, outputDir, vars) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to transform %s", file.FilePath)) + } + } + + return err +} + +func findTransformableFiles(dir string) ([]SourceFile, error) { + files := []SourceFile{} + err := filepath.WalkDir(dir, func(path string, info os.DirEntry, err error) error { + if info.IsDir() && util.Contains(IgnoredDirectories, info.Name()) { + return filepath.SkipDir + } + + if !info.IsDir() { + // Ignore all prompts.toml files and any top-level README.md + rootReadme := filepath.Join(dir, "README") + if util.Contains(IgnoredNames, info.Name()) || strings.HasPrefix(path, rootReadme) { + return nil + } + + relPath := strings.TrimPrefix(path, dir+"/") + if isTextfile(path) { + fileContent, err := ReadFile(path) + if err != nil { + return err + } + fileMode := info.Type().Perm() + files = append(files, SourceFile{FilePath: relPath, FileContent: fileContent, FileMode: fileMode}) + } else { + files = append(files, SourceFile{FilePath: relPath, FileContent: ""}) + } + } + return nil + }) + + return files, err +} + +func isTextfile(path string) bool { + fd, err := os.Open(path) + if err != nil { + return false + } + mtype, err := mimetype.DetectReader(fd) + if err != nil { + return false + } + + return strings.HasPrefix(mtype.String(), "text") +} diff --git a/pkg/internal/transform_test.go b/pkg/internal/transform_test.go new file mode 100644 index 00000000..cf11a336 --- /dev/null +++ b/pkg/internal/transform_test.go @@ -0,0 +1,93 @@ +package internal_test + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/buildpacks/scafall/pkg/internal" + + h "github.com/buildpacks/pack/testhelpers" + "github.com/sclevine/spec" +) + +func testApply(t *testing.T, when spec.G, it spec.S) { + when("Applying to a filesystem", func() { + it("correctly replaces strings in a filesytem", func() { + tmpDir, _ := ioutil.TempDir("", "test") + defer os.RemoveAll(tmpDir) + outputDir, _ := ioutil.TempDir("", "test") + defer os.RemoveAll(outputDir) + err := os.MkdirAll(filepath.Join(tmpDir, "/{{.Foo}}/{{.Foo}}"), 0766) + h.AssertNil(t, err) + f, err := os.Create(filepath.Join(tmpDir, "/{{.Foo}}/{{.Foo}}/{{.Foo}}.txt")) + h.AssertNil(t, err) + f.Write([]byte("{{.Foo}}")) + f.Close() + vars := map[string]string{"Foo": "Bar"} + + err = internal.Apply(tmpDir, vars, outputDir) + h.AssertNil(t, err) + + bar, err := os.Open(filepath.Join(outputDir, "/Bar/Bar/Bar.txt")) + h.AssertNil(t, err) + h.AssertNotNil(t, bar) + + var c string + c, err = internal.ReadFile(filepath.Join(outputDir, "/Bar/Bar/Bar.txt")) + h.AssertNil(t, err) + h.AssertContains(t, c, "Bar") + }) + }) +} + +func testApplyNoArgument(t *testing.T, when spec.G, it spec.S) { + when("Applying to a file without argument", func() { + it("does not replace the template variable", func() { + tmpDir, _ := ioutil.TempDir("", "test") + defer os.RemoveAll(tmpDir) + outputDir, _ := ioutil.TempDir("", "test") + defer os.RemoveAll(outputDir) + testFile := filepath.Join(tmpDir, "test.txt") + content := "{{ .Foo }}" + os.WriteFile(testFile, []byte(content), 0600) + + err := internal.Apply(tmpDir, nil, outputDir) + h.AssertNil(t, err) + + c, err := internal.ReadFile(filepath.Join(outputDir, "test.txt")) + h.AssertNil(t, err) + h.AssertContains(t, c, content) + }) + }) + + when("Applying to a filesystem without argument", func() { + it("does not replace the template variable", func() { + tmpDir, _ := ioutil.TempDir("", "test") + defer os.RemoveAll(tmpDir) + outputDir, _ := ioutil.TempDir("", "test") + defer os.RemoveAll(outputDir) + err := os.MkdirAll(filepath.Join(tmpDir, "/{{.Foo}}/{{.Foo}}"), 0766) + h.AssertNil(t, err) + f, err := os.Create(filepath.Join(tmpDir, "/{{.Foo}}/{{.Foo}}/{{.Foo}}.txt")) + h.AssertNil(t, err) + f.Write([]byte("{{.Foo}}")) + f.Close() + vars := map[string]string{"Bar": "bar"} + + err = internal.Apply(tmpDir, vars, outputDir) + h.AssertNil(t, err) + + fooTxt := filepath.Join(outputDir, "/{{.Foo}}/{{.Foo}}/{{.Foo}}.txt") + foo, err := os.Stat(fooTxt) + h.AssertNil(t, err) + h.AssertNotNil(t, foo) + + var c string + c, err = internal.ReadFile(filepath.Join(outputDir, "/{{.Foo}}/{{.Foo}}/{{.Foo}}.txt")) + h.AssertNil(t, err) + h.AssertContains(t, c, "{{.Foo}}") + }) + }) +} diff --git a/pkg/internal/util/util.go b/pkg/internal/util/util.go new file mode 100644 index 00000000..3912df4a --- /dev/null +++ b/pkg/internal/util/util.go @@ -0,0 +1,10 @@ +package util + +func Contains(strings []string, element string) bool { + for _, s := range strings { + if s == element { + return true + } + } + return false +}