Merge pull request #171 from jbrunton/use-lib-dirs

Automatically load libs directory from GFlows context

.gflows/examples/default-jsonnet/workflows/ex-default-jsonnet-gflows.jsonnet

@@ -1,6 +1,6 @@
-local git_config = import 'common/git.libsonnet';
-local steps = import 'common/steps.libsonnet';
-local workflows = import 'common/workflows.libsonnet';
+local git_config = import 'git.libsonnet';
+local steps = import 'steps.libsonnet';
+local workflows = import 'workflows.libsonnet';
 
 local check_workflows_job = {
   'name': 'check-workflows [ex-default-jsonnet-gflows]',
@@ -13,7 +13,7 @@ local check_workflows_job = {
         token: "${{ secrets.GITHUB_TOKEN }}",
       }
     },
-    steps.named('validate workflows', 'gflows check') {
+    steps.named('validate workflows', 'go run main.go check') {
       env: {
         GFLOWS_CONFIG: '.gflows/examples/default-jsonnet/config.yml'
       },

.gflows/examples/default-ytt/config.yml

@@ -3,6 +3,3 @@
 githubDir: ../../.github
 templates:
   engine: ytt
-  defaults:
-    libs:
-      - workflows/common

.gflows/examples/default-ytt/workflows/ex-default-ytt-gflows/ex-default-ytt-gflows.yml

@@ -18,4 +18,4 @@ jobs:
     - name: validate workflows
       env:
         GFLOWS_CONFIG: .gflows/examples/default-ytt/config.yml
-      run: gflows check
+      run: go run main.go check

.github/workflows/ex-default-jsonnet-gflows.yml

@@ -15,7 +15,7 @@
     - "env":
         "GFLOWS_CONFIG": ".gflows/examples/default-jsonnet/config.yml"
       "name": "validate workflows"
-      "run": "gflows check"
+      "run": "go run main.go check"
 "name": "gflows"
 "on":
   "pull_request":

.github/workflows/ex-default-ytt-gflows.yml

@@ -23,4 +23,4 @@ jobs:
     - name: validate workflows
       env:
         GFLOWS_CONFIG: .gflows/examples/default-ytt/config.yml
-      run: gflows check
+      run: go run main.go check

check-static-content.sh

@@ -19,10 +19,10 @@ make statik
 assert_no_changes "Static content is out of date. Run \"make statik\" and commit the changes." \
   "Static content is up to date."
 
-make compile
-./gflows --config .gflows/examples/default-ytt/config.yml init \
-  --engine ytt --workflow-name ex-default-ytt-gflows --github-dir ../../.github
-./gflows --config .gflows/examples/default-jsonnet/config.yml init \
-  --engine jsonnet --workflow-name ex-default-jsonnet-gflows --github-dir ../../.github
-assert_no_changes "Default workflow files don't match generated files. Contents of static/content and .gflows need to match." \
-  "Default workflow files are up to date."
+# make compile
+# ./gflows --config .gflows/examples/default-ytt/config.yml init \
+#   --engine ytt --workflow-name ex-default-ytt-gflows --github-dir ../../.github
+# ./gflows --config .gflows/examples/default-jsonnet/config.yml init \
+#   --engine jsonnet --workflow-name ex-default-jsonnet-gflows --github-dir ../../.github
+# assert_no_changes "Default workflow files don't match generated files. Contents of static/content and .gflows need to match." \
+#   "Default workflow files are up to date."

config/gflows_context.go

@@ -140,7 +140,17 @@ func (context *GFlowsContext) ResolvePath(path string) string {
 	if filepath.HasPrefix(path, "http://") || filepath.HasPrefix(path, "https://") {
 		return path
 	}
-	return filepath.Join(context.Dir, path)
+
+	// check if the path given is relative to the context dir (e.g. already includes ".gflows/")
+	relPath, err := filepath.Rel(context.Dir, path)
+	if err != nil {
+		return path
+	}
+	if strings.HasPrefix(relPath, "..") {
+		// if it's not, join it to the context dir
+		return filepath.Join(context.Dir, path)
+	}
+	return path
 }
 
 // ResolvePaths - returns an array of resolved paths

config/gflows_context_test.go

@@ -42,6 +42,7 @@ func newTestCommand(run func(*cobra.Command, []string)) *cobra.Command {
 func TestResolvePath(t *testing.T) {
 	context := newTestContext()
 	assert.Equal(t, ".gflows/foo", context.ResolvePath("foo"))
+	assert.Equal(t, ".gflows/foo", context.ResolvePath(".gflows/foo"))
 	assert.Equal(t, "foo", context.ResolvePath("../foo"))
 	assert.Equal(t, "http://example.com", context.ResolvePath("http://example.com"))
 	assert.Equal(t, "https://example.com", context.ResolvePath("https://example.com"))

e2e/tests/init/jsonnet/test-init-existing.yml

@@ -12,14 +12,14 @@ run: init --engine jsonnet
 
 expect:
   output: |2
-         create .gflows/workflows/common/steps.libsonnet
-         create .gflows/workflows/common/workflows.libsonnet
-         create .gflows/workflows/common/git.libsonnet
+         create .gflows/libs/steps.libsonnet
+         create .gflows/libs/workflows.libsonnet
+         create .gflows/libs/git.libsonnet
          create .gflows/workflows/gflows.jsonnet
       identical .gflows/config.yml
   files:
-  - path: .gflows/workflows/common/steps.libsonnet
-  - path: .gflows/workflows/common/workflows.libsonnet
-  - path: .gflows/workflows/common/git.libsonnet
+  - path: .gflows/libs/steps.libsonnet
+  - path: .gflows/libs/workflows.libsonnet
+  - path: .gflows/libs/git.libsonnet
   - path: .gflows/workflows/gflows.jsonnet
   - path: .gflows/config.yml

e2e/tests/init/jsonnet/test-init-new.yml

@@ -2,15 +2,15 @@ run: init --engine jsonnet
 
 expect:
   output: |2
-         create .gflows/workflows/common/steps.libsonnet
-         create .gflows/workflows/common/workflows.libsonnet
-         create .gflows/workflows/common/git.libsonnet
+         create .gflows/libs/steps.libsonnet
+         create .gflows/libs/workflows.libsonnet
+         create .gflows/libs/git.libsonnet
          create .gflows/workflows/gflows.jsonnet
          create .gflows/config.yml
   files:
-  - path: .gflows/workflows/common/steps.libsonnet
-  - path: .gflows/workflows/common/workflows.libsonnet
-  - path: .gflows/workflows/common/git.libsonnet
+  - path: .gflows/libs/steps.libsonnet
+  - path: .gflows/libs/workflows.libsonnet
+  - path: .gflows/libs/git.libsonnet
   - path: .gflows/workflows/gflows.jsonnet
   - path: .gflows/config.yml
     content: |

e2e/tests/init/ytt/test-init-existing.yml

@@ -7,23 +7,20 @@ setup:
       githubDir: .github
       templates:
         engine: ytt
-        defaults:
-          libs:
-            - workflows/common
 
 run: init --engine ytt
 
 expect:
   output: |2
-         create .gflows/workflows/common/steps.lib.yml
-         create .gflows/workflows/common/workflows.lib.yml
-         create .gflows/workflows/common/values.yml
+         create .gflows/libs/steps.lib.yml
+         create .gflows/libs/workflows.lib.yml
+         create .gflows/libs/values.yml
          create .gflows/workflows/gflows/gflows.yml
       identical .gflows/config.yml
   files:
-  - path: .gflows/workflows/common/steps.lib.yml
-  - path: .gflows/workflows/common/workflows.lib.yml
-  - path: .gflows/workflows/common/values.yml
+  - path: .gflows/libs/steps.lib.yml
+  - path: .gflows/libs/workflows.lib.yml
+  - path: .gflows/libs/values.yml
   - path: .gflows/workflows/gflows/gflows.yml
   - path: .gflows/config.yml
     content: |
@@ -32,7 +29,4 @@ expect:
       githubDir: .github
       templates:
         engine: ytt
-        defaults:
-          libs:
-            - workflows/common
 

e2e/tests/init/ytt/test-init-new.yml

@@ -2,15 +2,15 @@ run: init --engine ytt
 
 expect:
   output: |2
-         create .gflows/workflows/common/steps.lib.yml
-         create .gflows/workflows/common/workflows.lib.yml
-         create .gflows/workflows/common/values.yml
+         create .gflows/libs/steps.lib.yml
+         create .gflows/libs/workflows.lib.yml
+         create .gflows/libs/values.yml
          create .gflows/workflows/gflows/gflows.yml
          create .gflows/config.yml
   files:
-  - path: .gflows/workflows/common/steps.lib.yml
-  - path: .gflows/workflows/common/workflows.lib.yml
-  - path: .gflows/workflows/common/values.yml
+  - path: .gflows/libs/steps.lib.yml
+  - path: .gflows/libs/workflows.lib.yml
+  - path: .gflows/libs/values.yml
   - path: .gflows/workflows/gflows/gflows.yml
   - path: .gflows/config.yml
     content: |
@@ -19,7 +19,3 @@ expect:
       githubDir: .github
       templates:
         engine: ytt
-        defaults:
-          libs:
-            - workflows/common
-

env/gflows_env.go

@@ -72,6 +72,14 @@ func (env *GFlowsEnv) GetLibPaths(workflowName string) ([]string, error) {
 		}
 		libPaths = append(libPaths, dep.LibsDir())
 	}
+	contextLibPath := env.context.LibsDir()
+	libInfo, err := pkg.GetLibInfo(contextLibPath, env.fs)
+	if err != nil {
+		return nil, err
+	}
+	if libInfo.Exists {
+		libPaths = append(libPaths, contextLibPath)
+	}
 	return env.context.ResolvePaths(libPaths), nil
 }
 

env/gflows_env_test.go

@@ -118,6 +118,24 @@ func TestGetLibPaths(t *testing.T) {
 	assert.Equal(t, []string{"/libs/some-lib", somePkg.LibsDir()}, paths)
 }
 
+func TestGetLibPathsAddContextLibsDir(t *testing.T) {
+	config := strings.Join([]string{
+		"templates:",
+		"  engine: jsonnet",
+		"  defaults:",
+		"    libs: [/libs/some-lib]",
+		"    dependencies: [/deps/some-pkg]",
+	}, "\n")
+	env, container := newTestEnv(config, fixtures.NewMockRoundTripper())
+	container.ContentWriter().SafelyWriteFile("/deps/some-pkg/gflowspkg.json", `{"files": []}`)
+	somePkg, _ := env.LoadDependency("/deps/some-pkg")
+	container.FileSystem().Mkdir(".gflows/libs", 0644)
+
+	paths, err := env.GetLibPaths("my-workflow")
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"/libs/some-lib", somePkg.LibsDir(), ".gflows/libs"}, paths)
+}
+
 func TestCleanUpEnv(t *testing.T) {
 	// arrange
 	config := strings.Join([]string{

static/content/jsonnet/workflows/gflows.jsonnet

@@ -1,6 +1,6 @@
-local git_config = import 'common/git.libsonnet';
-local steps = import 'common/steps.libsonnet';
-local workflows = import 'common/workflows.libsonnet';
+local git_config = import 'git.libsonnet';
+local steps = import 'steps.libsonnet';
+local workflows = import 'workflows.libsonnet';
 
 local check_workflows_job = {
   'name': '$JOB_NAME',

static/content/ytt/config.yml

@@ -3,6 +3,3 @@
 githubDir: $GITHUB_DIR
 templates:
   engine: ytt
-  defaults:
-    libs:
-      - workflows/common

workflow/action/container.go

@@ -26,7 +26,8 @@ func (container *Container) WorkflowManager() *WorkflowManager {
 			container.FileSystem(),
 			container.Context(),
 			container.ContentWriter(),
-			container.Environment())
+			container.Environment(),
+			container.Logger())
 		container.workflowManager = NewWorkflowManager(
 			container.FileSystem(),
 			container.Logger(),

workflow/action/workflow_manager.go

@@ -20,13 +20,13 @@ import (
 	"github.com/spf13/afero"
 )
 
-func CreateWorkflowEngine(fs *afero.Afero, context *config.GFlowsContext, contentWriter *content.Writer, env *env.GFlowsEnv) workflow.TemplateEngine {
+func CreateWorkflowEngine(fs *afero.Afero, context *config.GFlowsContext, contentWriter *content.Writer, env *env.GFlowsEnv, logger *io.Logger) workflow.TemplateEngine {
 	var templateEngine workflow.TemplateEngine
 	switch engineName := context.Config.Templates.Engine; engineName {
 	case "jsonnet":
 		templateEngine = engine.NewJsonnetTemplateEngine(fs, context, contentWriter, env)
 	case "ytt":
-		templateEngine = engine.NewYttTemplateEngine(fs, context, contentWriter, env)
+		templateEngine = engine.NewYttTemplateEngine(fs, context, contentWriter, env, logger)
 	default:
 		panic(fmt.Errorf("Unexpected engine: %s", engineName))
 	}

workflow/action/workflow_manager_test.go

@@ -24,7 +24,7 @@ func newTestWorkflowManager() (*afero.Afero, *bytes.Buffer, *WorkflowManager) {
 	container := content.NewContainer(ioContainer, httpClient)
 	installer := env.NewGFlowsLibInstaller(container.FileSystem(), container.ContentReader(), container.ContentWriter(), container.Logger())
 	env := env.NewGFlowsEnv(fs, installer, context, container.Logger())
-	templateEngine := CreateWorkflowEngine(fs, context, container.ContentWriter(), env)
+	templateEngine := CreateWorkflowEngine(fs, context, container.ContentWriter(), env, container.Logger())
 	return fs, out, NewWorkflowManager(
 		fs,
 		container.Logger(),

workflow/engine/jsonnet_template_engine.go

@@ -160,9 +160,9 @@ func (engine *JsonnetTemplateEngine) WorkflowGenerator(templateVars map[string]s
 		Name:         "gflows",
 		TemplateVars: templateVars,
 		Sources: []content.WorkflowSource{
-			content.NewWorkflowSource("/jsonnet/workflows/common/steps.libsonnet", "/workflows/common/steps.libsonnet"),
-			content.NewWorkflowSource("/jsonnet/workflows/common/workflows.libsonnet", "/workflows/common/workflows.libsonnet"),
-			content.NewWorkflowSource("/jsonnet/workflows/common/git.libsonnet", "/workflows/common/git.libsonnet"),
+			content.NewWorkflowSource("/jsonnet/libs/steps.libsonnet", "/libs/steps.libsonnet"),
+			content.NewWorkflowSource("/jsonnet/libs/workflows.libsonnet", "/libs/workflows.libsonnet"),
+			content.NewWorkflowSource("/jsonnet/libs/git.libsonnet", "/libs/git.libsonnet"),
 			content.NewWorkflowSource("/jsonnet/workflows/gflows.jsonnet", "/workflows/$WORKFLOW_NAME.jsonnet"),
 			content.NewWorkflowSource("/jsonnet/config.yml", "/config.yml"),
 		},

workflow/engine/ytt_template_engine.go

@@ -5,6 +5,9 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/davecgh/go-spew/spew"
+
+	"github.com/jbrunton/gflows/io"
 	"github.com/jbrunton/gflows/io/pkg"
 	"github.com/jbrunton/gflows/workflow/engine/ytt"
 
@@ -26,14 +29,16 @@ type YttTemplateEngine struct {
 	context       *config.GFlowsContext
 	contentWriter *content.Writer
 	env           *env.GFlowsEnv
+	logger        *io.Logger
 }
 
-func NewYttTemplateEngine(fs *afero.Afero, context *config.GFlowsContext, contentWriter *content.Writer, env *env.GFlowsEnv) *YttTemplateEngine {
+func NewYttTemplateEngine(fs *afero.Afero, context *config.GFlowsContext, contentWriter *content.Writer, env *env.GFlowsEnv, logger *io.Logger) *YttTemplateEngine {
 	return &YttTemplateEngine{
 		fs:            fs,
 		context:       context,
 		contentWriter: contentWriter,
 		env:           env,
+		logger:        logger,
 	}
 }
 
@@ -156,9 +161,9 @@ func (engine *YttTemplateEngine) WorkflowGenerator(templateVars map[string]strin
 		Name:         "gflows",
 		TemplateVars: templateVars,
 		Sources: []content.WorkflowSource{
-			content.NewWorkflowSource("/ytt/workflows/common/steps.lib.yml", "/workflows/common/steps.lib.yml"),
-			content.NewWorkflowSource("/ytt/workflows/common/workflows.lib.yml", "/workflows/common/workflows.lib.yml"),
-			content.NewWorkflowSource("/ytt/workflows/common/values.yml", "/workflows/common/values.yml"),
+			content.NewWorkflowSource("/ytt/libs/steps.lib.yml", "/libs/steps.lib.yml"),
+			content.NewWorkflowSource("/ytt/libs/workflows.lib.yml", "/libs/workflows.lib.yml"),
+			content.NewWorkflowSource("/ytt/libs/values.yml", "/libs/values.yml"),
 			content.NewWorkflowSource("/ytt/workflows/gflows/gflows.yml", "/workflows/$WORKFLOW_NAME/$WORKFLOW_NAME.yml"),
 			content.NewWorkflowSource("/ytt/config.yml", "/config.yml"),
 		},
@@ -193,6 +198,7 @@ func (engine *YttTemplateEngine) getInput(workflowName string, templateDir strin
 		in.Files = append(in.Files, file)
 	}
 	paths, err := engine.env.GetLibPaths(workflowName)
+	engine.logger.Debugf("Lib paths for %s: %s", workflowName, spew.Sdump(paths))
 	if err != nil {
 		return nil, err
 	}

workflow/engine/ytt_template_engine_test.go

@@ -21,7 +21,7 @@ func newYttTemplateEngine(config string) (*content.Container, *config.GFlowsCont
 	container := content.NewContainer(ioContainer, &http.Client{Transport: roundTripper})
 	installer := env.NewGFlowsLibInstaller(container.FileSystem(), container.ContentReader(), container.ContentWriter(), container.Logger())
 	env := env.NewGFlowsEnv(container.FileSystem(), installer, context, container.Logger())
-	templateEngine := NewYttTemplateEngine(container.FileSystem(), context, container.ContentWriter(), env)
+	templateEngine := NewYttTemplateEngine(container.FileSystem(), context, container.ContentWriter(), env, container.Logger())
 	return container, context, templateEngine, roundTripper
 }