Reset switch in .flux.yaml

docs/references/fluxyaml-config-files.md

@@ -41,6 +41,9 @@ target path, or in a directory _above_ it in the git repository.
  - if no `.flux.yaml` file is found, the usual behaviour of looking
    for YAML files is adopted for that target path.
 
+ - a `.flux.yaml` file containing the `scanForFiles` directive resets
+   the behaviour to looking for YAML files. This is explained below.
+
 The manifests from all the target paths -- read from YAML files or
 generated -- are combined before applying to the cluster. If
 duplicates are detected, an error is logged and fluxd will abandon the
@@ -107,9 +110,58 @@ Note also that the configuration file would **not** take effect for
 `--git-path=.` (i.e., the top directory), because manifest generation
 will not look in subdirectories for a `.flux.yaml` file.
 
+### The `scanForFiles` directive
+
+The `scanForFiles` directive indicates that the target path should be
+treated as though it had _no_ `.flux.yaml` in effect. In other words,
+fluxd will look for YAML files under the directory, and update
+manifests directly by rewriting the YAML files.
+
+Here's an example `.flux.yaml` with the `scanForFiles` directive:
+
+```
+version: 1
+scanForFiles: {}
+```
+
+(The `{}` is an empty map, which acts as a placeholder value).
+
+This is to account for the case in which you have a `.flux.yaml`
+higher in the directory tree, applying to several target paths beneath
+it, but want to have a directory wth regular YAMLs as well.
+
+In the following example, the top-level `.flux.yaml` would take effect
+for `--git-path=staging` or `--git-path=production`.
+
+But if you wanted `yamls/permissions.yaml` to be applied (as it is),
+you could put a `.flux.yaml` containing `scanForFiles` in that directory, and
+specify `--git-path=staging,yamls`.
+
+```
+.
+├── .flux.yaml
+├── base
+│   ├── demo-ns.yaml
+│   ├── kustomization.yaml
+│   ├── podinfo-dep.yaml
+│   ├── podinfo-hpa.yaml
+│   └── podinfo-svc.yaml
+├── production
+│   ├── flux-patch.yaml
+│   ├── kustomization.yaml
+│   └── replicas-patch.yaml
+├── yamls
+│   ├── .flux.yaml # (with "scanForFiles" directive)
+│   └── permissions.yaml
+└── staging
+    ├── flux-patch.yaml
+    └── kustomization.yaml
+```
+
 ## How to construct a .flux.yaml file
 
-`.flux.yaml` files come in two varieties: "patch-updated", and
+Aside from the special case of the `scanForFiles` directive,
+`.flux.yaml` files come in two varieties: "patch-updated",
 "command-updated". These refer to the way in which [automated
 updates](./automated-image-update.md) are applied to files in the
 repo:

go.mod

@@ -67,6 +67,7 @@ require (
 	github.com/weaveworks/go-checkpoint v0.0.0-20170503165305-ebbb8b0518ab
 	github.com/weaveworks/promrus v1.2.0 // indirect
 	github.com/whilp/git-urls v0.0.0-20160530060445-31bac0d230fa
+	github.com/xeipuuv/gojsonschema v1.1.0
 	go.mozilla.org/sops/v3 v3.5.0
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
 	golang.org/x/sys v0.0.0-20191028164358-195ce5e7f934

pkg/manifests/configaware.go

@@ -81,6 +81,10 @@ func splitConfigFilesAndRawManifestPaths(baseDir string, paths []string) ([]*Con
 		if err != nil {
 			return nil, nil, fmt.Errorf("cannot parse config file: %s", err)
 		}
+		if cf.IsScanForFiles() {
+			rawManifestPaths = append(rawManifestPaths, path)
+			continue
+		}
 		configFiles = append(configFiles, cf)
 	}
 

pkg/manifests/configaware_test.go

@@ -262,7 +262,7 @@ spec:
 
 const mistakenConf = `
 version: 1
-commandUpdated: # <-- because this is commandUpdated, patchFile is ignored
+commandUpdated:
   generators:
     - command: |
        echo "apiVersion: extensions/v1beta1
@@ -283,7 +283,6 @@ commandUpdated: # <-- because this is commandUpdated, patchFile is ignored
        kind: Namespace
        metadata:
          name: demo"
-  patchFile: patchfile.yaml
 `
 
 // This tests that when using a config with no update commands, and
@@ -385,3 +384,30 @@ func TestDuplicateInGenerators(t *testing.T) {
 	assert.Error(t, err)
 	assert.Contains(t, err.Error(), "duplicate")
 }
+
+func TestSccanForFiles(t *testing.T) {
+	// +-- config
+	//   +-- .flux.yaml (patchUpdated)
+	//   +-- rawfiles
+	//     +-- .flux.yaml (scanForFiles)
+	//     +-- manifest.yaml
+
+	manifestyaml := `
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: foo-ns
+`
+
+	config, baseDir, cleanup := setup(t, []string{filepath.Join("config", "rawfiles")},
+		config{path: "config", fluxyaml: patchUpdatedEchoConfigFile},
+		config{path: filepath.Join("config", "rawfiles"), fluxyaml: "version: 1\nscanForFiles: {}\n"},
+	)
+	defer cleanup()
+
+	assert.NoError(t, ioutil.WriteFile(filepath.Join(baseDir, "config", "rawfiles", "manifest.yaml"), []byte(manifestyaml), 0600))
+
+	res, err := config.GetAllResourcesByID(context.Background())
+	assert.NoError(t, err)
+	assert.Contains(t, res, "default:namespace/foo-ns")
+}

pkg/manifests/configfile.go

@@ -11,8 +11,9 @@ import (
 	"path/filepath"
 	"time"
 
+	"github.com/ghodss/yaml"
 	"github.com/pkg/errors"
-	"gopkg.in/yaml.v2"
+	jsonschema "github.com/xeipuuv/gojsonschema"
 
 	"github.com/fluxcd/flux/pkg/image"
 	"github.com/fluxcd/flux/pkg/resource"
@@ -23,15 +24,79 @@ const (
 	CommandTimeout = time.Minute
 )
 
+// This is easier to read as YAML, trust me.
+const configSchemaYAML = `
+"$schema": http://json-schema.org/draft-07/schema#
+definitions:
+  command:
+    type: object
+    required: ['command']
+  version: { const: 1 }
+type: object
+oneOf:
+- required: ['version', 'commandUpdated']
+  properties:
+    version: { '$ref': '#/definitions/version' }
+    commandUpdated:
+      required: ['generators']
+      properties:
+        generators:
+          type: array
+          items: { '$ref': '#/definitions/command' }
+        updaters:
+          type: array
+          items:
+            type: object
+            properties:
+              containerImage: { '$ref': '#/definitions/command' }
+              policy: { '$ref': '#/definitions/command' }
+      additionalProperties: false
+- required: ['version', 'patchUpdated']
+  properties:
+    version: { '$ref': '#/definitions/version' }
+    patchUpdated:
+      required: ['generators', 'patchFile']
+      properties:
+        patchFile: { type: string }
+        generators:
+          type: array
+          items: { '$ref': '#/definitions/command' }
+      additionalProperties: false
+- required: ['version', 'scanForFiles']
+  properties:
+    version: { '$ref': '#/definitions/version' }
+    scanForFiles:
+      additionalProperties: false
+  additionalProperties: false
+`
+
+func mustCompileConfigSchema() *jsonschema.Schema {
+	j, err := yaml.YAMLToJSON([]byte(configSchemaYAML))
+	if err != nil {
+		panic(err)
+	}
+	sl := jsonschema.NewSchemaLoader()
+	sl.Validate = false
+	schema, err := sl.Compile(jsonschema.NewBytesLoader(j))
+	if err != nil {
+		panic(err)
+	}
+	return schema
+}
+
+var ConfigSchema = mustCompileConfigSchema()
+
 // ConfigFile holds the values necessary for generating and updating
 // manifests according to a `.flux.yaml` file. It does double duty as
 // the format for the file (to deserialise into), and the state
 // necessary for running commands.
 type ConfigFile struct {
-	Version int
+	Version int `json:"version"`
+
 	// Only one of the following should be set simultaneously
-	CommandUpdated *CommandUpdated `yaml:"commandUpdated"`
-	PatchUpdated   *PatchUpdated   `yaml:"patchUpdated"`
+	CommandUpdated *CommandUpdated `json:"commandUpdated,omitempty"`
+	PatchUpdated   *PatchUpdated   `json:"patchUpdated,omitempty"`
+	ScanForFiles   *ScanForFiles   `json:"scanForFiles,omitempty"`
 
 	// These are supplied, and can't be calculated from each other
 	configPath         string // the absolute path to the .flux.yaml
@@ -45,44 +110,87 @@ type ConfigFile struct {
 // CommandUpdated represents a config in which updates are done by
 // execing commands as given.
 type CommandUpdated struct {
-	Generators []Generator
-	Updaters   []Updater
+	Generators []Generator `json:"generators"`
+	Updaters   []Updater   `json:"updaters,omitempty"`
 }
 
 // Generator is an individual command for generating manifests.
 type Generator struct {
-	Command string
+	Command string `json:"command,omitempty"`
 }
 
 // Updater gives a means for updating image refs and a means for
 // updating policy in a manifest.
 type Updater struct {
-	ContainerImage ContainerImageUpdater `yaml:"containerImage"`
-	Policy         PolicyUpdater
+	ContainerImage ContainerImageUpdater `json:"containerImage,omitempty"`
+	Policy         PolicyUpdater         `json:"policy,omitempty"`
 }
 
 // ContainerImageUpdater is a command for updating the image used by a
 // container, in a manifest.
 type ContainerImageUpdater struct {
-	Command string
+	Command string `json:"command,omitempty"`
 }
 
 // PolicyUpdater is a command for updating a policy for a manifest.
 type PolicyUpdater struct {
-	Command string
+	Command string `json:"command,omitempty"`
 }
 
 // PatchUpdated represents a config in which updates are done by
 // maintaining a patch, which is calculating from, and applied to, the
 // generated manifests.
 type PatchUpdated struct {
-	Generators []Generator
-	PatchFile  string `yaml:"patchFile"`
+	Generators []Generator `json:"generators"`
+	PatchFile  string      `json:"patchFile,omitempty"`
+}
+
+// ScanForFiles represents a config in which the directory should be
+// treated as containing YAML files -- in other words, the normal mode
+// which looks for YAML files, and records changes by writing them
+// back to the original file.
+//
+// This can be used as a reset switch for a `--git-path`, if there's a
+// .flux.yaml higher in the directory structure.
+type ScanForFiles struct {
+}
+
+// IsScanForFiles returns true if the config file indicates that the
+// directory should be treated as containing YAML files (i.e., should
+// act as though there was no config file in operation). This can be
+// used to reset the directive given by a .flux.yaml higher in the
+// directory structure.
+func (cf *ConfigFile) IsScanForFiles() bool {
+	return cf.ScanForFiles != nil
+}
+
+func ParseConfigFile(fileBytes []byte, result *ConfigFile) error {
+	// The file contents are unmarshaled into a map so that we will
+	// see any extraneous fields. This is important, for example, for
+	// detecting when someone's made a commandUpdated config but
+	// mistakenly included a patchFile, thinking it will work.
+	var intermediate map[string]interface{}
+	if err := yaml.Unmarshal(fileBytes, &intermediate); err != nil {
+		return fmt.Errorf("cannot parse: %s", err)
+	}
+	validation, err := ConfigSchema.Validate(jsonschema.NewGoLoader(intermediate))
+	if err != nil {
+		return fmt.Errorf("cannot validate: %s", err)
+	}
+	if !validation.Valid() {
+		errs := ""
+		for _, e := range validation.Errors() {
+			errs = errs + "\n" + e.String()
+		}
+		return fmt.Errorf("config file is not valid: %s", errs)
+	}
+
+	return yaml.Unmarshal(fileBytes, result)
 }
 
 // NewConfigFile constructs a ConfigFile for the relative gitPath,
-// from the config file at the absolute path configPath, with the absolute
-// workingDir.
+// from the config file at the absolute path configPath, with the
+// absolute workingDir.
 func NewConfigFile(gitPath, configPath, workingDir string) (*ConfigFile, error) {
 	result := &ConfigFile{
 		configPath:         configPath,
@@ -100,20 +208,8 @@ func NewConfigFile(gitPath, configPath, workingDir string) (*ConfigFile, error)
 	if err != nil {
 		return nil, fmt.Errorf("cannot read: %s", err)
 	}
-	if err := yaml.Unmarshal(fileBytes, result); err != nil {
-		return nil, fmt.Errorf("cannot parse: %s", err)
-	}
 
-	switch {
-	case result.Version != 1:
-		return nil, errors.New("incorrect version, only version 1 is supported for now")
-	case (result.CommandUpdated != nil && result.PatchUpdated != nil) ||
-		(result.CommandUpdated == nil && result.PatchUpdated == nil):
-		return nil, errors.New("a single commandUpdated or patchUpdated entry must be defined")
-	case result.PatchUpdated != nil && result.PatchUpdated.PatchFile == "":
-		return nil, errors.New("patchUpdated's patchFile cannot be empty")
-	}
-	return result, nil
+	return result, ParseConfigFile(fileBytes, result)
 }
 
 // -- entry points for using a config file to generate or update manifests