Allow import of all files in a directory to the job container (#29)

* Specifying a directory under files for a task copies all files of this directory to the job container

README.md

@@ -211,18 +211,24 @@ metadata:
 
 ### File Handling
 
-Files can be added to your running tasks by specifying them in the `files` section of your tasks:
+Single files or all files in a directory can be added to your running tasks by specifying them in the `files` section of
+your tasks:
 
 ```yaml
 files:
   - locust/basic.py
   - locust/import.py
   - locust/locust.conf
+  - /helm
 ```
 
-This is done by using an `initcontainer` for the scheduled Kubernetes Job which prepares the `èmptyDir` volume mounted
-to the Kubernetes Job. Within the Job itself, the files will be available within the `keptn` folder. The naming of the
-files and the location will be preserved.
+The above settings will make the listed single files and all files in the `helm` directory and its subdirectories
+available in your task. Files can be listed with or without a starting `/`, it will be handled as absolute path for both
+cases.
+
+This setup is done by using an `initcontainer` for the scheduled Kubernetes Job which prepares the `emptyDir` volume
+mounted to the Kubernetes Job. Within the Job itself, the files will be available within the `keptn` folder. The naming
+of the files and the location will be preserved.
 
 When using these files in your container command, please make sure to reference them by prepending the `keptn` path.
 E.g.:

pkg/file/file.go

@@ -2,29 +2,24 @@ package file
 
 import (
 	"fmt"
+	"github.com/spf13/afero"
 	"keptn-sandbox/job-executor-service/pkg/config"
 	"keptn-sandbox/job-executor-service/pkg/keptn"
 	"log"
-	"net/url"
 	"path/filepath"
-
-	"github.com/spf13/afero"
 )
 
 // MountFiles requests all specified files of a task from the keptn configuration service and copies them to /keptn
 func MountFiles(actionName string, taskName string, fs afero.Fs, configService keptn.ConfigService) error {
 
-	// https://github.com/keptn/keptn/issues/2707
-	resource, err := configService.GetKeptnResource(url.QueryEscape("job/config.yaml"))
+	resource, err := configService.GetKeptnResource(fs, "job/config.yaml")
 	if err != nil {
-		log.Printf("Could not find config for job-executor-service")
-		return err
+		return fmt.Errorf("could not find config for job-executor-service: %v", err)
 	}
 
 	configuration, err := config.NewConfig(resource)
 	if err != nil {
-		log.Printf("Could not parse config: %s", err)
-		return err
+		return fmt.Errorf("could not parse config: %s", err)
 	}
 
 	found, action := configuration.FindActionByName(actionName)
@@ -37,44 +32,49 @@ func MountFiles(actionName string, taskName string, fs afero.Fs, configService k
 		return fmt.Errorf("no task found with name '%s'", taskName)
 	}
 
-	for _, fileName := range task.Files {
+	for _, resourcePath := range task.Files {
+		fileNotFound := true
 
-		resource, err = configService.GetKeptnResource(url.QueryEscape(fileName))
+		allServiceResources, err := configService.GetAllKeptnResources(fs, resourcePath)
 		if err != nil {
-			log.Printf("Could not find file %s for task %s", fileName, taskName)
-			return err
+			return fmt.Errorf("could not retrieve resources for task '%v': %v", taskName, err)
 		}
 
-		// Our mount starts with /keptn
-		dir := filepath.Join("/keptn", filepath.Dir(fileName))
-		fullFilePath := filepath.Join("/keptn", fileName)
+		for resourceURI, resourceContent := range allServiceResources {
 
-		err := fs.MkdirAll(dir, 0700)
-		if err != nil {
-			log.Printf("Could not create directory %s for file %s", dir, fileName)
-			return err
-		}
+			// Our mount starts with /keptn
+			dir := filepath.Join("/keptn", filepath.Dir(resourceURI))
+			fullFilePath := filepath.Join("/keptn", resourceURI)
 
-		file, err := fs.Create(fullFilePath)
-		if err != nil {
-			log.Printf("Could not create file %s", fileName)
-			return err
-		}
+			err := fs.MkdirAll(dir, 0700)
+			if err != nil {
+				return fmt.Errorf("could not create directory %s for file %s: %v", dir, resourceURI, err)
+			}
 
-		_, err = file.Write(resource)
-		defer func() {
-			err = file.Close()
+			file, err := fs.Create(fullFilePath)
 			if err != nil {
-				log.Printf("Could not close file %s", file.Name())
+				return fmt.Errorf("could not create file %s: %v", resourceURI, err)
 			}
-		}()
 
-		if err != nil {
-			log.Printf("Could not write to file %s", fileName)
-			return err
+			_, err = file.Write(resourceContent)
+			defer func() {
+				err = file.Close()
+				if err != nil {
+					log.Printf("could not close file %s: %v", file.Name(), err)
+				}
+			}()
+
+			if err != nil {
+				return fmt.Errorf("could not write to file %s: %v", fullFilePath, err)
+			}
+
+			log.Printf("successfully moved file %s to %s", resourceURI, fullFilePath)
+			fileNotFound = false
 		}
 
-		log.Printf("Successfully moved file %s to %s", fileName, fullFilePath)
+		if fileNotFound {
+			return fmt.Errorf("could not find file or directory %s for task %s", resourcePath, taskName)
+		}
 	}
 
 	return nil

pkg/file/file_test.go

@@ -6,6 +6,7 @@ import (
 	"github.com/spf13/afero"
 	"gotest.tools/assert"
 	"keptn-sandbox/job-executor-service/pkg/keptn"
+	keptnfake "keptn-sandbox/job-executor-service/pkg/keptn/fake"
 	"testing"
 )
 
@@ -20,33 +21,37 @@ actions:
           match: "health"
     tasks:
       - name: "task"
-        files: 
-          - locust/basic.py
+        files:
+          - /helm/values.yaml
+          - locust
         image: "locustio/locust"
-        cmd: "locust -f /keptn/locust/locustfile.py"
+        cmd: "locust -f /keptn/locust/basic.py"
 `
 
 const pythonFile = `
 // This is a python file
 `
 
-const escapedSlash = "%2F"
+const yamlFile = `
+// This is a yaml file
+`
 
-func CreateKeptnConfigServiceMock(t *testing.T) *keptn.MockConfigService {
+func CreateKeptnConfigServiceMock(t *testing.T) *keptnfake.MockConfigService {
 
 	mockCtrl := gomock.NewController(t)
 	defer mockCtrl.Finish()
 
-	return keptn.NewMockConfigService(mockCtrl)
+	return keptnfake.NewMockConfigService(mockCtrl)
 }
 
 func TestMountFiles(t *testing.T) {
 
 	fs := afero.NewMemMapFs()
 	configServiceMock := CreateKeptnConfigServiceMock(t)
 
-	configServiceMock.EXPECT().GetKeptnResource("job"+escapedSlash+"config.yaml").Times(1).Return([]byte(simpleConfig), nil)
-	configServiceMock.EXPECT().GetKeptnResource("locust"+escapedSlash+"basic.py").Times(1).Return([]byte(pythonFile), nil)
+	configServiceMock.EXPECT().GetKeptnResource(fs, "job/config.yaml").Times(1).Return([]byte(simpleConfig), nil)
+	configServiceMock.EXPECT().GetAllKeptnResources(fs, "locust").Times(1).Return(map[string][]byte{"locust/basic.py": []byte(pythonFile), "locust/functional.py": []byte(pythonFile)}, nil)
+	configServiceMock.EXPECT().GetAllKeptnResources(fs, "/helm/values.yaml").Times(1).Return(map[string][]byte{"helm/values.yaml": []byte(yamlFile)}, nil)
 
 	err := MountFiles("action", "task", fs, configServiceMock)
 	assert.NilError(t, err)
@@ -58,14 +63,22 @@ func TestMountFiles(t *testing.T) {
 	file, err := afero.ReadFile(fs, "/keptn/locust/basic.py")
 	assert.NilError(t, err)
 	assert.Equal(t, pythonFile, string(file))
+
+	exists, err = afero.Exists(fs, "/keptn/helm/values.yaml")
+	assert.NilError(t, err)
+	assert.Check(t, exists)
+
+	file, err = afero.ReadFile(fs, "/keptn/helm/values.yaml")
+	assert.NilError(t, err)
+	assert.Equal(t, yamlFile, string(file))
 }
 
 func TestMountFilesConfigFileNotFound(t *testing.T) {
 
 	fs := afero.NewMemMapFs()
 	configServiceMock := CreateKeptnConfigServiceMock(t)
 
-	configServiceMock.EXPECT().GetKeptnResource("job"+escapedSlash+"config.yaml").Times(1).Return(nil, errors.New("not found"))
+	configServiceMock.EXPECT().GetKeptnResource(fs, "job/config.yaml").Times(1).Return(nil, errors.New("not found"))
 
 	err := MountFiles("action", "task", fs, configServiceMock)
 	assert.ErrorContains(t, err, "not found")
@@ -76,7 +89,7 @@ func TestMountFilesConfigFileNotValid(t *testing.T) {
 	fs := afero.NewMemMapFs()
 	configServiceMock := CreateKeptnConfigServiceMock(t)
 
-	configServiceMock.EXPECT().GetKeptnResource("job"+escapedSlash+"config.yaml").Times(1).Return([]byte(pythonFile), nil)
+	configServiceMock.EXPECT().GetKeptnResource(fs, "job/config.yaml").Times(1).Return([]byte(pythonFile), nil)
 
 	err := MountFiles("action", "task", fs, configServiceMock)
 	assert.ErrorContains(t, err, "cannot unmarshal")
@@ -87,7 +100,7 @@ func TestMountFilesNoActionMatch(t *testing.T) {
 	fs := afero.NewMemMapFs()
 	configServiceMock := CreateKeptnConfigServiceMock(t)
 
-	configServiceMock.EXPECT().GetKeptnResource("job"+escapedSlash+"config.yaml").Times(1).Return([]byte(simpleConfig), nil)
+	configServiceMock.EXPECT().GetKeptnResource(fs, "job/config.yaml").Times(1).Return([]byte(simpleConfig), nil)
 
 	err := MountFiles("actionNotMatching", "task", fs, configServiceMock)
 	assert.ErrorContains(t, err, "no action found with name 'actionNotMatching'")
@@ -98,7 +111,7 @@ func TestMountFilesNoTaskMatch(t *testing.T) {
 	fs := afero.NewMemMapFs()
 	configServiceMock := CreateKeptnConfigServiceMock(t)
 
-	configServiceMock.EXPECT().GetKeptnResource("job"+escapedSlash+"config.yaml").Times(1).Return([]byte(simpleConfig), nil)
+	configServiceMock.EXPECT().GetKeptnResource(fs, "job/config.yaml").Times(1).Return([]byte(simpleConfig), nil)
 
 	err := MountFiles("action", "taskNotMatching", fs, configServiceMock)
 	assert.ErrorContains(t, err, "no task found with name 'taskNotMatching'")
@@ -109,9 +122,33 @@ func TestMountFilesFileNotFound(t *testing.T) {
 	fs := afero.NewMemMapFs()
 	configServiceMock := CreateKeptnConfigServiceMock(t)
 
-	configServiceMock.EXPECT().GetKeptnResource("job"+escapedSlash+"config.yaml").Times(1).Return([]byte(simpleConfig), nil)
-	configServiceMock.EXPECT().GetKeptnResource("locust"+escapedSlash+"basic.py").Times(1).Return(nil, errors.New("not found"))
+	configServiceMock.EXPECT().GetKeptnResource(fs, "job/config.yaml").Times(1).Return([]byte(simpleConfig), nil)
+	configServiceMock.EXPECT().GetAllKeptnResources(fs, "/helm/values.yaml").Times(1).Return(nil, errors.New("not found"))
 
 	err := MountFiles("action", "task", fs, configServiceMock)
 	assert.ErrorContains(t, err, "not found")
 }
+
+func TestMountFilesWithLocalFileSystem(t *testing.T) {
+
+	fs := afero.NewMemMapFs()
+	configService := keptn.NewConfigService(true, "", "", "", nil)
+	err := afero.WriteFile(fs, "job/config.yaml", []byte(simpleConfig), 0644)
+	assert.NilError(t, err)
+	err = afero.WriteFile(fs, "/helm/values.yaml", []byte("here be awesome configuration"), 0644)
+	assert.NilError(t, err)
+	err = afero.WriteFile(fs, "locust/basic.py", []byte("here be awesome test code"), 0644)
+	assert.NilError(t, err)
+	err = afero.WriteFile(fs, "locust/functional.py", []byte("here be more awesome test code"), 0644)
+	assert.NilError(t, err)
+
+	err = MountFiles("action", "task", fs, configService)
+	assert.NilError(t, err)
+
+	_, err = fs.Stat("/keptn/helm/values.yaml")
+	assert.NilError(t, err)
+	_, err = fs.Stat("/keptn/locust/basic.py")
+	assert.NilError(t, err)
+	_, err = fs.Stat("/keptn/locust/functional.py")
+	assert.NilError(t, err)
+}