Specifying a directory under files for a task copies all files of thi…

…s directory to the job container

README.md

@@ -211,18 +211,24 @@ metadata:
 
 ### File Handling
 
-Files can be added to your running tasks by specifying them in the `files` section of your tasks:
+Single files or all files in a directory can be added to your running tasks by specifying them in the `files` section of
+your tasks:
 
 ```yaml
 files:
   - locust/basic.py
   - locust/import.py
   - locust/locust.conf
+  - /helm
 ```
 
-This is done by using an `initcontainer` for the scheduled Kubernetes Job which prepares the `èmptyDir` volume mounted
-to the Kubernetes Job. Within the Job itself, the files will be available within the `keptn` folder. The naming of the
-files and the location will be preserved.
+The above settings will make the listed single files and all files in the `helm` directory and its subdirectories
+available in your task. Files can be listed with or without a starting `/`, it will be handled as absolute path for both
+cases.
+
+This setup is done by using an `initcontainer` for the scheduled Kubernetes Job which prepares the `emptyDir` volume
+mounted to the Kubernetes Job. Within the Job itself, the files will be available within the `keptn` folder. The naming
+of the files and the location will be preserved.
 
 When using these files in your container command, please make sure to reference them by prepending the `keptn` path.
 E.g.:

pkg/file/file.go

@@ -2,29 +2,24 @@ package file
 
 import (
 	"fmt"
+	"github.com/spf13/afero"
 	"keptn-sandbox/job-executor-service/pkg/config"
 	"keptn-sandbox/job-executor-service/pkg/keptn"
 	"log"
-	"net/url"
 	"path/filepath"
-
-	"github.com/spf13/afero"
 )
 
 // MountFiles requests all specified files of a task from the keptn configuration service and copies them to /keptn
 func MountFiles(actionName string, taskName string, fs afero.Fs, configService keptn.ConfigService) error {
 
-	// https://github.com/keptn/keptn/issues/2707
-	resource, err := configService.GetKeptnResource(url.QueryEscape("job/config.yaml"))
+	resource, err := configService.GetKeptnResource(fs, "job/config.yaml")
 	if err != nil {
-		log.Printf("Could not find config for job-executor-service")
-		return err
+		return fmt.Errorf("could not find config for job-executor-service: %v", err)
 	}
 
 	configuration, err := config.NewConfig(resource)
 	if err != nil {
-		log.Printf("Could not parse config: %s", err)
-		return err
+		return fmt.Errorf("could not parse config: %s", err)
 	}
 
 	found, action := configuration.FindActionByName(actionName)
@@ -37,44 +32,49 @@ func MountFiles(actionName string, taskName string, fs afero.Fs, configService k
 		return fmt.Errorf("no task found with name '%s'", taskName)
 	}
 
-	for _, fileName := range task.Files {
+	for _, resourcePath := range task.Files {
+		fileNotFound := true
 
-		resource, err = configService.GetKeptnResource(url.QueryEscape(fileName))
+		allServiceResources, err := configService.GetAllKeptnResources(fs, resourcePath)
 		if err != nil {
-			log.Printf("Could not find file %s for task %s", fileName, taskName)
-			return err
+			return fmt.Errorf("could not retrieve resources for task '%v': %v", taskName, err)
 		}
 
-		// Our mount starts with /keptn
-		dir := filepath.Join("/keptn", filepath.Dir(fileName))
-		fullFilePath := filepath.Join("/keptn", fileName)
+		for resourceURI, resourceContent := range allServiceResources {
 
-		err := fs.MkdirAll(dir, 0700)
-		if err != nil {
-			log.Printf("Could not create directory %s for file %s", dir, fileName)
-			return err
-		}
+			// Our mount starts with /keptn
+			dir := filepath.Join("/keptn", filepath.Dir(resourceURI))
+			fullFilePath := filepath.Join("/keptn", resourceURI)
 
-		file, err := fs.Create(fullFilePath)
-		if err != nil {
-			log.Printf("Could not create file %s", fileName)
-			return err
-		}
+			err := fs.MkdirAll(dir, 0700)
+			if err != nil {
+				return fmt.Errorf("could not create directory %s for file %s: %v", dir, resourceURI, err)
+			}
 
-		_, err = file.Write(resource)
-		defer func() {
-			err = file.Close()
+			file, err := fs.Create(fullFilePath)
 			if err != nil {
-				log.Printf("Could not close file %s", file.Name())
+				return fmt.Errorf("could not create file %s: %v", resourceURI, err)
 			}
-		}()
 
-		if err != nil {
-			log.Printf("Could not write to file %s", fileName)
-			return err
+			_, err = file.Write(resourceContent)
+			defer func() {
+				err = file.Close()
+				if err != nil {
+					log.Printf("could not close file %s: %v", file.Name(), err)
+				}
+			}()
+
+			if err != nil {
+				return fmt.Errorf("could not write to file %s: %v", fullFilePath, err)
+			}
+
+			log.Printf("successfully moved file %s to %s", resourceURI, fullFilePath)
+			fileNotFound = false
 		}
 
-		log.Printf("Successfully moved file %s to %s", fileName, fullFilePath)
+		if fileNotFound {
+			return fmt.Errorf("could not find file or directory %s for task %s", resourcePath, taskName)
+		}
 	}
 
 	return nil

pkg/file/file_test.go

@@ -20,17 +20,20 @@ actions:
           match: "health"
     tasks:
       - name: "task"
-        files: 
-          - locust/basic.py
+        files:
+          - /helm/values.yaml
+          - locust
         image: "locustio/locust"
-        cmd: "locust -f /keptn/locust/locustfile.py"
+        cmd: "locust -f /keptn/locust/basic.py"
 `
 
 const pythonFile = `
 // This is a python file
 `
 
-const escapedSlash = "%2F"
+const yamlFile = `
+// This is a yaml file
+`
 
 func CreateKeptnConfigServiceMock(t *testing.T) *keptn.MockConfigService {
 
@@ -45,8 +48,9 @@ func TestMountFiles(t *testing.T) {
 	fs := afero.NewMemMapFs()
 	configServiceMock := CreateKeptnConfigServiceMock(t)
 
-	configServiceMock.EXPECT().GetKeptnResource("job"+escapedSlash+"config.yaml").Times(1).Return([]byte(simpleConfig), nil)
-	configServiceMock.EXPECT().GetKeptnResource("locust"+escapedSlash+"basic.py").Times(1).Return([]byte(pythonFile), nil)
+	configServiceMock.EXPECT().GetKeptnResource(fs, "job/config.yaml").Times(1).Return([]byte(simpleConfig), nil)
+	configServiceMock.EXPECT().GetAllKeptnResources(fs, "locust").Times(1).Return(map[string][]byte{"locust/basic.py": []byte(pythonFile), "locust/functional.py": []byte(pythonFile)}, nil)
+	configServiceMock.EXPECT().GetAllKeptnResources(fs, "/helm/values.yaml").Times(1).Return(map[string][]byte{"helm/values.yaml": []byte(yamlFile)}, nil)
 
 	err := MountFiles("action", "task", fs, configServiceMock)
 	assert.NilError(t, err)
@@ -58,14 +62,22 @@ func TestMountFiles(t *testing.T) {
 	file, err := afero.ReadFile(fs, "/keptn/locust/basic.py")
 	assert.NilError(t, err)
 	assert.Equal(t, pythonFile, string(file))
+
+	exists, err = afero.Exists(fs, "/keptn/helm/values.yaml")
+	assert.NilError(t, err)
+	assert.Check(t, exists)
+
+	file, err = afero.ReadFile(fs, "/keptn/helm/values.yaml")
+	assert.NilError(t, err)
+	assert.Equal(t, yamlFile, string(file))
 }
 
 func TestMountFilesConfigFileNotFound(t *testing.T) {
 
 	fs := afero.NewMemMapFs()
 	configServiceMock := CreateKeptnConfigServiceMock(t)
 
-	configServiceMock.EXPECT().GetKeptnResource("job"+escapedSlash+"config.yaml").Times(1).Return(nil, errors.New("not found"))
+	configServiceMock.EXPECT().GetKeptnResource(fs, "job/config.yaml").Times(1).Return(nil, errors.New("not found"))
 
 	err := MountFiles("action", "task", fs, configServiceMock)
 	assert.ErrorContains(t, err, "not found")
@@ -76,7 +88,7 @@ func TestMountFilesConfigFileNotValid(t *testing.T) {
 	fs := afero.NewMemMapFs()
 	configServiceMock := CreateKeptnConfigServiceMock(t)
 
-	configServiceMock.EXPECT().GetKeptnResource("job"+escapedSlash+"config.yaml").Times(1).Return([]byte(pythonFile), nil)
+	configServiceMock.EXPECT().GetKeptnResource(fs, "job/config.yaml").Times(1).Return([]byte(pythonFile), nil)
 
 	err := MountFiles("action", "task", fs, configServiceMock)
 	assert.ErrorContains(t, err, "cannot unmarshal")
@@ -87,7 +99,7 @@ func TestMountFilesNoActionMatch(t *testing.T) {
 	fs := afero.NewMemMapFs()
 	configServiceMock := CreateKeptnConfigServiceMock(t)
 
-	configServiceMock.EXPECT().GetKeptnResource("job"+escapedSlash+"config.yaml").Times(1).Return([]byte(simpleConfig), nil)
+	configServiceMock.EXPECT().GetKeptnResource(fs, "job/config.yaml").Times(1).Return([]byte(simpleConfig), nil)
 
 	err := MountFiles("actionNotMatching", "task", fs, configServiceMock)
 	assert.ErrorContains(t, err, "no action found with name 'actionNotMatching'")
@@ -98,7 +110,7 @@ func TestMountFilesNoTaskMatch(t *testing.T) {
 	fs := afero.NewMemMapFs()
 	configServiceMock := CreateKeptnConfigServiceMock(t)
 
-	configServiceMock.EXPECT().GetKeptnResource("job"+escapedSlash+"config.yaml").Times(1).Return([]byte(simpleConfig), nil)
+	configServiceMock.EXPECT().GetKeptnResource(fs, "job/config.yaml").Times(1).Return([]byte(simpleConfig), nil)
 
 	err := MountFiles("action", "taskNotMatching", fs, configServiceMock)
 	assert.ErrorContains(t, err, "no task found with name 'taskNotMatching'")
@@ -109,9 +121,33 @@ func TestMountFilesFileNotFound(t *testing.T) {
 	fs := afero.NewMemMapFs()
 	configServiceMock := CreateKeptnConfigServiceMock(t)
 
-	configServiceMock.EXPECT().GetKeptnResource("job"+escapedSlash+"config.yaml").Times(1).Return([]byte(simpleConfig), nil)
-	configServiceMock.EXPECT().GetKeptnResource("locust"+escapedSlash+"basic.py").Times(1).Return(nil, errors.New("not found"))
+	configServiceMock.EXPECT().GetKeptnResource(fs, "job/config.yaml").Times(1).Return([]byte(simpleConfig), nil)
+	configServiceMock.EXPECT().GetAllKeptnResources(fs, "/helm/values.yaml").Times(1).Return(nil, errors.New("not found"))
 
 	err := MountFiles("action", "task", fs, configServiceMock)
 	assert.ErrorContains(t, err, "not found")
 }
+
+func TestWithLocalFileSystem(t *testing.T) {
+
+	fs := afero.NewMemMapFs()
+	configService := keptn.NewConfigService(true, "", "", "", nil)
+	err := afero.WriteFile(fs, "job/config.yaml", []byte(simpleConfig), 0644)
+	assert.NilError(t, err)
+	err = afero.WriteFile(fs, "/helm/values.yaml", []byte("here be awesome configuration"), 0644)
+	assert.NilError(t, err)
+	err = afero.WriteFile(fs, "locust/basic.py", []byte("here be awesome test code"), 0644)
+	assert.NilError(t, err)
+	err = afero.WriteFile(fs, "locust/functional.py", []byte("here be more awesome test code"), 0644)
+	assert.NilError(t, err)
+
+	err = MountFiles("action", "task", fs, configService)
+	assert.NilError(t, err)
+
+	_, err = fs.Stat("/keptn/helm/values.yaml")
+	assert.NilError(t, err)
+	_, err = fs.Stat("/keptn/locust/basic.py")
+	assert.NilError(t, err)
+	_, err = fs.Stat("/keptn/locust/functional.py")
+	assert.NilError(t, err)
+}

pkg/keptn/config_service.go

@@ -3,14 +3,18 @@ package keptn
 import (
 	"fmt"
 	api "github.com/keptn/go-utils/pkg/api/utils"
+	"github.com/spf13/afero"
+	"net/url"
 	"os"
+	"strings"
 )
 
 //go:generate mockgen -source=config_service.go -destination=config_service_mock.go -package=keptn ConfigService
 
 // ConfigService provides methods to work with the keptn configuration service
 type ConfigService interface {
-	GetKeptnResource(resource string) ([]byte, error)
+	GetKeptnResource(fs afero.Fs, resource string) ([]byte, error)
+	GetAllKeptnResources(fs afero.Fs, resource string) (map[string][]byte, error)
 }
 
 type configServiceImpl struct {
@@ -33,15 +37,16 @@ func NewConfigService(useLocalFileSystem bool, project string, stage string, ser
 }
 
 // GetKeptnResource returns a resource from the configuration repo based on the incoming cloud events project, service and stage
-func (k *configServiceImpl) GetKeptnResource(resource string) ([]byte, error) {
+func (k *configServiceImpl) GetKeptnResource(fs afero.Fs, resource string) ([]byte, error) {
 
 	// if we run in a runlocal mode we are just getting the file from the local disk
 	if k.useLocalFileSystem {
-		return k.getKeptnResourceFromLocal(resource)
+		return k.getKeptnResourceFromLocal(fs, resource)
 	}
 
 	// get it from KeptnBase
-	requestedResource, err := k.resourceHandler.GetServiceResource(k.project, k.stage, k.service, resource)
+	// https://github.com/keptn/keptn/issues/2707
+	requestedResource, err := k.resourceHandler.GetServiceResource(k.project, k.stage, k.service, url.QueryEscape(resource))
 
 	// return Nil in case resource couldn't be retrieved
 	if err != nil || requestedResource.ResourceContent == "" {
@@ -51,13 +56,79 @@ func (k *configServiceImpl) GetKeptnResource(resource string) ([]byte, error) {
 	return []byte(requestedResource.ResourceContent), nil
 }
 
+// GetAllKeptnResources returns a map of keptn resources (key=URI, value=content) from the configuration repo with
+// prefix 'resource' (matched with and without leading '/')
+func (k *configServiceImpl) GetAllKeptnResources(fs afero.Fs, resource string) (map[string][]byte, error) {
+
+	// if we run in a runlocal mode we are just getting the file from the local disk
+	if k.useLocalFileSystem {
+		return k.getKeptnResourcesFromLocal(fs, resource)
+	}
+
+	// get it from KeptnBase
+	requestedResources, err := k.resourceHandler.GetAllServiceResources(k.project, k.stage, k.service)
+	if err != nil {
+		return nil, fmt.Errorf("resources not found: %s", err)
+	}
+
+	keptnResources := make(map[string][]byte)
+	for _, serviceResource := range requestedResources {
+		// match against with and without starting slash
+		resourceURIWithoutSlash := strings.Replace(*serviceResource.ResourceURI, "/", "", 1)
+		if strings.HasPrefix(*serviceResource.ResourceURI, resource) || strings.HasPrefix(resourceURIWithoutSlash, resource) {
+			keptnResourceContent, err := k.GetKeptnResource(fs, *serviceResource.ResourceURI)
+			if err != nil {
+				return nil, fmt.Errorf("could not find file %s", *serviceResource.ResourceURI)
+			}
+			keptnResources[*serviceResource.ResourceURI] = keptnResourceContent
+		}
+	}
+
+	return keptnResources, nil
+}
+
 /**
  * Retrieves a resource (=file) from the local file system. Basically checks if the file is available and if so returns it
  */
-func (k *configServiceImpl) getKeptnResourceFromLocal(resource string) ([]byte, error) {
-	_, err := os.Stat(resource)
-	if err == nil {
-		return []byte(resource), nil
+func (k *configServiceImpl) getKeptnResourceFromLocal(fs afero.Fs, resource string) ([]byte, error) {
+	_, err := fs.Stat(resource)
+	if err != nil {
+		return nil, err
+	}
+
+	content, err := afero.ReadFile(fs, resource)
+	if err != nil {
+		return nil, err
 	}
-	return nil, err
+	return content, nil
+}
+
+/**
+* Retrieves a resource (=file or all files of a directory) from the local file system. Basically checks if the file/directory
+  is available and if so returns it or its files
+*/
+func (k *configServiceImpl) getKeptnResourcesFromLocal(fs afero.Fs, resource string) (map[string][]byte, error) {
+	resources := make(map[string][]byte)
+	err := afero.Walk(fs, resource, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if info.IsDir() {
+			return nil
+		}
+
+		content, err := k.getKeptnResourceFromLocal(fs, path)
+		if err != nil {
+			return err
+		}
+		resources[path] = content
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return resources, nil
 }