feat: Saving Graphql mutation object to Persistent Volume (#3460)

* Initial version of graphql middleware

* Saving file to Persistent Volume

* Minor fixes

* change the way of saving Gardener Shoot to PV

* delete unnecessary files

* fix naming

* Added storing graphQL query

* Compilation error

* Compilation error

* Compilation error

* Minor refactor

* Unit test fix

* Functions for dumping spec and graphql moved to testkit package

* Fixes in GraphQL dump

* Fixes in GraphQL dump

* Added logging

* Added logs for debugging

* Fixed issue with dumping GraphQL

* Fixed issue with dumping GraphQL

* Introduced TestDataWriter

* Compilation fix

* Minor refactoring

* Names of the dumped files aligned with KIM

---------

Co-authored-by: Arkadiusz Galwas <arkadiusz.galwas@sap.com>

components/provisioner/cmd/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"github.com/kyma-project/control-plane/components/provisioner/internal/util/testkit"
 	"net/http"
 	"sync"
 	"time"
@@ -157,14 +158,16 @@ func main() {
 
 	k8sClientProvider := k8s.NewK8sClientProvider()
 
+	testDataWriter := testkit.NewTestDataWriter(gardenerNamespace, "/testdata/provisioner", cfg.Gardener.EnableDumpShootSpec)
+
 	adminKubeconfigRequest := gardenerClient.SubResource("adminkubeconfig")
 	kubeconfigProvider := gardener.NewKubeconfigProvider(shootClient, adminKubeconfigRequest, secretsInterface)
 
 	provisioningQueue := queue.CreateProvisioningQueue(cfg.ProvisioningTimeout, dbsFactory, shootClient, cfg.OperatorRoleBinding, k8sClientProvider, kubeconfigProvider)
 	shootUpgradeQueue := queue.CreateShootUpgradeQueue(cfg.ProvisioningTimeout, dbsFactory, shootClient, cfg.OperatorRoleBinding, k8sClientProvider, kubeconfigProvider)
 	deprovisioningQueue := queue.CreateDeprovisioningQueue(cfg.DeprovisioningTimeout, dbsFactory, shootClient)
 
-	provisioner := gardener.NewProvisioner(gardenerNamespace, shootClient, dbsFactory, cfg.Gardener.AuditLogsPolicyConfigMap, cfg.Gardener.MaintenanceWindowConfigPath, cfg.Gardener.EnableDumpShootSpec)
+	provisioner := gardener.NewProvisioner(gardenerNamespace, shootClient, dbsFactory, cfg.Gardener.AuditLogsPolicyConfigMap, cfg.Gardener.MaintenanceWindowConfigPath, testDataWriter)
 	shootController, err := newShootController(gardenerNamespace, gardenerClusterConfig, dbsFactory, cfg.Gardener.AuditLogsTenantConfigPath)
 	exitOnError(err, "Failed to create Shoot controller.")
 	go func() {
@@ -188,7 +191,7 @@ func main() {
 
 	tenantUpdater := api.NewTenantUpdater(dbsFactory.NewReadWriteSession())
 	validator := api.NewValidator()
-	resolver := api.NewResolver(provisioningSVC, validator, tenantUpdater)
+	resolver := api.NewResolver(provisioningSVC, validator, tenantUpdater, testDataWriter)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()

components/provisioner/internal/api/resolver.go

@@ -3,7 +3,6 @@ package api
 import (
 	"context"
 	"fmt"
-
 	"github.com/kyma-project/control-plane/components/provisioner/internal/api/middlewares"
 	"github.com/pkg/errors"
 
@@ -14,31 +13,40 @@ import (
 )
 
 type Resolver struct {
-	provisioning  provisioning.Service
-	validator     Validator
-	tenantUpdater TenantUpdater
+	provisioning   provisioning.Service
+	validator      Validator
+	tenantUpdater  TenantUpdater
+	testDataWriter InputDataWriter
+}
+
+type InputDataWriter interface {
+	PersistGraphQL(mutation gqlschema.ProvisionRuntimeInput) (string, error)
+	Enabled() bool
 }
 
 func (r *Resolver) Mutation() gqlschema.MutationResolver {
 	return &Resolver{
-		provisioning:  r.provisioning,
-		validator:     r.validator,
-		tenantUpdater: r.tenantUpdater,
+		provisioning:   r.provisioning,
+		validator:      r.validator,
+		tenantUpdater:  r.tenantUpdater,
+		testDataWriter: r.testDataWriter,
 	}
 }
 func (r *Resolver) Query() gqlschema.QueryResolver {
 	return &Resolver{
-		provisioning:  r.provisioning,
-		validator:     r.validator,
-		tenantUpdater: r.tenantUpdater,
+		provisioning:   r.provisioning,
+		validator:      r.validator,
+		tenantUpdater:  r.tenantUpdater,
+		testDataWriter: r.testDataWriter,
 	}
 }
 
-func NewResolver(provisioningService provisioning.Service, validator Validator, tenantUpdater TenantUpdater) *Resolver {
+func NewResolver(provisioningService provisioning.Service, validator Validator, tenantUpdater TenantUpdater, testDataWriter InputDataWriter) *Resolver {
 	return &Resolver{
-		provisioning:  provisioningService,
-		validator:     validator,
-		tenantUpdater: tenantUpdater,
+		provisioning:   provisioningService,
+		validator:      validator,
+		tenantUpdater:  tenantUpdater,
+		testDataWriter: testDataWriter,
 	}
 }
 
@@ -59,6 +67,17 @@ func (r *Resolver) ProvisionRuntime(ctx context.Context, config gqlschema.Provis
 
 	log.Infof("Requested provisioning of Runtime %s.", config.RuntimeInput.Name)
 
+	if r.testDataWriter.Enabled() {
+		log.Infof("Saving GraphQL query for Runtime %s", config.RuntimeInput.Name)
+		path, err := r.testDataWriter.PersistGraphQL(config)
+
+		if err == nil {
+			log.Infof("GraphQL query dumped to %s", path)
+		} else {
+			log.Errorf("Failed to dump GraphQL mutation for Runtime %s: %s", config.RuntimeInput.Name, err)
+		}
+	}
+
 	operationStatus, err := r.provisioning.ProvisionRuntime(config, tenant, subAccount)
 	if err != nil {
 		log.Errorf("Failed to provision Runtime %s: %s", config.RuntimeInput.Name, err)

components/provisioner/internal/api/resolver_integration_with_gardener_test.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"github.com/kyma-project/control-plane/components/provisioner/internal/util/testkit"
+	"os"
 	"path/filepath"
 	"strings"
 	"testing"
@@ -180,7 +182,10 @@ func TestProvisioning_ProvisionRuntimeWithDatabase(t *testing.T) {
 			//uuidGeneratorMock.On("New").Return(config.upgradeID).Once()
 			//uuidGeneratorMock.On("New").Return(config.deprovisioningID).Once()
 
-			provisioner := gardener.NewProvisioner(namespace, shootInterface, dbsFactory, auditLogPolicyCMName, maintenanceWindowConfigPath, false)
+			tmpDir, err := os.MkdirTemp("", "")
+			require.NoError(t, err)
+
+			provisioner := gardener.NewProvisioner(namespace, shootInterface, dbsFactory, auditLogPolicyCMName, maintenanceWindowConfigPath, testkit.NewTestDataWriter("kyma-dev", tmpDir, true))
 
 			inputConverter := provisioning.NewInputConverter(uuidGeneratorMock, "Project", defaultEnableKubernetesVersionAutoUpdate, defaultEnableMachineImageVersionAutoUpdate, defaultEnableIMDSv2)
 			graphQLConverter := provisioning.NewGraphQLConverter()
@@ -201,7 +206,7 @@ func TestProvisioning_ProvisionRuntimeWithDatabase(t *testing.T) {
 
 			tenantUpdater := api.NewTenantUpdater(dbsFactory.NewReadWriteSession())
 
-			resolver := api.NewResolver(provisioningService, validator, tenantUpdater)
+			resolver := api.NewResolver(provisioningService, validator, tenantUpdater, testkit.NewTestDataWriter("kyma-dev", tmpDir, true))
 
 			fullConfig := gqlschema.ProvisionRuntimeInput{RuntimeInput: &runtimeInput, ClusterConfig: &clusterConfig}
 

components/provisioner/internal/api/resolver_test.go

@@ -2,6 +2,7 @@ package api_test
 
 import (
 	"context"
+	"github.com/kyma-project/control-plane/components/provisioner/internal/util/testkit"
 	"testing"
 
 	"github.com/kyma-project/control-plane/components/provisioner/internal/apperrors"
@@ -61,7 +62,7 @@ func TestResolver_ProvisionRuntime(t *testing.T) {
 		provisioningService := &mocks.Service{}
 		validator := &validatorMocks.Validator{}
 		tenantUpdater := &validatorMocks.TenantUpdater{}
-		resolver := api.NewResolver(provisioningService, validator, tenantUpdater)
+		resolver := api.NewResolver(provisioningService, validator, tenantUpdater, &testkit.TestDataWriter{})
 
 		tenantUpdater.On("GetTenant", ctx).Return(tenant, nil)
 
@@ -112,7 +113,7 @@ func TestResolver_ProvisionRuntime(t *testing.T) {
 		provisioningService := &mocks.Service{}
 		validator := &validatorMocks.Validator{}
 		tenantUpdater := &validatorMocks.TenantUpdater{}
-		provisioner := api.NewResolver(provisioningService, validator, tenantUpdater)
+		provisioner := api.NewResolver(provisioningService, validator, tenantUpdater, &testkit.TestDataWriter{})
 
 		kymaConfig := &gqlschema.KymaConfigInput{
 			Version: "1.5",
@@ -136,7 +137,7 @@ func TestResolver_ProvisionRuntime(t *testing.T) {
 		provisioningService := &mocks.Service{}
 		validator := &validatorMocks.Validator{}
 		tenantUpdater := &validatorMocks.TenantUpdater{}
-		provisioner := api.NewResolver(provisioningService, validator, tenantUpdater)
+		provisioner := api.NewResolver(provisioningService, validator, tenantUpdater, &testkit.TestDataWriter{})
 
 		kymaConfig := &gqlschema.KymaConfigInput{
 			Version: "1.5",
@@ -168,7 +169,7 @@ func TestResolver_ProvisionRuntime(t *testing.T) {
 		provisioningService := &mocks.Service{}
 		validator := &validatorMocks.Validator{}
 		tenantUpdater := &validatorMocks.TenantUpdater{}
-		provisioner := api.NewResolver(provisioningService, validator, tenantUpdater)
+		provisioner := api.NewResolver(provisioningService, validator, tenantUpdater, &testkit.TestDataWriter{})
 
 		kymaConfig := &gqlschema.KymaConfigInput{
 			Version: "1.5",
@@ -208,7 +209,7 @@ func TestResolver_DeprovisionRuntime(t *testing.T) {
 		provisioningService := &mocks.Service{}
 		validator := &validatorMocks.Validator{}
 		tenantUpdater := &validatorMocks.TenantUpdater{}
-		provisioner := api.NewResolver(provisioningService, validator, tenantUpdater)
+		provisioner := api.NewResolver(provisioningService, validator, tenantUpdater, &testkit.TestDataWriter{})
 
 		expectedID := "ec781980-0533-4098-aab7-96b535569732"
 
@@ -228,7 +229,7 @@ func TestResolver_DeprovisionRuntime(t *testing.T) {
 		provisioningService := &mocks.Service{}
 		validator := &validatorMocks.Validator{}
 		tenantUpdater := &validatorMocks.TenantUpdater{}
-		provisioner := api.NewResolver(provisioningService, validator, tenantUpdater)
+		provisioner := api.NewResolver(provisioningService, validator, tenantUpdater, &testkit.TestDataWriter{})
 		provisioningService.On("DeprovisionRuntime", runtimeID).Return("", apperrors.Internal("Deprovisioning fails because reasons"))
 		tenantUpdater.On("GetAndUpdateTenant", runtimeID, ctx).Return(nil)
 
@@ -246,7 +247,7 @@ func TestResolver_DeprovisionRuntime(t *testing.T) {
 		provisioningService := &mocks.Service{}
 		validator := &validatorMocks.Validator{}
 		tenantUpdater := &validatorMocks.TenantUpdater{}
-		provisioner := api.NewResolver(provisioningService, validator, tenantUpdater)
+		provisioner := api.NewResolver(provisioningService, validator, tenantUpdater, &testkit.TestDataWriter{})
 		expectedID := "ec781980-0533-4098-aab7-96b535569732"
 
 		ctx := context.Background()
@@ -273,7 +274,7 @@ func TestResolver_RuntimeStatus(t *testing.T) {
 		validator := &validatorMocks.Validator{}
 		tenantUpdater := &validatorMocks.TenantUpdater{}
 
-		provisioner := api.NewResolver(provisioningService, validator, tenantUpdater)
+		provisioner := api.NewResolver(provisioningService, validator, tenantUpdater, &testkit.TestDataWriter{})
 
 		operationID := "acc5040c-3bb6-47b8-8651-07f6950bd0a7"
 		message := "some message"
@@ -307,7 +308,7 @@ func TestResolver_RuntimeStatus(t *testing.T) {
 		validator := &validatorMocks.Validator{}
 		tenantUpdater := &validatorMocks.TenantUpdater{}
 
-		provisioner := api.NewResolver(provisioningService, validator, tenantUpdater)
+		provisioner := api.NewResolver(provisioningService, validator, tenantUpdater, &testkit.TestDataWriter{})
 
 		provisioningService.On("RuntimeStatus", runtimeID).Return(nil, apperrors.Internal("Runtime status fails"))
 		tenantUpdater.On("GetAndUpdateTenant", runtimeID, ctx).Return(nil)
@@ -332,7 +333,7 @@ func TestResolver_RuntimeOperationStatus(t *testing.T) {
 		validator := &validatorMocks.Validator{}
 		tenantUpdater := &validatorMocks.TenantUpdater{}
 
-		provisioner := api.NewResolver(provisioningService, validator, tenantUpdater)
+		provisioner := api.NewResolver(provisioningService, validator, tenantUpdater, &testkit.TestDataWriter{})
 
 		operationID := "acc5040c-3bb6-47b8-8651-07f6950bd0a7"
 		message := "some message"
@@ -363,7 +364,7 @@ func TestResolver_RuntimeOperationStatus(t *testing.T) {
 		tenantUpdater := &validatorMocks.TenantUpdater{}
 
 		validator.On("ValidateTenantForOperation", operationID, tenant).Return(nil)
-		provisioner := api.NewResolver(provisioningService, validator, tenantUpdater)
+		provisioner := api.NewResolver(provisioningService, validator, tenantUpdater, &testkit.TestDataWriter{})
 
 		provisioningService.On("RuntimeOperationStatus", operationID).Return(nil, apperrors.Internal("Some error"))
 		tenantUpdater.On("GetAndUpdateTenant", runtimeID, ctx).Return(nil)
@@ -401,7 +402,7 @@ func TestResolver_UpgradeShoot(t *testing.T) {
 		validator.On("ValidateUpgradeShootInput", upgradeShootInput).Return(nil)
 		provisioningService.On("UpgradeGardenerShoot", runtimeID, upgradeShootInput).Return(operation, nil)
 
-		resolver := api.NewResolver(provisioningService, validator, tenantUpdater)
+		resolver := api.NewResolver(provisioningService, validator, tenantUpdater, &testkit.TestDataWriter{})
 
 		//when
 		status, err := resolver.UpgradeShoot(ctx, runtimeID, upgradeShootInput)
@@ -423,7 +424,7 @@ func TestResolver_UpgradeShoot(t *testing.T) {
 		validator.On("ValidateUpgradeShootInput", upgradeShootInput).Return(apperrors.BadRequest("error"))
 		tenantUpdater.On("GetAndUpdateTenant", runtimeID, ctx).Return(nil)
 
-		resolver := api.NewResolver(provisioningService, validator, tenantUpdater)
+		resolver := api.NewResolver(provisioningService, validator, tenantUpdater, &testkit.TestDataWriter{})
 
 		//when
 		_, err := resolver.UpgradeShoot(ctx, runtimeID, upgradeShootInput)

components/provisioner/internal/gardener/provisioner.go

@@ -4,13 +4,11 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io"
 	"os"
 	"time"
 
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
-	gardener "github.com/gardener/gardener/pkg/apis/core/v1beta1"
 	"github.com/kyma-project/control-plane/components/provisioner/internal/apperrors"
 	"github.com/kyma-project/control-plane/components/provisioner/internal/util"
 	"github.com/mitchellh/mapstructure"
@@ -24,8 +22,6 @@ import (
 	"github.com/kyma-project/control-plane/components/provisioner/internal/provisioning/persistence/dbsession"
 	log "github.com/sirupsen/logrus"
 	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
-
-	"sigs.k8s.io/yaml"
 )
 
 //go:generate mockery --name=Client
@@ -35,20 +31,25 @@ type Client interface {
 	Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1beta1.Shoot, err error)
 }
 
+type OutputDataWriter interface {
+	PersistShoot(shoot *v1beta1.Shoot) (string, error)
+	Enabled() bool
+}
+
 func NewProvisioner(
 	namespace string,
 	shootClient Client,
 	factory dbsession.Factory,
 	policyConfigMapName string,
 	maintenanceWindowConfigPath string,
-	enableDumpShootSpec bool) *GardenerProvisioner {
+	testDataWriter OutputDataWriter) *GardenerProvisioner {
 	return &GardenerProvisioner{
 		namespace:                   namespace,
 		shootClient:                 shootClient,
 		dbSessionFactory:            factory,
 		policyConfigMapName:         policyConfigMapName,
 		maintenanceWindowConfigPath: maintenanceWindowConfigPath,
-		enableDumpShootSpec:         enableDumpShootSpec,
+		testDataWriter:              testDataWriter,
 	}
 }
 
@@ -58,7 +59,7 @@ type GardenerProvisioner struct {
 	dbSessionFactory            dbsession.Factory
 	policyConfigMapName         string
 	maintenanceWindowConfigPath string
-	enableDumpShootSpec         bool
+	testDataWriter              OutputDataWriter
 }
 
 func (g *GardenerProvisioner) ProvisionCluster(cluster model.Cluster, operationId string) apperrors.AppError {
@@ -98,14 +99,15 @@ func (g *GardenerProvisioner) ProvisionCluster(cluster model.Cluster, operationI
 		g.applyAuditConfig(shootTemplate)
 	}
 
-	if g.enableDumpShootSpec {
-		path := fmt.Sprintf("%s/%s-%s.yaml", "/testdata/provisioner", shootTemplate.Namespace, shootTemplate.Name)
-		if err := persist(path, shootTemplate); err != nil {
+	if g.testDataWriter.Enabled() {
+		log.Infof("Saving Shoot spec for %s Runtime", cluster.ID)
+		path, err := g.testDataWriter.PersistShoot(shootTemplate)
+
+		if err == nil {
+			log.Infof("Shoot spec dumped to %s", path)
+		} else {
 			log.Errorf("Error marshaling Shoot spec: %s", err.Error())
 		}
-		log.Infof("Shoot spec dumped to %s", path)
-	} else {
-		log.Infof("Shoot Spec Dump feature is disabled")
 	}
 
 	_, k8serr := g.shootClient.Create(context.Background(), shootTemplate, v1.CreateOptions{})
@@ -117,31 +119,6 @@ func (g *GardenerProvisioner) ProvisionCluster(cluster model.Cluster, operationI
 	return nil
 }
 
-var getWriter = func(filePath string) (io.Writer, error) {
-	file, err := os.Create(filePath)
-	if err != nil {
-		return nil, fmt.Errorf("unable to create file: %w", err)
-	}
-	return file, nil
-}
-
-func persist(path string, s *gardener.Shoot) error {
-	writer, err := getWriter(path)
-	if err != nil {
-		return fmt.Errorf("unable to create file: %w", err)
-	}
-
-	b, err := yaml.Marshal(s)
-	if err != nil {
-		return fmt.Errorf("unable to marshal shoot: %w", err)
-	}
-
-	if _, err = writer.Write(b); err != nil {
-		return fmt.Errorf("unable to write to file: %w", err)
-	}
-	return nil
-}
-
 func (g *GardenerProvisioner) UpgradeCluster(clusterID string, upgradeConfig model.GardenerConfig) apperrors.AppError {
 	err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
 		shoot, err := g.shootClient.Get(context.Background(), upgradeConfig.Name, v1.GetOptions{})