diff --git a/api/v1alpha1/dspipeline_types.go b/api/v1alpha1/dspipeline_types.go index 527dbe453..cd32096b1 100644 --- a/api/v1alpha1/dspipeline_types.go +++ b/api/v1alpha1/dspipeline_types.go @@ -80,6 +80,13 @@ type APIServer struct { // for the api server to use instead. CustomServerConfig *ScriptConfigMap `json:"customServerConfigMap,omitempty"` + // Include custom kfp-launcher ConfigMap with the deployment of this DSP API Server. Default: false + // +kubebuilder:validation:Optional + EnableCustomKfpLauncher bool `json:"enableCustomKfpLauncher"` + // CustomKfpLauncherConfig is a custom config file that you can provide + // for the api server to use instead of the one provided with DSPO. + CustomKfpLauncherConfig *KfpLauncherConfigMap `json:"customKfpLauncherConfigMap,omitempty"` + // Default: true // Deprecated: DSP V1 only, will be removed in the future. // +kubebuilder:default:=true @@ -157,6 +164,12 @@ type ScriptConfigMap struct { Key string `json:"key,omitempty"` } +type KfpLauncherConfigMap struct { + Name string `json:"name,omitempty"` + DefaultPipelineRoot string `json:"defaultPipelineRoot,omitempty"` + Providers string `json:"providers,omitempty"` +} + type PersistenceAgent struct { // Enable DS Pipelines Operator management of Persisence Agent. Setting Deploy to false disables operator reconciliation. Default: true // +kubebuilder:default:=true diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index d918f712d..abcea859e 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -44,6 +44,11 @@ func (in *APIServer) DeepCopyInto(out *APIServer) { *out = new(ScriptConfigMap) **out = **in } + if in.CustomKfpLauncherConfig != nil { + in, out := &in.CustomKfpLauncherConfig, &out.CustomKfpLauncherConfig + *out = new(KfpLauncherConfigMap) + **out = **in + } if in.ArtifactScriptConfigMap != nil { in, out := &in.ArtifactScriptConfigMap, &out.ArtifactScriptConfigMap *out = new(ScriptConfigMap) @@ -327,6 +332,21 @@ func (in *GRPC) DeepCopy() *GRPC { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KfpLauncherConfigMap) DeepCopyInto(out *KfpLauncherConfigMap) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KfpLauncherConfigMap. +func (in *KfpLauncherConfigMap) DeepCopy() *KfpLauncherConfigMap { + if in == nil { + return nil + } + out := new(KfpLauncherConfigMap) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MLMD) DeepCopyInto(out *MLMD) { *out = *in diff --git a/config/crd/bases/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml b/config/crd/bases/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml index f2ce77647..9351f2dd3 100644 --- a/config/crd/bases/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml +++ b/config/crd/bases/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml @@ -106,6 +106,18 @@ spec: description: 'Default: true Deprecated: DSP V1 only, will be removed in the future.' type: boolean + customKfpLauncherConfigMap: + description: CustomKfpLauncherConfig is a custom config file that + you can provide for the api server to use instead of the one + provided with DSPO. + properties: + defaultPipelineRoot: + type: string + name: + type: string + providers: + type: string + type: object customServerConfigMap: description: CustomServerConfig is a custom config file that you can provide for the api server to use instead. @@ -126,6 +138,10 @@ spec: Server. Setting Deploy to false disables operator reconciliation. Default: true' type: boolean + enableCustomKfpLauncher: + description: 'Include custom kfp-launcher ConfigMap with the deployment + of this DSP API Server. Default: false' + type: boolean enableOauth: default: true description: 'Create an Openshift Route for this DSP API Server. diff --git a/config/internal/apiserver/default/kfp_launcher_config.yaml.tmpl b/config/internal/apiserver/default/kfp_launcher_config.yaml.tmpl index 1d497b584..711d7ff62 100644 --- a/config/internal/apiserver/default/kfp_launcher_config.yaml.tmpl +++ b/config/internal/apiserver/default/kfp_launcher_config.yaml.tmpl @@ -1,18 +1,23 @@ apiVersion: v1 data: - {{ if .ObjectStorageConnection.BasePath }} - defaultPipelineRoot: s3://{{.ObjectStorageConnection.Bucket}}/{{.ObjectStorageConnection.BasePath}} + {{ if .APIServer.EnableCustomKfpLauncher }} + defaultPipelineRoot: {{.APIServer.CustomKfpLauncherConfig.DefaultPipelineRoot}} + providers: '[{{.APIServer.CustomKfpLauncherConfig.Providers}}]' {{ else }} - defaultPipelineRoot: s3://{{.ObjectStorageConnection.Bucket}} + {{ if .ObjectStorageConnection.BasePath }} + defaultPipelineRoot: s3://{{.ObjectStorageConnection.Bucket}}/{{.ObjectStorageConnection.BasePath}} + {{ else }} + defaultPipelineRoot: s3://{{.ObjectStorageConnection.Bucket}} + {{ end }} + providers: | + s3: + endpoint: {{.ObjectStorageConnection.Endpoint}} + region: {{.ObjectStorageConnection.Region}} + defaultProviderSecretRef: + secretName: {{.ObjectStorageConnection.CredentialsSecret.SecretName}} + accessKeyKey: {{.ObjectStorageConnection.CredentialsSecret.AccessKey}} + secretKeyKey: {{.ObjectStorageConnection.CredentialsSecret.SecretKey}} {{ end }} - providers: | - s3: - endpoint: {{.ObjectStorageConnection.Endpoint}} - region: {{.ObjectStorageConnection.Region}} - defaultProviderSecretRef: - secretName: {{.ObjectStorageConnection.CredentialsSecret.SecretName}} - accessKeyKey: {{.ObjectStorageConnection.CredentialsSecret.AccessKey}} - secretKeyKey: {{.ObjectStorageConnection.CredentialsSecret.SecretKey}} kind: ConfigMap metadata: name: kfp-launcher diff --git a/controllers/config/defaults.go b/controllers/config/defaults.go index 1f7c02ba0..fd6a4a84c 100644 --- a/controllers/config/defaults.go +++ b/controllers/config/defaults.go @@ -53,6 +53,10 @@ const ( CustomServerConfigMapNameKey = "config.json" DSPServicePrefix = "ds-pipeline" + CustomKfpLauncherConfigMapName = "kfp-launcher" + CustomKfpLauncherConfigMapDefaultPipelineRoot = "defaultPipelineRoot" + CustomKfpLauncherConfigMapProviders = "providers" + DefaultDBSecretNamePrefix = "ds-pipeline-db-" DefaultDBSecretKey = "password" GeneratedDBPasswordLength = 12 diff --git a/controllers/dspipeline_params.go b/controllers/dspipeline_params.go index d009527e9..0cc1328ee 100644 --- a/controllers/dspipeline_params.go +++ b/controllers/dspipeline_params.go @@ -610,6 +610,33 @@ func (p *DSPAParams) ExtractParams(ctx context.Context, dsp *dspa.DataSciencePip } } + if p.APIServer.CustomKfpLauncherConfig == nil { + p.APIServer.CustomKfpLauncherConfig = &dspa.KfpLauncherConfigMap{ + Name: config.CustomKfpLauncherConfigMapName, + DefaultPipelineRoot: config.CustomKfpLauncherConfigMapDefaultPipelineRoot, + Providers: config.CustomKfpLauncherConfigMapProviders, + } + } else { + customKfpLauncherCFGMapName := p.APIServer.CustomKfpLauncherConfig.Name + customKfpLauncherConfigMap, err := util.GetConfigMap(ctx, customKfpLauncherCFGMapName, p.Namespace, client) + if err != nil { + // If the custom kfp-launcher configmap is not available, that is OK + if !apierrs.IsNotFound(err) { + log.Info(fmt.Sprintf("Encountered error when attempting to fetch ConfigMap: [%s], Error: %v", customKfpLauncherCFGMapName, err)) + return err + } + } else { + // Consume both defaultPipelineRoot and providers. + defaultPipelineRootValue := util.GetConfigMapValue(p.APIServer.CustomKfpLauncherConfig.DefaultPipelineRoot, customKfpLauncherConfigMap) + providersValue := util.GetConfigMapValue(p.APIServer.CustomKfpLauncherConfig.Providers, customKfpLauncherConfigMap) + p.APIServer.CustomKfpLauncherConfig = &dspa.KfpLauncherConfigMap{ + Name: config.CustomKfpLauncherConfigMapName, + DefaultPipelineRoot: defaultPipelineRootValue, + Providers: providersValue, + } + } + } + // Track whether the "ca-bundle.crt" configmap key from odh-trusted-ca bundle // was found, this will be used to decide whether we need to account for this // ourselves later or not.