Configure yqla mrjob syslibs (#409)

* Configure yqla mrjob syslibs * Update yqlConfig * Make mrjob dlls configurable * Omitempty mr_job_system_libs_with_md5 * Unify option name --------- Co-authored-by: Marina Pereskokova <mvpereskokova@gmail.com>
ytsaurus · Jan 9, 2025 · d2db891 · d2db891
1 parent 04391f7
commit d2db891
Show file tree

Hide file tree

Showing 5 changed files with 32 additions and 3 deletions.
diff --git a/api/v1/ytsaurus_types.go b/api/v1/ytsaurus_types.go
@@ -538,6 +538,11 @@ type StrawberryControllerSpec struct {
 
 type YQLAgentSpec struct {
 	InstanceSpec `json:",inline"`
+
+	// Sets mrjob dynamic library paths in yql agent config.
+	//+kubebuilder:default:=false
+	//+optional
+	ConfigureMrJobSystemLibs bool `json:"configureMrJobSystemLibs"`
 }
 
 type QueueAgentSpec struct {

diff --git a/config/crd/bases/cluster.ytsaurus.tech_ytsaurus.yaml b/config/crd/bases/cluster.ytsaurus.tech_ytsaurus.yaml
@@ -35297,6 +35297,10 @@ spec:
                             type: array
                         type: object
                     type: object
+                  configureMrJobSystemLibs:
+                    default: false
+                    description: Sets mrjob dynamic library paths in yql agent config.
+                    type: boolean
                   enableAntiAffinity:
                     description: 'Deprecated: use Affinity.PodAntiAffinity instead.'
                     type: boolean

diff --git a/docs/api.md b/docs/api.md
@@ -2152,6 +2152,7 @@ _Appears in:_
 | `setHostnameAsFqdn` _boolean_ | SetHostnameAsFQDN indicates whether to set the hostname as FQDN. | true |  |
 | `terminationGracePeriodSeconds` _integer_ | Optional duration in seconds the pod needs to terminate gracefully. |  |  |
 | `nativeTransport` _[RPCTransportSpec](#rpctransportspec)_ | Component config for native RPC bus transport. |  |  |
+| `configureMrJobSystemLibs` _boolean_ | Sets mrjob dynamic library paths in yql agent config. | false |  |
 
 
 #### Ytsaurus

diff --git a/pkg/ytconfig/yql_agent.go b/pkg/ytconfig/yql_agent.go
@@ -11,10 +11,15 @@ type ClusterMapping struct {
 	Default bool   `yson:"default"`
 }
 
+type MrJobSystemLibsWithMd5 struct {
+	File string `yson:"file"`
+}
+
 type GatewayConfig struct {
-	MRJobBinary    string           `yson:"mr_job_bin"`
-	UDFDirectory   string           `yson:"mr_job_udfs_dir"`
-	ClusterMapping []ClusterMapping `yson:"cluster_mapping"`
+	MRJobBinary            string                   `yson:"mr_job_bin"`
+	UDFDirectory           string                   `yson:"mr_job_udfs_dir"`
+	ClusterMapping         []ClusterMapping         `yson:"cluster_mapping"`
+	MrJobSystemLibsWithMd5 []MrJobSystemLibsWithMd5 `yson:"mr_job_system_libs_with_md5,omitempty"`
 }
 
 type YQLAgent struct {
@@ -50,6 +55,16 @@ func getYQLAgentServerCarcass(spec *ytv1.YQLAgentSpec) (YQLAgentServer, error) {
 	c.User = "yql_agent"
 
 	c.YQLAgent.GatewayConfig.UDFDirectory = "/usr/lib/yql"
+	if spec.ConfigureMrJobSystemLibs {
+		c.YQLAgent.GatewayConfig.MrJobSystemLibsWithMd5 = []MrJobSystemLibsWithMd5{
+			MrJobSystemLibsWithMd5{
+				File: "/usr/lib/yql/libiconv.so",
+			},
+			MrJobSystemLibsWithMd5{
+				File: "/usr/lib/yql/liblibidn-dynamic.so",
+			},
+		}
+	}
 	c.YQLAgent.GatewayConfig.MRJobBinary = "/usr/bin/mrjob"
 	c.YQLAgent.YqlPluginSharedLibrary = "/usr/lib/yql/libyqlplugin.so"
 

diff --git a/ytop-chart/templates/crds/ytsaurus.cluster.ytsaurus.tech.yaml b/ytop-chart/templates/crds/ytsaurus.cluster.ytsaurus.tech.yaml
@@ -35308,6 +35308,10 @@ spec:
                             type: array
                         type: object
                     type: object
+                  configureMrJobSystemLibs:
+                    default: false
+                    description: Sets mrjob dynamic library paths in yql agent config.
+                    type: boolean
                   enableAntiAffinity:
                     description: 'Deprecated: use Affinity.PodAntiAffinity instead.'
                     type: boolean