Merge branch 'main' into add-maxcompute-batch-predictor

api/.golangci.yml

@@ -1,9 +1,6 @@
 run:
   timeout: 10m
   modules-download-mode: readonly
-  skip-dirs:
-  - client
-  - '.*/mocks/.*'
 
 linters:
 # See https://golangci-lint.run/usage/linters/
@@ -17,3 +14,6 @@ issues:
   exclude-use-default: false
   max-issues-per-linter: 0
   max-same-issues: 0
+  exclude-dirs:
+    - client
+    - '.*/mocks/.*'

api/cluster/controller.go

@@ -536,7 +536,7 @@ func (c *controller) createSecrets(ctx context.Context, modelService *models.Ser
 		return fmt.Errorf("failed creating secret for model %s in namespace %s: %w", modelService.Name, modelService.Namespace, err)
 	}
 
-	if modelService.Transformer != nil {
+	if modelService.Transformer != nil && modelService.Transformer.Enabled {
 		transformerSecretName := fmt.Sprintf("%s-transformer", modelService.Name)
 		err = c.createSecretForComponent(ctx, transformerSecretName, modelService.Transformer.Secrets, modelService.Namespace, projectID)
 		if err != nil {

api/config/observability.go

@@ -1,6 +1,9 @@
 package config
 
-import "time"
+import (
+	"strings"
+	"time"
+)
 
 // ObservabilityPublisher
 type ObservabilityPublisher struct {
@@ -26,8 +29,19 @@ type KafkaConsumer struct {
 
 // ArizeSink
 type ArizeSink struct {
-	APIKey   string
-	SpaceKey string
+	APIKey              string
+	SpaceKey            string
+	EnabledModelSerials string
+}
+
+func (az ArizeSink) IsEnabled(modelSerial string) bool {
+	for _, ems := range strings.Split(az.EnabledModelSerials, ",") {
+		if ems == modelSerial {
+			return true
+		}
+	}
+
+	return false
 }
 
 // BigQuerySink

api/models/observability_publisher.go

@@ -43,6 +43,10 @@ type WorkerData struct {
 	ResourceRequest *WorkerResourceRequest
 }
 
+func (wd *WorkerData) GetModelSerial() string {
+	return fmt.Sprintf("%s_%s", wd.Project, wd.ModelName)
+}
+
 func NewWorkerData(modelVersion *Version, model *Model, observabilityPublisher *ObservabilityPublisher, resourceRequest *WorkerResourceRequest) *WorkerData {
 	return &WorkerData{
 		ModelName:       model.Name,

api/pkg/observability/deployment/deployment.go

@@ -261,28 +261,33 @@ func (c *deployer) applySecret(ctx context.Context, data *models.WorkerData) (se
 }
 
 func (c *deployer) createSecretSpec(data *models.WorkerData) (*corev1.Secret, error) {
-	consumerCfg := &ConsumerConfig{
-		Project:         data.Project,
-		ModelID:         data.ModelName,
-		ModelVersion:    data.ModelVersion,
-		InferenceSchema: data.ModelSchemaSpec,
-		ObservationSinks: []ObservationSink{
-			{
-				Type: Arize,
-				Config: ArizeSink{
-					APIKey:   c.consumerConfig.ArizeSink.APIKey,
-					SpaceKey: c.consumerConfig.ArizeSink.SpaceKey,
-				},
-			},
-			{
-				Type: BQ,
-				Config: BigQuerySink{
-					Project: c.consumerConfig.BigQuerySink.Project,
-					Dataset: c.consumerConfig.BigQuerySink.Dataset,
-					TTLDays: c.consumerConfig.BigQuerySink.TTLDays,
-				},
+	observationSinks := []ObservationSink{
+		{
+			Type: BQ,
+			Config: BigQuerySink{
+				Project: c.consumerConfig.BigQuerySink.Project,
+				Dataset: c.consumerConfig.BigQuerySink.Dataset,
+				TTLDays: c.consumerConfig.BigQuerySink.TTLDays,
 			},
 		},
+	}
+
+	if c.consumerConfig.ArizeSink.IsEnabled(data.GetModelSerial()) {
+		observationSinks = append(observationSinks, ObservationSink{
+			Type: Arize,
+			Config: ArizeSink{
+				APIKey:   c.consumerConfig.ArizeSink.APIKey,
+				SpaceKey: c.consumerConfig.ArizeSink.SpaceKey,
+			},
+		})
+	}
+
+	consumerCfg := &ConsumerConfig{
+		Project:          data.Project,
+		ModelID:          data.ModelName,
+		ModelVersion:     data.ModelVersion,
+		InferenceSchema:  data.ModelSchemaSpec,
+		ObservationSinks: observationSinks,
 		ObservationSource: &ObserVationSource{
 			Type: Kafka,
 			Config: &KafkaSource{

ui/src/assets/scss/Secrets.scss

@@ -0,0 +1,38 @@
+/**
+ * Copyright 2020 The Merlin Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+.Secrets {
+  .euiTableRowCell > .euiTableCellContent {
+    display: block;
+  }
+}
+
+input[type="text"].inlineTableInput {
+  float: left;
+  padding: 0;
+  font-size: inherit !important;
+  line-height: inherit !important;
+  font-family: inherit !important;
+  height: inherit !important;
+  transition: inherit !important;
+  background-color: inherit !important;
+  box-shadow: none !important;
+  min-width: 90px;
+
+  &:focus {
+    background-size: 0 100%;
+  }
+}

ui/src/components/EnvVarsConfigTable.js

@@ -24,13 +24,13 @@ export const EnvVarsConfigTable = ({ variables = [] }) => {
     {
       field: "name",
       name: "Name",
-      width: "20%",
+      width: "40%",
       sortable: true
     },
     {
       field: "value",
       name: "Value",
-      width: "80%",
+      width: "60%",
       sortable: true
     }
   ];

ui/src/components/SecretsConfigTable.js

@@ -0,0 +1,52 @@
+/**
+ * Copyright 2020 The Merlin Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from "react";
+import PropTypes from "prop-types";
+import { EuiInMemoryTable, EuiText } from "@elastic/eui";
+
+export const SecretsConfigTable = ({ variables = [] }) => {
+  const columns = [
+    {
+      field: "mlp_secret_name",
+      name: "MLP Secret Name",
+      width: "40%",
+      sortable: true
+    },
+    {
+      field: "env_var_name",
+      name: "Environment Variable Name",
+      width: "60%",
+      sortable: true
+    }
+  ];
+
+  return variables.length ? (
+    <EuiInMemoryTable
+      items={variables}
+      columns={columns}
+      itemId="name"
+    />
+  ) : (
+    <EuiText size="s" color="subdued">
+      Not available
+    </EuiText>
+  );
+};
+
+SecretsConfigTable.propTypes = {
+  variables: PropTypes.array.isRequired
+};

ui/src/pages/job/components/JobConfig.js

@@ -1,5 +1,6 @@
 import { EuiFlexGroup, EuiFlexItem } from "@elastic/eui";
 import { EnvVarsConfigTable } from "../../../components/EnvVarsConfigTable";
+import { SecretsConfigTable } from "../../../components/SecretsConfigTable";
 import {
   ConfigSection,
   ConfigSectionPanel,
@@ -35,10 +36,20 @@ const JobConfig = ({ job }) => {
           <EuiFlexGroup gutterSize="xl">
             <EuiFlexItem>
               <ConfigSectionPanel>
-                <ConfigSectionPanelTitle title="Environment Variables" />
-                <EnvVarsConfigTable
-                  variables={job.config.env_vars ? job.config.env_vars : []}
-                />
+                <EuiFlexGroup direction="column" gutterSize="m">
+                  <EuiFlexItem>
+                    <ConfigSectionPanelTitle title="Environment Variables" />
+                    <EnvVarsConfigTable
+                      variables={job.config.env_vars ? job.config.env_vars : []}
+                    />
+                  </EuiFlexItem>
+                  <EuiFlexItem>
+                    <ConfigSectionPanelTitle title="Secrets" />
+                    <SecretsConfigTable
+                      variables={job.config.secrets ? job.config.secrets : []}
+                    />
+                  </EuiFlexItem>
+                </EuiFlexGroup>
               </ConfigSectionPanel>
             </EuiFlexItem>
 

ui/src/pages/job/components/ResourcesConfigTable.js

@@ -38,7 +38,7 @@ export const ResourcesConfigTable = ({
       compressed
       type="responsiveColumn"
       listItems={items}
-      columnWidths={[1, 1]}
+      columnWidths={[2, 4]}
     />
   );
 };

ui/src/pages/job/form/JobFormOthers.js

@@ -26,6 +26,7 @@ import PropTypes from "prop-types";
 import React, { Fragment, useContext } from "react";
 import { ImageBuilderSection } from "../../version/components/forms/components/ImageBuilderSection";
 import { EnvironmentVariablesForm } from "./components/EnvironmentVariablesForm";
+import { SecretsForm } from "./components/SecretsForm";
 import { ModelVersionSelect } from "./components/ModelVersionsSelect";
 import { ResourceRequestForm } from "./components/ResourceRequestForm";
 import { ServiceAccountSelect } from "./components/ServiceAccountSelect";
@@ -38,6 +39,7 @@ export const JobFormOthers = ({ versions, isSelectVersionDisabled }) => {
     setServiceAccountName,
     setResourceRequest,
     setEnvVars,
+    setSecrets,
     onChangeHandler,
   } = useContext(JobFormContext);
   const { onChange } = useOnChangeHandler(onChangeHandler);
@@ -49,7 +51,7 @@ export const JobFormOthers = ({ versions, isSelectVersionDisabled }) => {
           <EuiFlexItem>
             <ModelVersionSelect
               isDisabled={isSelectVersionDisabled}
-              selected={job.version_id}
+              selected={job.version_id.toString()}
               versions={versions}
               onChange={(selected) => {
                 setVersionId(selected);
@@ -77,13 +79,21 @@ export const JobFormOthers = ({ versions, isSelectVersionDisabled }) => {
       />
 
       <EuiSpacer size="xl" />
+
       <EuiFlexGroup>
-        <EuiFlexItem style={{ maxWidth: 400 }}>
+        <EuiFlexItem>
           <EnvironmentVariablesForm
             variables={job.config.env_vars}
             onChange={setEnvVars}
           />
         </EuiFlexItem>
+
+        <EuiFlexItem>
+          <SecretsForm
+            variables={job.config.secrets}
+            onChange={setSecrets}
+          />
+        </EuiFlexItem>
       </EuiFlexGroup>
 
       <EuiSpacer size="l" />

ui/src/pages/job/form/components/EnvironmentVariablesForm.js

@@ -125,7 +125,6 @@ export const EnvironmentVariablesForm = ({ variables, onChange }) => {
         className="EnvVariables"
         columns={columns}
         items={items}
-        hasActions={true}
       />
     </Fragment>
   );