move storage to plugin for k8s

microsoft · Dec 12, 2019 · 1ab4cff · 1ab4cff
1 parent 013eecd
commit 1ab4cff
Show file tree

Hide file tree

Showing 4 changed files with 79 additions and 5 deletions.
diff --git a/src/webportal/src/app/job-submission/components/data/data-component.jsx b/src/webportal/src/app/job-submission/components/data/data-component.jsx
@@ -18,8 +18,25 @@ import {
 import config from '../../../config/webportal.config';
 import { JobData } from '../../models/data/job-data';
 import { Hint } from '../sidebar/hint';
-import { PROTOCOL_TOOLTIPS } from '../../utils/constants';
-import { isNil } from 'lodash';
+import { PROTOCOL_TOOLTIPS, PAI_PLUGIN } from '../../utils/constants';
+import { isNil, isEmpty, get } from 'lodash';
+
+const generateUpdatedRuntimePlugins = (storageConfigs, oriPlugins) => {
+  const updatedPlugins = oriPlugins.filter(
+    plugin => plugin.plugin !== 'teamwise_storage',
+  );
+
+  if (!isEmpty(storageConfigs)) {
+    const storagePlugin = {
+      plugin: 'teamwise_storage',
+      parameters: {
+        storageConfigNames: storageConfigs.map(config => config.name),
+      },
+    };
+    updatedPlugins.push(storagePlugin);
+  }
+  return updatedPlugins;
+};
 
 function reducer(state, action) {
   let jobData;
@@ -40,6 +57,17 @@ function reducer(state, action) {
         action.value,
         true,
       );
+      if (config.launcherType === 'k8s') {
+        const plugins = get(action, ['extras', PAI_PLUGIN], []);
+        const updatedExtras = {
+          ...action.extras,
+          [PAI_PLUGIN]: generateUpdatedRuntimePlugins(
+            action.value.selectedConfigs,
+            plugins,
+          ),
+        };
+        action.onExtrasChange(updatedExtras);
+      }
       action.onChange(jobData);
       return jobData;
     default:
@@ -66,7 +94,7 @@ export const DataComponent = React.memo(props => {
     port,
     apiPath,
   );
-  const { onChange, storageConfigs } = props;
+  const { onChange, storageConfigs, extras, onExtrasChange } = props;
   const [teamConfigs, setTeamConfigs] = useState();
   const [teamServers, setTeamServers] = useState();
   const [defaultTeamConfigs, setDefaultTeamConfigs] = useState();
@@ -152,9 +180,15 @@ export const DataComponent = React.memo(props => {
 
   const onMountDirChange = useCallback(
     mountDir => {
-      dispatch({ type: 'mountDir', value: mountDir, onChange: onChange });
+      dispatch({
+        type: 'mountDir',
+        value: mountDir,
+        onChange: onChange,
+        extras: extras,
+        onExtrasChange: onExtrasChange,
+      });
     },
-    [onChange],
+    [onChange, onExtrasChange],
   );
 
   return (
@@ -207,5 +241,7 @@ DataComponent.propTypes = {
   onSelect: PropTypes.func,
   jobName: PropTypes.string,
   onChange: PropTypes.func.isRequired,
+  extras: PropTypes.object,
+  onExtrasChange: PropTypes.func.isRequired,
   storageConfigs: PropTypes.array,
 };
diff --git a/src/webportal/src/app/job-submission/components/data/team-storage.jsx b/src/webportal/src/app/job-submission/components/data/team-storage.jsx
@@ -69,11 +69,21 @@ export const TeamStorage = ({
     });
   });
 
+  const [mountPoints, setMountPoints] = useState(() => {
+    return mountDirs.selectedConfigs.flatMap(ele =>
+      ele.mountInfos.map(mountInfo => mountInfo.mountPoint),
+    );
+  });
+
   useEffect(() => {
     const names = defaultTeamConfigs.map(element => {
       return element.name;
     });
+    const mountPoints = defaultTeamConfigs.flatMap(ele =>
+      ele.mountInfos.map(mountInfo => mountInfo.mountPoint),
+    );
     setSelectedConfigNames(names);
+    setMountPoints(mountPoints);
   }, [defaultTeamConfigs]);
 
   const [teamDetail, setTeamDetail] = useState({ isOpen: false });
@@ -113,20 +123,40 @@ export const TeamStorage = ({
             }
             onChange={(ev, isChecked) => {
               let newSelectedConfigNames = [];
+              let updatedMountPoints = [...mountPoints];
               if (!isChecked && selectedConfigNames.includes(item.name)) {
                 const idx = selectedConfigNames.indexOf(item.name);
                 newSelectedConfigNames = [
                   ...selectedConfigNames.slice(0, idx),
                   ...selectedConfigNames.slice(idx + 1),
                 ];
+                const toRemovedMountPoints = item.mountInfos.map(
+                  mountInfo => mountInfo.mountPoint,
+                );
+                updatedMountPoints = updatedMountPoints.filter(
+                  mountPoint => !toRemovedMountPoints.includes(mountPoint),
+                );
               } else if (
                 isChecked &&
                 !selectedConfigNames.includes(item.name)
               ) {
+                for (const mountInfo of item.mountInfos) {
+                  if (mountPoints.includes(mountInfo.mountPoint)) {
+                    alert(
+                      `Mount point error! More than one mount point ${mountInfo.mountPoint}!`,
+                    );
+                    return;
+                  }
+                }
                 newSelectedConfigNames = cloneDeep(selectedConfigNames);
                 newSelectedConfigNames.push(item.name);
+
+                item.mountInfos.forEach(mountInfo =>
+                  updatedMountPoints.push(mountInfo.mountPoint),
+                );
               }
               setSelectedConfigNames(newSelectedConfigNames);
+              setMountPoints(updatedMountPoints);
             }}
           />
         );

diff --git a/src/webportal/src/app/job-submission/job-submission-page.jsx b/src/webportal/src/app/job-submission/job-submission-page.jsx
@@ -430,6 +430,8 @@ export const JobSubmissionPage = ({
                     onSelect={selectData}
                     jobName={jobInformation.name}
                     onChange={setJobData}
+                    extras={extras}
+                    onExtrasChange={setExtras}
                     storageConfigs={storageConfigs}
                   />
                   <ToolComponent

diff --git a/src/webportal/src/app/job-submission/utils/utils.js b/src/webportal/src/app/job-submission/utils/utils.js
@@ -14,6 +14,7 @@ import {
   AUTO_GENERATE_NOTIFY,
   PAI_STORAGE,
 } from './constants';
+import config from '../../config/webportal.config';
 
 const HIDE_SECRET = '******';
 
@@ -187,6 +188,11 @@ export async function populateProtocolWithDataAndTensorboard(
   protocol,
   jobData,
 ) {
+  // for k8s, we use runtime plugin and not inject code into the command
+  if (config.launcherType === 'k8s') {
+    return;
+  }
+
   // add tensorboard commands
   addTensorBoardCommandsToProtocolTaskRoles(protocol);