Support job transfer (#5082)

* add config

* modify job detail page

* add job transfer

* update user profile page

* fix

src/database-controller/sdk/index.js

@@ -290,7 +290,7 @@ class DatabaseModel {
           allowNull: false,
         },
         name: {
-          type: Sequelize.STRING(64),
+          type: Sequelize.STRING(512),
           allowNull: false,
         },
       },

src/webportal/config/webpack.common.js

@@ -72,6 +72,7 @@ const config = (env, argv) => ({
     jobDetail: './src/app/job/job-view/fabric/job-detail.jsx',
     taskAttempt: './src/app/job/job-view/fabric/task-attempt.jsx',
     jobEvent: './src/app/job/job-view/fabric/job-event.jsx',
+    jobTransfer: './src/app/job/job-view/fabric/job-transfer.jsx',
     virtualClusters: './src/app/vc/vc.component.js',
     services: './src/app/cluster-view/services/services.component.js',
     hardware: './src/app/cluster-view/hardware/hardware.component.js',
@@ -343,6 +344,10 @@ const config = (env, argv) => ({
       filename: 'job-event.html',
       chunks: ['layout', 'jobEvent'],
     }),
+    generateHtml({
+      filename: 'job-transfer.html',
+      chunks: ['layout', 'jobTransfer'],
+    }),
     generateHtml({
       filename: 'virtual-clusters.html',
       chunks: ['layout', 'virtualClusters'],

src/webportal/config/webportal.py

@@ -63,6 +63,7 @@ def apply_config(plugin):
             'uri': uri,
             'plugins': json.dumps([apply_config(plugin) for plugin in plugins]),
             'webportal-address': master_ip,
+            'enable-job-transfer': self.service_configuration['enable-job-transfer'],
         }
 
     #### All service and main module (kubrenetes, machine) is generated. And in this check steps, you could refer to the service object model which you will used in your own service, and check its existence and correctness.

src/webportal/config/webportal.yaml

@@ -18,3 +18,5 @@
 service_type: "common"
 
 server-port: 9286
+
+enable-job-transfer: false

src/webportal/deploy/webportal.yaml.template

@@ -81,6 +81,13 @@ spec:
 {%- endif %}
         - name: PROM_SCRAPE_TIME
           value: {{ cluster_cfg['prometheus']['scrape_interval'] * 10 }}s
+{% if cluster_cfg['webportal']['enable-job-transfer'] %}
+        - name: ENABLE_JOB_TRANSFER
+          value: "true"
+{% else %}
+        - name: ENABLE_JOB_TRANSFER
+          value: "false"
+{% endif %}
         - name: WEBPORTAL_PLUGINS
           # A raw JSON formatted value is required here.
           value: |

src/webportal/src/app/env.js.template

@@ -13,6 +13,7 @@ window.ENV = {
   alertManagerUri: '${ALERT_MANAGER_URI}/alert-manager',
   launcherType: '${LAUNCHER_TYPE}',
   launcherScheduler: '${LAUNCHER_SCHEDULER}',
+  enableJobTransfer: '${ENABLE_JOB_TRANSFER}',
 };
 
 window.PAI_PLUGINS = [${WEBPORTAL_PLUGINS}][0] || [];

src/webportal/src/app/job/job-view/fabric/job-detail.jsx

@@ -52,6 +52,14 @@ import TaskRoleContainerList from './job-detail/components/task-role-container-l
 import TaskRoleCount from './job-detail/components/task-role-count';
 import MonacoPanel from '../../../components/monaco-panel';
 
+const params = new URLSearchParams(window.location.search);
+// the user who is viewing this page
+const userName = cookies.get('user');
+// the user of the job
+const userNameOfTheJob = params.get('username');
+// is the user viewing his/her own job?
+const isViewingSelf = userName === userNameOfTheJob;
+
 class JobDetail extends React.Component {
   constructor(props) {
     super(props);
@@ -70,6 +78,7 @@ class JobDetail extends React.Component {
       loadingAttempt: false,
       monacoProps: null,
       modalTitle: '',
+      jobTransferInfo: null,
     };
     this.stop = this.stop.bind(this);
     this.reload = this.reload.bind(this);
@@ -157,6 +166,9 @@ class JobDetail extends React.Component {
     if (isNil(this.state.selectedAttemptIndex)) {
       nextState.selectedAttemptIndex = nextState.jobInfo.jobStatus.retries;
     }
+    nextState.jobTransferInfo = this.generateTransferState(
+      nextState.jobInfo.tags,
+    );
     this.setState(nextState);
   }
 
@@ -278,6 +290,53 @@ class JobDetail extends React.Component {
     }
   }
 
+  generateTransferState(tags) {
+    try {
+      // find out successfully transferred beds
+      const transferredPrefix = 'pai-transferred-to-';
+      const transferredURLs = [];
+      const transferredClusterSet = new Set();
+      for (let tag of tags) {
+        if (tag.startsWith(transferredPrefix)) {
+          tag = tag.substr(transferredPrefix.length);
+          const urlPosition = tag.lastIndexOf('-url-');
+          if (urlPosition !== -1) {
+            transferredClusterSet.add(tag.substr(0, urlPosition));
+            transferredURLs.push(tag.substr(urlPosition + 5));
+          }
+        }
+      }
+      // find out failed transfer attempts
+      const transferAttemptPrefix = 'pai-transfer-attempt-to-';
+      const transferFailedClusters = [];
+      for (let tag of tags) {
+        if (tag.startsWith(transferAttemptPrefix)) {
+          tag = tag.substr(transferAttemptPrefix.length);
+          const urlPosition = tag.lastIndexOf('-url-');
+          if (urlPosition !== -1) {
+            const cluster = tag.substr(0, urlPosition);
+            const clusterURL = tag.substr(urlPosition + 5);
+            if (!transferredClusterSet.has(cluster)) {
+              transferFailedClusters.push({
+                alias: cluster,
+                uri: clusterURL,
+              });
+            }
+          }
+        }
+      }
+
+      return { transferredURLs, transferFailedClusters };
+    } catch (err) {
+      // in case there is error with the tag parsing
+      console.error(err);
+      return {
+        transferredURLs: [],
+        transferFailedClusters: [],
+      };
+    }
+  }
+
   render() {
     const {
       loading,
@@ -289,7 +348,14 @@ class JobDetail extends React.Component {
       sshInfo,
       selectedAttemptIndex,
       loadingAttempt,
+      jobTransferInfo,
     } = this.state;
+    const transferredURLs = get(jobTransferInfo, 'transferredURLs', []);
+    const transferFailedClusters = get(
+      jobTransferInfo,
+      'transferFailedClusters',
+      [],
+    );
 
     const attemptIndexOptions = [];
     if (!isNil(jobInfo)) {
@@ -305,7 +371,9 @@ class JobDetail extends React.Component {
       return <SpinnerLoading />;
     } else {
       return (
-        <Context.Provider value={{ sshInfo, rawJobConfig, jobConfig }}>
+        <Context.Provider
+          value={{ sshInfo, rawJobConfig, jobConfig, isViewingSelf }}
+        >
           <Stack styles={{ root: { margin: '30px' } }} gap='l1'>
             <Top />
             {!isEmpty(error) && (
@@ -315,6 +383,54 @@ class JobDetail extends React.Component {
                 </MessageBar>
               </div>
             )}
+            {transferredURLs.length > 0 && (
+              <div className={t.bgWhite}>
+                <MessageBar messageBarType={MessageBarType.warning}>
+                  <Text variant='mediumPlus'>
+                    This job has been transferred to{' '}
+                    {transferredURLs
+                      .map(url => (
+                        <a
+                          href={url}
+                          key={url}
+                          target='_blank'
+                          rel='noopener noreferrer'
+                        >
+                          {url}
+                        </a>
+                      ))
+                      .reduce((prev, curr) => [prev, ', ', curr])}
+                    .{' '}
+                  </Text>
+                </MessageBar>
+              </div>
+            )}
+            {isViewingSelf && transferFailedClusters.length > 0 && (
+              <div className={t.bgWhite}>
+                <MessageBar messageBarType={MessageBarType.warning}>
+                  <Text variant='mediumPlus'>
+                    You have transfer attempts to cluster{' '}
+                    {transferFailedClusters
+                      .map(item => (
+                        <a
+                          href={item.uri}
+                          key={item.alias}
+                          target='_blank'
+                          rel='noopener noreferrer'
+                        >
+                          {item.alias}
+                        </a>
+                      ))
+                      .reduce((prev, curr) => [prev, ', ', curr])}
+                    . Please go to{' '}
+                    {transferFailedClusters.length > 1
+                      ? 'these clusters'
+                      : 'the cluster'}{' '}
+                    to check whether the transfer is successful.
+                  </Text>
+                </MessageBar>
+              </div>
+            )}
             <Summary
               className={t.mt3}
               jobInfo={jobInfo}

src/webportal/src/app/job/job-view/fabric/job-detail/components/clone-button.jsx

@@ -15,14 +15,15 @@
 // DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-import React, { useMemo } from 'react';
+import React, { useMemo, useContext } from 'react';
 import PropTypes from 'prop-types';
 import qs from 'querystring';
 import { get, isNil } from 'lodash';
 import { PrimaryButton } from 'office-ui-fabric-react';
 import { isClonable, isJobV2 } from '../util';
+import Context from './context';
 
-const CloneButton = ({ rawJobConfig, namespace, jobName }) => {
+const CloneButton = ({ rawJobConfig, namespace, jobName, enableTransfer }) => {
   const [href, onClick] = useMemo(() => {
     // TODO: align same format of jobname with each submit ways
     const queryOld = {
@@ -41,9 +42,11 @@ const CloneButton = ({ rawJobConfig, namespace, jobName }) => {
     // default
     if (isNil(pluginId)) {
       if (isJobV2(rawJobConfig)) {
-        return [`/submit.html?${qs.stringify(queryNew)}`, undefined];
+        // give a dummy function for onClick because split button depends on it to work
+        return [`/submit.html?${qs.stringify(queryNew)}`, () => {}];
       } else {
-        return [`/submit_v1.html?${qs.stringify(queryNew)}`, undefined];
+        // give a dummy function for onClick because split button depends on it to work
+        return [`/submit_v1.html?${qs.stringify(queryNew)}`, () => {}];
       }
     }
     // plugin
@@ -84,14 +87,50 @@ const CloneButton = ({ rawJobConfig, namespace, jobName }) => {
     ];
   }, [rawJobConfig]);
 
-  return (
-    <PrimaryButton
-      text='Clone'
-      href={href}
-      onClick={onClick}
-      disabled={!isClonable(rawJobConfig)}
-    />
-  );
+  let cloneButton;
+  // Only when transfer job is enabled, and the owner of this job is the one
+  // who is viewing it, show the transfer option.
+  const { isViewingSelf } = useContext(Context);
+  if (enableTransfer && isViewingSelf) {
+    cloneButton = (
+      <PrimaryButton
+        text='Clone'
+        split
+        menuProps={{
+          items: [
+            {
+              key: 'transfer',
+              text: 'Transfer',
+              iconProps: { iconName: 'Forward' },
+              onClick: () => {
+                const query = {
+                  userName: namespace,
+                  jobName: jobName,
+                };
+                window.location.href = `job-transfer.html?${qs.stringify(
+                  query,
+                )}`;
+              },
+            },
+          ],
+        }}
+        href={href}
+        onClick={onClick}
+        disabled={!isClonable(rawJobConfig)}
+      />
+    );
+  } else {
+    cloneButton = (
+      <PrimaryButton
+        text='Clone'
+        href={href}
+        onClick={onClick}
+        disabled={!isClonable(rawJobConfig)}
+      />
+    );
+  }
+
+  return cloneButton;
 };
 
 CloneButton.propTypes = {

src/webportal/src/app/job/job-view/fabric/job-detail/components/context.jsx

@@ -21,6 +21,7 @@ const Context = React.createContext({
   jobConfig: null,
   rawJobConfig: null,
   sshInfo: null,
+  isViewingSelf: null,
 });
 
 export default Context;

src/webportal/src/app/job/job-view/fabric/job-detail/components/summary.jsx

@@ -454,6 +454,7 @@ export default class Summary extends React.Component {
                   namespace={namespace}
                   jobName={jobName}
                   rawJobConfig={rawJobConfig}
+                  enableTransfer={config.enableJobTransfer === 'true'}
                 />
               </span>
               <span className={c(t.ml2)}>