diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobHistoryMapper.java b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobHistoryMapper.java index a6c58f5c27..a0d27ed480 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobHistoryMapper.java +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobHistoryMapper.java @@ -127,4 +127,6 @@ void updateJobHistoryCancelById( List selectJobHistoryByTaskidList( @Param("idList") List idList, @Param("umUser") String username); + + List selectJobHistoryNoMetrics(JobHistory jobReq); } diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/entity/QueryTaskVO.java b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/entity/QueryTaskVO.java index 90007bece8..6cc9714f63 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/entity/QueryTaskVO.java +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/entity/QueryTaskVO.java @@ -59,11 +59,13 @@ public class QueryTaskVO { private boolean canRetry; private String observeInfo; - private String isReuse; + private Boolean isReuse; private Date requestStartTime; private Date requestEndTime; private Long requestSpendTime; + private String metrics; + public List getSubJobs() { return subJobs; } @@ -298,11 +300,11 @@ public void setExecuteUser(String executeUser) { this.executeUser = executeUser; } - public String getIsReuse() { + public Boolean getIsReuse() { return isReuse; } - public void setIsReuse(String isReuse) { + public void setIsReuse(Boolean isReuse) { this.isReuse = isReuse; } @@ -329,4 +331,12 @@ public Long getRequestSpendTime() { public void setRequestSpendTime(Long requestSpendTime) { this.requestSpendTime = requestSpendTime; } + + public String getMetrics() { + return metrics; + } + + public void setMetrics(String metrics) { + this.metrics = metrics; + } } diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java index 8f5cdedd60..eccaa16686 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java @@ -20,8 +20,6 @@ import org.apache.linkis.common.conf.Configuration; import org.apache.linkis.governance.common.constant.job.JobRequestConstants; import org.apache.linkis.governance.common.entity.job.QueryException; -import org.apache.linkis.governance.common.protocol.conf.DepartmentRequest; -import org.apache.linkis.governance.common.protocol.conf.DepartmentResponse; import org.apache.linkis.jobhistory.cache.impl.DefaultQueryCacheManager; import org.apache.linkis.jobhistory.conf.JobhistoryConfiguration; import org.apache.linkis.jobhistory.conversions.TaskConversions; @@ -106,10 +104,11 @@ public Message getTaskByID(HttpServletRequest req, @PathVariable("id") Long jobI || Configuration.isDepartmentAdmin(username)) { username = null; } - JobHistory jobHistory = jobHistoryQueryService.getJobHistoryByIdAndName(jobId, username); + JobHistory jobHistory = + jobHistoryQueryService.getJobHistoryByIdAndNameNoMetrics(jobId, username); try { - if (JobhistoryConfiguration.JOB_HISTORY_QUERY_EXECUTION_CODE_SWITCH() && null != jobHistory) { + if (null != jobHistory) { QueryUtils.exchangeExecutionCode(jobHistory); } } catch (Exception e) { @@ -210,17 +209,11 @@ public Message list( username = null; } } else if (null != isDeptView && isDeptView) { - Object responseObject = sender.ask(new DepartmentRequest(username)); - if (responseObject instanceof DepartmentResponse) { - DepartmentResponse departmentResponse = (DepartmentResponse) responseObject; - if (StringUtils.isNotBlank(departmentResponse.departmentId())) { - departmentId = departmentResponse.departmentId(); - if (StringUtils.isNotBlank(proxyUser)) { - username = proxyUser; - } else { - username = null; - } - } + departmentId = JobhistoryUtils.getDepartmentByuser(username); + if (StringUtils.isNotBlank(departmentId)) { + username = proxyUser; + } else { + username = null; } } if (StringUtils.isBlank(instance)) { @@ -496,23 +489,42 @@ public Message jobeExtraInfo( if (null == jobId) { return Message.error("Invalid jobId cannot be empty"); } - if (Configuration.isJobHistoryAdmin(username) - || Configuration.isAdmin(username) - || Configuration.isDepartmentAdmin(username)) { + JobHistory jobHistory = null; + if (Configuration.isJobHistoryAdmin(username) || Configuration.isAdmin(username)) { username = null; + jobHistory = jobHistoryQueryService.getJobHistoryByIdAndName(jobId, username); + } else { + if (Configuration.isDepartmentAdmin(username)) { + String departmentId = JobhistoryUtils.getDepartmentByuser(username); + if (StringUtils.isNotBlank(departmentId)) { + List list = + jobHistoryQueryService.search( + jobId, null, null, null, null, null, null, null, null, departmentId, null); + if (!CollectionUtils.isEmpty(list)) { + jobHistory = list.get(0); + } + } + } else { + jobHistory = jobHistoryQueryService.getJobHistoryByIdAndName(jobId, username); + } } - JobHistory jobHistory = jobHistoryQueryService.getJobHistoryByIdAndName(jobId, username); - String runtime = TaskConversions.getJobRuntime(jobHistory); - try { - if (null != jobHistory) { + + if (jobHistory == null) { + return Message.error( + "The corresponding job was not found, or there may be no permission to view the job" + + "(没有找到对应的job,也可能是没有查看该job的权限)"); + } else { + try { QueryUtils.exchangeExecutionCode(jobHistory); + } catch (Exception e) { + log.error("Exchange executionCode for job with id : {} failed, {}", jobHistory.getId(), e); } - } catch (Exception e) { - log.error("Exchange executionCode for job with id : {} failed, {}", jobHistory.getId(), e); } - return Message.ok() - .data("runtime", runtime) - .data("executionCode", jobHistory.getExecutionCode()); + Map metricsMap = + BDPJettyServerHelper.gson().fromJson(jobHistory.getMetrics(), Map.class); + metricsMap.put("executionCode", jobHistory.getExecutionCode()); + metricsMap.put("runtime", TaskConversions.getJobRuntime(metricsMap)); + return Message.ok().data("metricsMap", metricsMap); } @ApiOperation( @@ -563,7 +575,8 @@ public void downloadJobList( List queryTaskVOList = BDPJettyServerHelper.gson() .fromJson(jsonStr, new TypeToken>() {}.getType()); - byte[] bytes = JobhistoryUtils.downLoadJobToExcel(queryTaskVOList, language); + byte[] bytes = + JobhistoryUtils.downLoadJobToExcel(queryTaskVOList, language, isAdminView, isDeptView); response.setCharacterEncoding(Consts.UTF_8.toString()); response.addHeader("Content-Type", "application/json;charset=UTF-8"); response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/util/JobhistoryUtils.java b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/util/JobhistoryUtils.java index abf1191430..fffa6a5536 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/util/JobhistoryUtils.java +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/util/JobhistoryUtils.java @@ -17,11 +17,16 @@ package org.apache.linkis.jobhistory.util; +import org.apache.linkis.common.conf.Configuration; import org.apache.linkis.common.utils.Utils; +import org.apache.linkis.governance.common.protocol.conf.DepartmentRequest; +import org.apache.linkis.governance.common.protocol.conf.DepartmentResponse; import org.apache.linkis.jobhistory.conversions.TaskConversions; import org.apache.linkis.jobhistory.entity.QueryTaskVO; +import org.apache.linkis.rpc.Sender; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.poi.ss.usermodel.*; import org.apache.poi.xssf.usermodel.XSSFWorkbook; @@ -32,24 +37,36 @@ public class JobhistoryUtils { public static String headersStr = - "任务ID,来源,查询语句,状态,已耗时,关键信息,应用/引擎,创建时间,是否复用,申请开始时间,申请结束时间,申请花费时间"; + "任务ID,来源,查询语句,状态,已耗时,关键信息,是否复用,申请开始时间,申请结束时间,申请耗时,应用/引擎,用户,创建时间"; public static String headersEnStr = - "JobID,Source,Execution Code,Status,Time Elapsed,Key Information,App / Engine,Created at,IsRuse,Application Start Time,Application End Time,Application Takes Time"; + "JobID,Source,Execution Code,Status,Time Elapsed,Key Information,IsRuse,Application Start Time,Application End Time,Application Takes Time,App / Engine,User,Created at"; + private static Sender sender = + Sender.getSender( + Configuration.CLOUD_CONSOLE_CONFIGURATION_SPRING_APPLICATION_NAME().getValue());; - public static byte[] downLoadJobToExcel(List jobHistoryList, String language) + public static byte[] downLoadJobToExcel( + List jobHistoryList, String language, Boolean isAdminView, Boolean isDeptView) throws IOException { ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); Workbook workbook = new XSSFWorkbook(); byte[] byteArray = new byte[0]; - Sheet sheet = workbook.createSheet("任务信息表"); // Create header row Row headerRow = sheet.createRow(0); String headers = ""; + Boolean viewResult = isAdminView || isDeptView; if (!"en".equals(language)) { - headers = headersStr; + if (viewResult) { + headers = headersStr; + } else { + headers = headersStr.replace(",用户", ""); + } } else { - headers = headersEnStr; + if (viewResult) { + headers = headersEnStr; + } else { + headers = headersEnStr.replace(",User", ""); + } } String[] headersArray = headers.split(","); for (int i = 0; i < headersArray.length; i++) { @@ -62,26 +79,39 @@ public static byte[] downLoadJobToExcel(List jobHistoryList, String Row row = sheet.createRow(rowNum++); row.createCell(0).setCellValue(queryTaskVO.getTaskID()); row.createCell(1).setCellValue(queryTaskVO.getSourceTailor()); - row.createCell(2).setCellValue(queryTaskVO.getExecutionCode()); + String executionCode = queryTaskVO.getExecutionCode(); + if (executionCode.length() >= 32767) { + executionCode = executionCode.substring(0, 32767); + } + row.createCell(2).setCellValue(executionCode); row.createCell(3).setCellValue(queryTaskVO.getStatus()); if (null == queryTaskVO.getCostTime()) { queryTaskVO.setCostTime(0L); } row.createCell(4).setCellValue(Utils.msDurationToString(queryTaskVO.getCostTime())); row.createCell(5).setCellValue(queryTaskVO.getErrDesc()); - row.createCell(6) + if (null == queryTaskVO.getIsReuse()) { + row.createCell(6).setCellValue(""); + } else { + row.createCell(6).setCellValue(queryTaskVO.getIsReuse()); + } + row.createCell(7).setCellValue(TaskConversions.dateFomat(queryTaskVO.getRequestStartTime())); + row.createCell(8).setCellValue(TaskConversions.dateFomat(queryTaskVO.getRequestEndTime())); + if (null == queryTaskVO.getRequestSpendTime()) { + queryTaskVO.setRequestSpendTime(0L); + } + row.createCell(9).setCellValue(Utils.msDurationToString(queryTaskVO.getRequestSpendTime())); + row.createCell(10) .setCellValue( queryTaskVO.getExecuteApplicationName() + "/" + queryTaskVO.getRequestApplicationName()); - row.createCell(7).setCellValue(TaskConversions.dateFomat(queryTaskVO.getCreatedTime())); - row.createCell(8).setCellValue(queryTaskVO.getIsReuse()); - row.createCell(9).setCellValue(TaskConversions.dateFomat(queryTaskVO.getRequestStartTime())); - row.createCell(10).setCellValue(TaskConversions.dateFomat(queryTaskVO.getRequestEndTime())); - if (null == queryTaskVO.getRequestSpendTime()) { - queryTaskVO.setRequestSpendTime(0L); + if (viewResult) { + row.createCell(11).setCellValue(queryTaskVO.getUmUser()); + row.createCell(12).setCellValue(TaskConversions.dateFomat(queryTaskVO.getCreatedTime())); + } else { + row.createCell(11).setCellValue(TaskConversions.dateFomat(queryTaskVO.getCreatedTime())); } - row.createCell(11).setCellValue(Utils.msDurationToString(queryTaskVO.getRequestSpendTime())); } try { workbook.write(outputStream); @@ -94,4 +124,16 @@ public static byte[] downLoadJobToExcel(List jobHistoryList, String } return byteArray; } + + public static String getDepartmentByuser(String username) { + String departmentId = ""; + Object responseObject = sender.ask(new DepartmentRequest(username)); + if (responseObject instanceof DepartmentResponse) { + DepartmentResponse departmentResponse = (DepartmentResponse) responseObject; + if (StringUtils.isNotBlank(departmentResponse.departmentId())) { + departmentId = departmentResponse.departmentId(); + } + } + return departmentId; + } } diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/resources/mapper/common/JobHistoryMapper.xml b/linkis-public-enhancements/linkis-jobhistory/src/main/resources/mapper/common/JobHistoryMapper.xml index 64f2a025e2..3a4913dfda 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/resources/mapper/common/JobHistoryMapper.xml +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/resources/mapper/common/JobHistoryMapper.xml @@ -46,8 +46,8 @@ `id`, `job_req_id`, `submit_user`, `execute_user`, `source`, `labels`, `params`, - `progress`, `status`, `log_path`, `error_code`, `error_desc`, `created_time`, `updated_time`, - `instances`, `metrics`,`engine_type`, `execution_code`, `result_location` + `status`, `log_path`, `error_code`, `error_desc`, `created_time`, `updated_time`, + `instances`,`engine_type`, `result_location`, `observe_info` @@ -80,6 +80,24 @@ + + + diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/conversions/TaskConversions.scala b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/conversions/TaskConversions.scala index 6efab76e1e..dee1d2c3f7 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/conversions/TaskConversions.scala +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/conversions/TaskConversions.scala @@ -31,13 +31,13 @@ import org.apache.linkis.protocol.constants.TaskConstant import org.apache.linkis.protocol.utils.ZuulEntranceUtils import org.apache.linkis.server.{toScalaBuffer, toScalaMap, BDPJettyServerHelper} -import org.apache.commons.lang3.StringUtils +import org.apache.commons.lang3.{BooleanUtils, StringUtils} import org.springframework.beans.BeanUtils import java.text.SimpleDateFormat import java.util -import java.util.Date +import java.util.{Date, Map} import scala.collection.JavaConverters.{asScalaBufferConverter, mapAsScalaMapConverter} @@ -278,7 +278,9 @@ object TaskConversions extends Logging { null != metrics && metrics.containsKey(TaskConstant.JOB_IS_REUSE) && metrics .get(TaskConstant.JOB_IS_REUSE) != null ) { - taskVO.setIsReuse(metrics.get(TaskConstant.JOB_IS_REUSE).toString) + + taskVO.setIsReuse(BooleanUtils.toBoolean(metrics.get(TaskConstant.JOB_IS_REUSE).toString)) + } var requestStartTime: Date = null @@ -313,7 +315,7 @@ object TaskConversions extends Logging { taskVO.setCostTime(System.currentTimeMillis() - createTime.getTime) } } - if (metrics.containsKey(TaskConstant.ENGINE_INSTANCE)) { + if (null != metrics && metrics.containsKey(TaskConstant.ENGINE_INSTANCE)) { taskVO.setEngineInstance(metrics.get(TaskConstant.ENGINE_INSTANCE).toString) } else if (TaskStatus.Failed.toString.equals(job.getStatus)) { taskVO.setCanRetry(true) @@ -342,6 +344,7 @@ object TaskConversions extends Logging { } } taskVO.setObserveInfo(job.getObserveInfo) + taskVO.setMetrics(job.getMetrics) taskVO } @@ -377,9 +380,7 @@ object TaskConversions extends Logging { } } - def getJobRuntime(jobHistory: JobHistory): String = { - val metricsMap = - BDPJettyServerHelper.gson.fromJson((jobHistory.getMetrics), classOf[util.Map[String, Object]]) + def getJobRuntime(metricsMap: util.Map[String, String]): String = { var runTime = "" if (metricsMap.containsKey(TaskConstant.JOB_COMPLETE_TIME)) { val completeTime = dealString2Date( diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/JobHistoryQueryService.java b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/JobHistoryQueryService.java index b859009ba3..fde82b0ed2 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/JobHistoryQueryService.java +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/JobHistoryQueryService.java @@ -40,6 +40,8 @@ public interface JobHistoryQueryService { JobHistory getJobHistoryByIdAndName(Long jobID, String userName); + JobHistory getJobHistoryByIdAndNameNoMetrics(Long jobID, String userName); + List search(Long jobId, String username, String creator, String status, Date sDate, Date eDate, String engineType, Long startJobId, String instance, String departmentId, String engineInstance); Integer countUndoneTasks(String username, String creator, Date sDate, Date eDate, String engineType, Long startJobId); diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala index 4a73fa1f09..a9fe5a5f1e 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala @@ -28,6 +28,7 @@ import org.apache.linkis.governance.common.entity.job.{ } import org.apache.linkis.governance.common.protocol.conf.EntranceInstanceConfRequest import org.apache.linkis.governance.common.protocol.job._ +import org.apache.linkis.jobhistory.conf.JobhistoryConfiguration import org.apache.linkis.jobhistory.conversions.TaskConversions._ import org.apache.linkis.jobhistory.dao.JobHistoryMapper import org.apache.linkis.jobhistory.entity.{JobHistory, QueryJobHistory} @@ -255,6 +256,22 @@ class JobHistoryQueryServiceImpl extends JobHistoryQueryService with Logging { if (jobHistoryList.isEmpty) null else jobHistoryList.get(0) } + override def getJobHistoryByIdAndNameNoMetrics( + jobId: java.lang.Long, + userName: String + ): JobHistory = { + val jobReq = new JobHistory + jobReq.setId(jobId) + jobReq.setSubmitUser(userName) + if (JobhistoryConfiguration.JOB_HISTORY_QUERY_EXECUTION_CODE_SWITCH) { + val jobHistoryList = jobHistoryMapper.selectJobHistory(jobReq) + if (jobHistoryList.isEmpty) null else jobHistoryList.get(0) + } else { + val jobHistoryList = jobHistoryMapper.selectJobHistoryNoMetrics(jobReq) + if (jobHistoryList.isEmpty) null else jobHistoryList.get(0) + } + } + override def search( jobId: lang.Long, username: String,