Skip to content

Commit

Permalink
【1.7.1】python物料管理 (apache#583)
Browse files Browse the repository at this point in the history 
* 代码由AI自动生成接口和AI插件生成

* 代码由AI自动生成接口和AI插件生成

* 代码由人工修改

---------

Co-authored-by: “v_kkhuang” <“420895376@qq.com”>
  • Loading branch information
@v-kkhuang
v-kkhuang and “v_kkhuang” authored Aug 24, 2024
1 parent 934b205 commit 227511f
Show file tree
Hide file tree
Showing 8 changed files with 753 additions and 3 deletions.
66 changes: 66 additions & 0 deletions ...es-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1423,6 +1423,72 @@ public Message encryptPath(
return Message.ok().data("data", fileMD5Str);
}

@ApiOperation(value = "Python模块上传", notes = "上传Python模块文件并返回文件地址", response = Message.class)
@ApiImplicitParams({
@ApiImplicitParam(name = "file", required = true, dataType = "MultipartFile", value = "上传的文件"),
@ApiImplicitParam(name = "fileName", required = true, dataType = "String", value = "文件名称")
})
@RequestMapping(path = "/python-upload", method = RequestMethod.POST)
public Message pythonUpload(
HttpServletRequest req,
@RequestParam("file") MultipartFile file,
@RequestParam(value = "fileName", required = false) String fileName)
throws WorkSpaceException, IOException {

// 获取登录用户
String username = ModuleUserUtils.getOperationUser(req, "pythonUpload");

// 校验文件名称
if (StringUtils.isBlank(fileName)) {
return Message.error("文件名称不能为空");
}
// 获取文件名称
String fileNameSuffix = fileName.substring(0, fileName.lastIndexOf("."));
if (!fileNameSuffix.matches("^[a-zA-Z][a-zA-Z0-9_]{0,49}$")) {
return Message.error("模块名称错误，仅支持数字字母下划线，且以字母开头，长度最大50");
}

// 校验文件类型
if (!file.getOriginalFilename().endsWith(".py")
&& !file.getOriginalFilename().endsWith(".zip")) {
return Message.error("仅支持.py和.zip格式模块文件");
}

// 校验文件大小
if (file.getSize() > 50 * 1024 * 1024) {
return Message.error("限制最大单个文件50M");
}

// 定义目录路径
String path = "hdfs:///appcom/linkis/udf/" + username;
FsPath fsPath = new FsPath(path);

// 获取文件系统实例
FileSystem fileSystem = fsService.getFileSystem(username, fsPath);

// 确认目录是否存在，不存在则创建新目录
if (!fileSystem.exists(fsPath)) {
try {
fileSystem.mkdirs(fsPath);
} catch (IOException e) {
return Message.error("创建目录失败：" + e.getMessage());
}
}

// 构建新的文件路径
String newPath = fsPath.getPath() + "/" + file.getOriginalFilename();
FsPath fsPathNew = new FsPath(newPath);

// 上传文件
try (InputStream is = file.getInputStream();
OutputStream outputStream = fileSystem.write(fsPathNew, true)) {
IOUtils.copy(is, outputStream);
} catch (IOException e) {
return Message.error("文件上传失败：" + e.getMessage());
}
// 返回成功消息并包含文件地址
return Message.ok().data("filePath", newPath);
}
/**
* *
*
Expand Down
268 changes: 268 additions & 0 deletions ...nhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,14 @@

package org.apache.linkis.udf.api;

import org.apache.linkis.common.conf.Configuration;
import org.apache.linkis.server.Message;
import org.apache.linkis.server.utils.ModuleUserUtils;
import org.apache.linkis.udf.entity.PythonModuleInfo;
import org.apache.linkis.udf.entity.UDFInfo;
import org.apache.linkis.udf.entity.UDFTree;
import org.apache.linkis.udf.excepiton.UDFException;
import org.apache.linkis.udf.service.PythonModuleInfoService;
import org.apache.linkis.udf.service.UDFService;
import org.apache.linkis.udf.service.UDFTreeService;
import org.apache.linkis.udf.utils.ConstantVar;
Expand All @@ -39,6 +42,7 @@
import org.springframework.util.StringUtils;
import org.springframework.web.bind.annotation.*;

import javax.annotation.Nullable;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

Expand All @@ -49,6 +53,7 @@

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.github.pagehelper.PageHelper;
import com.github.pagehelper.PageInfo;
import com.github.xiaoymin.knife4j.annotations.ApiOperationSupport;
import com.google.common.collect.Lists;
Expand All @@ -73,6 +78,7 @@ public class UDFRestfulApi {
@Autowired private UDFService udfService;

@Autowired private UDFTreeService udfTreeService;
@Autowired private PythonModuleInfoService pythonModuleInfoService;

ObjectMapper mapper = new ObjectMapper();

Expand Down Expand Up @@ -1013,4 +1019,266 @@ public Message versionInfo(
}
return message;
}

/**
* Python物料查询
*
* @param name python模块名称
* @param engineType 引擎类型(all,spark,python)
* @param username 用户名
* @param isLoad 是否加载（0-未加载，1-已加载）
* @param isExpire 是否过期（0-未过期，1-已过期）
* @param pageNow 页码
* @param pageSize 每页展示数据条数
*/
@RequestMapping(path = "/python-list", method = RequestMethod.GET)
@ApiOperation(value = "查询Python模块列表", notes = "根据条件查询Python模块信息")
@ApiImplicitParams({
@ApiImplicitParam(name = "name", value = "Python模块名称", required = false, dataType = "String"),
@ApiImplicitParam(
name = "engineType",
value = "引擎类型（all, spark, python）",
required = false,
dataType = "String"),
@ApiImplicitParam(name = "username", value = "用户名", required = false, dataType = "String"),
@ApiImplicitParam(
name = "isLoad",
value = "是否加载（0-未加载，1-已加载）",
required = false,
dataType = "Integer"),
@ApiImplicitParam(
name = "isExpire",
value = "是否过期（0-未过期，1-已过期）",
required = false,
dataType = "Integer"),
@ApiImplicitParam(name = "pageNow", value = "页码", required = false, dataType = "Integer"),
@ApiImplicitParam(name = "pageSize", value = "每页展示数据条数", required = false, dataType = "Integer")
})
public Message pythonList(
@RequestParam(value = "name", required = false) String name,
@RequestParam(value = "engineType", required = false) String engineType,
@RequestParam(value = "username", required = false) String username,
@RequestParam(value = "isLoad", required = false) Integer isLoad,
@RequestParam(value = "isExpire", required = false) Integer isExpire,
@RequestParam(value = "pageNow", required = false) Integer pageNow,
@RequestParam(value = "pageSize", required = false) Integer pageSize,
HttpServletRequest req) {

// 获取登录用户
String user = ModuleUserUtils.getOperationUser(req, "pythonList");

// 参数校验
if (org.apache.commons.lang3.StringUtils.isBlank(name)) name = null;
if (org.apache.commons.lang3.StringUtils.isBlank(engineType)) engineType = null;
if (pageNow == null) pageNow = 1;
if (pageSize == null) pageSize = 10;

// 根据管理员权限设置username
if (Configuration.isAdmin(user)) {
if (username == null) username = null;
} else {
username = user;
}

// 分页设置
PageHelper.startPage(pageNow, pageSize);
try {
// 执行数据库查询
PythonModuleInfo pythonModuleInfo = new PythonModuleInfo();
pythonModuleInfo.setName(name);
pythonModuleInfo.setEngineType(engineType);
pythonModuleInfo.setCreateUser(username);
pythonModuleInfo.setIsLoad(isLoad);
pythonModuleInfo.setIsExpire(isExpire);
List<PythonModuleInfo> pythonList = pythonModuleInfoService.getByConditions(pythonModuleInfo);
PageInfo<PythonModuleInfo> pageInfo = new PageInfo<>(pythonList);
// 封装返回结果
return Message.ok().data("pythonList", pythonList).data("totalPage", pageInfo.getTotal());
} finally {
// 关闭分页
PageHelper.clearPage();
}
}

/**
* Python物料删除
*
* @param id id
* @param isExpire 0-未过期，1-已过期
*/
@RequestMapping(path = "/python-delete", method = RequestMethod.GET)
@ApiOperation(value = "删除Python模块", notes = "根据模块ID删除Python模块,管理员可以删除任何模块，普通用户只能删除自己创建的模块")
@ApiImplicitParams({
@ApiImplicitParam(name = "id", value = "模块ID", required = true, dataType = "Long"),
@ApiImplicitParam(
name = "isExpire",
value = "模块是否过期（0：未过期，1：已过期）",
required = true,
dataType = "int")
})
public Message pythonDelete(
@RequestParam(value = "id", required = false) Long id,
@RequestParam(value = "isExpire", required = false) int isExpire,
HttpServletRequest req,
HttpServletResponse resp) {
// 打印审计日志并获取登录用户
String user = ModuleUserUtils.getOperationUser(req, "pythonDelete");

// 参数校验
if (id == null) {
return Message.error("Invalid parameters: id is null");
}
if (isExpire != 0 && isExpire != 1) {
return Message.error("Invalid parameters: isExpire must be 0 or 1");
}
PythonModuleInfo pythonModuleInfo = new PythonModuleInfo();
pythonModuleInfo.setId(id);
// 根据id查询Python模块信息
PythonModuleInfo moduleInfo = pythonModuleInfoService.getByUserAndNameAndId(pythonModuleInfo);
if (moduleInfo == null) {
return Message.ok(); // 如果不存在则直接返回成功
}

// 判断是否是管理员
if (!Configuration.isAdmin(user)) {
// 如果不是管理员，检查创建用户是否与当前用户一致
if (!moduleInfo.getCreateUser().equals(user)) {
return Message.error("无权删除他人Python模块");
}
}

// 更新Python模块信息
moduleInfo.setIsExpire(1);
moduleInfo.setUpdateUser(user);
moduleInfo.setUpdateTime(new Date());
// 修改数据库中的模块名称和文件名称
String newName = moduleInfo.getName() + "_" + System.currentTimeMillis();
String newPath = moduleInfo.getPath() + "_" + System.currentTimeMillis();
moduleInfo.setPath(newPath);
moduleInfo.setName(newName);
pythonModuleInfoService.updatePythonModuleInfo(moduleInfo);
return Message.ok();
}

/** Python物料新增/更新 */
@ApiOperation(value = "Python物料新增/更新", notes = "根据传入的Python物料信息新增或更新")
@ApiImplicitParams({
@ApiImplicitParam(
name = "Python物料新增/更新Request",
value = "Python物料新增/更新请求体",
required = false,
dataType = "PythonModuleInfo")
})
@RequestMapping(value = "/python-save", method = RequestMethod.POST)
public Message request(
@Nullable @RequestBody PythonModuleInfo pythonModuleInfo,
HttpServletRequest httpReq,
HttpServletResponse httpResp) {

// 获取登录用户
String userName = ModuleUserUtils.getOperationUser(httpReq, "pythonSave");

// 入参校验
if (org.apache.commons.lang3.StringUtils.isBlank(pythonModuleInfo.getName())) {
return Message.error("模块名称：不能为空");
}
if (org.apache.commons.lang3.StringUtils.isBlank(pythonModuleInfo.getPath())) {
return Message.error("模块物料：不能为空");
}
if (org.apache.commons.lang3.StringUtils.isBlank(pythonModuleInfo.getEngineType())) {
return Message.error("引擎类型：不能为空");
}
if (pythonModuleInfo.getIsLoad() == null) {
return Message.error("是否加载：不能为空");
}
if (pythonModuleInfo.getIsExpire() == null) {
return Message.error("是否过期：不能为空");
}
String path = pythonModuleInfo.getPath();
String fileName = path.substring(path.lastIndexOf("/") + 1, path.lastIndexOf("."));
if (!pythonModuleInfo.getName().equals(fileName)) {
return Message.error("模块名称与物料文件名称必须一样");
}
// 根据id判断是插入还是更新
if (pythonModuleInfo.getId() == null) {
PythonModuleInfo moduleInfo = pythonModuleInfoService.getByUserAndNameAndId(pythonModuleInfo);
// 插入逻辑
if (moduleInfo != null) {
return Message.error("模块" + moduleInfo.getName() + "已存在");
}
pythonModuleInfo.setCreateTime(new Date());
pythonModuleInfo.setUpdateTime(new Date());
pythonModuleInfo.setCreateUser(userName);
pythonModuleInfo.setUpdateUser(userName);
pythonModuleInfoService.insertPythonModuleInfo(pythonModuleInfo);
return Message.ok().data("id", pythonModuleInfo.getId());
} else {
PythonModuleInfo pythonModuleTmp = new PythonModuleInfo();
pythonModuleTmp.setId(pythonModuleInfo.getId());
PythonModuleInfo moduleInfo = pythonModuleInfoService.getByUserAndNameAndId(pythonModuleTmp);
// 更新逻辑
if (moduleInfo == null) {
return Message.error("未找到该Python模块");
}
if (!Configuration.isAdmin(userName) && !userName.equals(moduleInfo.getCreateUser())) {
return Message.error("无权编辑他人Python模块");
}
if (moduleInfo.getIsExpire() != 0) {
return Message.error("当前模块已过期，不允许进行修改操作");
}
pythonModuleInfo.setUpdateUser(userName);
pythonModuleInfo.setUpdateTime(new Date());
pythonModuleInfoService.updatePythonModuleInfo(pythonModuleInfo);
}
return Message.ok();
}

/**
* python文件是否存在查询
*
* @param fileName 文件名称
*/
@RequestMapping(path = "/python-file-exist", method = RequestMethod.GET)
@ApiOperation(value = "查询Python文件是否存在", notes = "根据用户名和文件名查询Python模块信息，如果存在则返回true，否则返回false")
@ApiImplicitParams({
@ApiImplicitParam(
name = "fileName",
value = "Python文件名",
required = true,
dataType = "string",
paramType = "query"),
@ApiImplicitParam(
name = "Authorization",
value = "Bearer token",
required = true,
dataType = "string",
paramType = "header")
})
public Message pythonFileExist(
@RequestParam(value = "fileName", required = false) String fileName, HttpServletRequest req) {
// 审计日志打印并获取登录用户
String userName = ModuleUserUtils.getOperationUser(req, "pythonFileExist");

// 参数校验
if (org.apache.commons.lang3.StringUtils.isBlank(fileName)) {
return Message.error("参数fileName不能为空");
}
String fileNameWithoutExtension = fileName.substring(0, fileName.lastIndexOf("."));
if (!fileNameWithoutExtension.matches("^[a-zA-Z][a-zA-Z0-9_]{0,49}$")) {
return Message.error("只支持数字字母下划线，且以字母开头，长度最大50");
}

// 封装PythonModuleInfo对象并查询数据库
PythonModuleInfo pythonModuleInfo = new PythonModuleInfo();
pythonModuleInfo.setName(fileNameWithoutExtension);
pythonModuleInfo.setCreateUser(userName);
PythonModuleInfo moduleInfo = pythonModuleInfoService.getByUserAndNameAndId(pythonModuleInfo);

// 根据查询结果返回相应信息
if (moduleInfo == null) {
return Message.ok().data("result", true);
} else {
return Message.error("模块" + fileName + "已存在，如需重新上传请先删除旧的模块");
}
}
}
Loading

0 comments on commit 227511f

Please sign in to comment.