From 90a1495b9d765c9e1d48f689dbcf977cbdb2768e Mon Sep 17 00:00:00 2001
From: dengzedong <dengzedong@email>
Date: 星期二, 31 十二月 2024 17:23:13 +0800
Subject: [PATCH] 模型结果修改为存在唯一索引则修改，不存在插入。避免死锁问题
---
iailab-module-model/iailab-module-model-biz/src/main/java/com/iailab/module/model/mpk/service/impl/MpkFileServiceImpl.java | 175 +++++++++++++++++++++++++++------------------------------
1 files changed, 83 insertions(+), 92 deletions(-)
diff --git a/iailab-module-model/iailab-module-model-biz/src/main/java/com/iailab/module/model/mpk/service/impl/MpkFileServiceImpl.java b/iailab-module-model/iailab-module-model-biz/src/main/java/com/iailab/module/model/mpk/service/impl/MpkFileServiceImpl.java
index 62151a7..f141bdf 100644
--- a/iailab-module-model/iailab-module-model-biz/src/main/java/com/iailab/module/model/mpk/service/impl/MpkFileServiceImpl.java
+++ b/iailab-module-model/iailab-module-model-biz/src/main/java/com/iailab/module/model/mpk/service/impl/MpkFileServiceImpl.java
@@ -1,7 +1,6 @@
package com.iailab.module.model.mpk.service.impl;
import cn.hutool.core.io.FileUtil;
-import cn.hutool.core.util.RuntimeUtil;
import cn.hutool.core.util.ZipUtil;
import com.alibaba.fastjson.JSON;
import com.baomidou.dynamic.datasource.annotation.DSTransactional;
@@ -15,7 +14,10 @@
import com.iailab.framework.security.core.util.SecurityFrameworkUtils;
import com.iailab.framework.tenant.core.context.TenantContextHolder;
import com.iailab.module.infra.api.config.ConfigApi;
+import com.iailab.module.model.mcs.pre.service.MmModelArithSettingsService;
+import com.iailab.module.model.mcs.sche.service.StScheduleModelSettingService;
import com.iailab.module.model.mpk.common.MdkConstant;
+import com.iailab.module.model.mpk.common.utils.CmdUtils;
import com.iailab.module.model.mpk.common.utils.DllUtils;
import com.iailab.module.model.mpk.common.utils.GenUtils;
import com.iailab.module.model.mpk.dao.MpkFileDao;
@@ -39,10 +41,12 @@
import org.springframework.util.CollectionUtils;
import org.springframework.web.multipart.MultipartFile;
-import java.io.*;
+import java.io.File;
+import java.io.IOException;
import java.net.URLClassLoader;
import java.nio.file.Files;
import java.util.*;
+import java.util.function.Function;
import java.util.stream.Collectors;
/**
@@ -67,6 +71,12 @@
private ProjectPackageHistoryModelService projectPackageHistoryModelService;
@Autowired
+ private MmModelArithSettingsService mmModelArithSettingsService;
+
+ @Autowired
+ private StScheduleModelSettingService stScheduleModelSettingService;
+
+ @Autowired
private ConfigApi configApi;
@Value("${mpk.bak-file-path}")
@@ -87,19 +97,22 @@
@Override
public List<MpkFileDTO> list(Map<String, Object> params) {
- List<MpkFileEntity> entityList = baseDao.selectList(getWrapper(params).orderByDesc("create_date"));
+// List<MpkFileEntity> entityList = baseDao.selectList(getWrapper(params).orderByDesc("create_date"));
+ List<MpkFileDTO> list = baseDao.list(params);
- return ConvertUtils.sourceToTarget(entityList, MpkFileDTO.class);
+ return list;
}
private QueryWrapper<MpkFileEntity> getWrapper(Map<String, Object> params) {
+ String pyChineseName = (String) params.get("pyChineseName");
String pyName = (String) params.get("pyName");
String pyType = (String) params.get("pyType");
String remark = (String) params.get("remark");
String label = (String) params.get("label");
QueryWrapper<MpkFileEntity> wrapper = new QueryWrapper<>();
- wrapper.like(StringUtils.isNotBlank(pyName), "py_name", pyName)
+ wrapper.like(StringUtils.isNotBlank(pyChineseName), "py_chinese_name", pyChineseName)
+ .like(StringUtils.isNotBlank(pyName), "py_name", pyName)
.eq(StringUtils.isNotBlank(pyType), "py_type", pyType)
.like(StringUtils.isNotBlank(remark), "remark", remark);
@@ -166,38 +179,43 @@
@Override
@DSTransactional(rollbackFor = Exception.class)
public void update(MpkFileDTO dto) {
- // 判断py文件是否修改
- MpkFileEntity oldEntity = selectById(dto.getId());
- if (!oldEntity.getFilePath().equals(dto.getFilePath())) {
- // 将备份的pyd文件，转移到MDK_PKGS环境变量下，并添加方法的默认参数（pyFile，模型路径）
- String mdkPkgs = System.getenv("MDK_PKGS");
- String fileName = dto.getFilePath().substring(dto.getFilePath().lastIndexOf("\\") + 1,dto.getFilePath().lastIndexOf(".pyd"));
- String pyFilePath = mdkPkgs + File.separator + dto.getPyModule().replace(".", File.separator) + File.separator + fileName + ".pyd";
+ // 添加方法的默认参数（pyFile，模型路径）
+ String mdkPkgs = System.getenv("MDK_PKGS");
+ String fileName = dto.getFilePath().substring(dto.getFilePath().lastIndexOf("\\") + 1,dto.getFilePath().lastIndexOf(".pyd"));
+ String pyFilePath = mdkPkgs + File.separator + dto.getPyModule().replace(".", File.separator) + File.separator + fileName + ".pyd";
+ // 判断文件是否存在,不存在的话将备份的pyd文件，转移到MDK_PKGS环境变量下
+ File pyFile = new File(pyFilePath);
+ if (!pyFile.exists()) {
FileUtil.mkParentDirs(pyFilePath);
FileUtil.copy(dto.getFilePath(), pyFilePath, true);
+ }
- // 添加/修改参数
- for (ModelMethodDTO method : dto.getModelMethods()) {
- List<MethodSettingDTO> methodSettings = method.getMethodSettings();
- if (methodSettings.stream().anyMatch(e -> e.getSettingKey().equals(MdkConstant.PY_FILE_KEY))) {
- methodSettings.forEach(e -> {
- if (e.getSettingKey().equals(MdkConstant.PY_FILE_KEY)) {
- e.setValue(dto.getPyModule() + "." + fileName);
- }
- });
- }else {
- MethodSettingDTO setting = new MethodSettingDTO();
- setting.setId(UUID.randomUUID().toString());
- setting.setMethodId(method.getId());
- setting.setSettingKey(MdkConstant.PY_FILE_KEY);
- setting.setValue(dto.getPyModule() + "." + fileName);
- setting.setName("模型路径");
- setting.setType("input");
- setting.setValueType("string");
- methodSettings.add(setting);
- }
+ // 添加/修改参数
+ for (ModelMethodDTO method : dto.getModelMethods()) {
+ List<MethodSettingDTO> methodSettings = method.getMethodSettings();
+ if (methodSettings.stream().anyMatch(e -> e.getSettingKey().equals(MdkConstant.PY_FILE_KEY))) {
+ methodSettings.forEach(e -> {
+ if (e.getSettingKey().equals(MdkConstant.PY_FILE_KEY)) {
+ e.setValue(dto.getPyModule() + "." + fileName);
+ }
+ });
+ }else {
+ MethodSettingDTO setting = new MethodSettingDTO();
+ setting.setId(UUID.randomUUID().toString());
+ setting.setMethodId(method.getId());
+ setting.setSettingKey(MdkConstant.PY_FILE_KEY);
+ setting.setValue(dto.getPyModule() + "." + fileName);
+ setting.setName("模型路径");
+ setting.setType("input");
+ setting.setValueType("string");
+ methodSettings.add(setting);
}
}
+
+ // 修改预测项pyFile参数
+ mmModelArithSettingsService.updatePyFile(dto.getPyModule(),fileName);
+ // 修改调度项pyFile参数
+ stScheduleModelSettingService.updatePyFile(dto.getPyModule(),fileName);
MpkFileEntity entity = ConvertUtils.sourceToTarget(dto, MpkFileEntity.class);
entity.setUpdater(SecurityFrameworkUtils.getLoginUserId());
@@ -269,27 +287,24 @@
}
List<String> javaFilePaths = new ArrayList<>();
- List<String> cppFilePaths = new ArrayList<>();
//生成java文件
- File javaFile = new File(dirPath.getAbsolutePath() + File.separator + "IAILMDK" + File.separator + entity.getPkgName().replace(".", File.separator) + File.separator + entity.getPyName() + ".java");
+ File javaFile = new File(dirPath.getAbsolutePath() + File.separator + MdkConstant.IAILMDK + File.separator + entity.getPkgName().replace(".", File.separator) + File.separator + entity.getPyName() + ".java");
GenUtils.drawTemplate("abstract.java.vm", dataContext, javaFile);
javaFilePaths.add(javaFile.getAbsolutePath());
//生成Impl.java文件
- File implJavaFile = new File(dirPath.getAbsolutePath() + File.separator + "IAILMDK" + File.separator + entity.getPkgName().replace(".", File.separator) + File.separator + MdkConstant.IMPL + File.separator + entity.getPyName() + "Impl.java");
+ File implJavaFile = new File(dirPath.getAbsolutePath() + File.separator + MdkConstant.IAILMDK + File.separator + entity.getPkgName().replace(".", File.separator) + File.separator + MdkConstant.IMPL + File.separator + entity.getPyName() + "Impl.java");
GenUtils.drawTemplate("impl.java.vm", dataContext, implJavaFile);
javaFilePaths.add(implJavaFile.getAbsolutePath());
//生成.cpp文件
File cppFile = new File(dirPath.getAbsolutePath() + File.separator + MdkConstant.C + File.separator + entity.getPyName() + ".cpp");
GenUtils.drawTemplate("cpp.vm", dataContext, cppFile);
- cppFilePaths.add(cppFile.getAbsolutePath());
//生成Jni.cpp文件
File jniCppFile = new File(dirPath.getAbsolutePath() + File.separator + MdkConstant.C + File.separator + entity.getPyName() + "Jni.cpp");
GenUtils.drawTemplate("Jni.cpp.vm", dataContext, jniCppFile);
- cppFilePaths.add(jniCppFile.getAbsolutePath());
//生成.h文件
File hFile = new File(dirPath.getAbsolutePath() + File.separator + MdkConstant.C + File.separator + entity.getPyName() + ".h");
@@ -309,23 +324,23 @@
//生成dll文件
String dllSavePath = dirPath.getAbsolutePath() + File.separator + MdkConstant.LIBS + File.separator + "IAIL.MDK.Mid.Jni.dll";
- createDllFile(dirPath.getAbsolutePath(),cppFilePaths,dllSavePath);
+ createDllFile(dirPath.getAbsolutePath(),dllSavePath);
//备份dll文件，用于后续运行
String dllBakPath = mpkTenantBakFilePath + File.separator + MdkConstant.DLL + File.separator + entity.getPyName() + ".dll";
FileUtil.mkParentDirs(dllBakPath);
FileUtil.copy(dllSavePath, dllBakPath, true);
//utils + env java文件
- File utilsJavaFile = new File(dirPath.getAbsolutePath() + File.separator + "IAILMDK" + File.separator + "iail" + File.separator + "mdk" + File.separator + "model" + File.separator + "utils" + File.separator + "AlgsUtils.java");
+ File utilsJavaFile = new File(dirPath.getAbsolutePath() + File.separator + MdkConstant.IAILMDK + File.separator + "iail" + File.separator + "mdk" + File.separator + "model" + File.separator + "utils" + File.separator + "AlgsUtils.java");
FileUtil.mkParentDirs(utilsJavaFile);
FileUtil.copy(mpkResources + File.separator +"IAILMDK/utils/AlgsUtils.java", utilsJavaFile.getAbsolutePath(), true);
javaFilePaths.add(utilsJavaFile.getAbsolutePath());
- File envJavaFile = new File(dirPath.getAbsolutePath() + File.separator + "IAILMDK" + File.separator + "iail" + File.separator + "mdk" + File.separator + "model" + File.separator + "common" + File.separator + "Environment.java");
+ File envJavaFile = new File(dirPath.getAbsolutePath() + File.separator + MdkConstant.IAILMDK + File.separator + "iail" + File.separator + "mdk" + File.separator + "model" + File.separator + "common" + File.separator + "Environment.java");
FileUtil.mkParentDirs(envJavaFile);
FileUtil.copy(mpkResources + File.separator +"IAILMDK/common/Environment.java", envJavaFile.getAbsolutePath(), true);
javaFilePaths.add(envJavaFile.getAbsolutePath());
// 生成class文件
- createClassFile(javaFilePaths);
+ createClassFile(dirPath.getAbsolutePath(),javaFilePaths);
// 删除java源文件
deleteJavaFile(javaFilePaths);
// 打jar包
@@ -360,12 +375,12 @@
@Override
@DSTransactional(rollbackFor = Exception.class)
- public byte[] packageModel(List<String> ids, String projectId, String projectName, String zipFileName, String logs, String version) throws IOException, InterruptedException {
+ public byte[] packageModel(String projectId, String projectName, String zipFileName, String logs, String version) throws IOException {
Long tenantId = TenantContextHolder.getTenantId();
// 备份文件 租户隔离
String mpkTenantBakFilePath = mpkBakFilePath + File.separator + tenantId;
- List<MpkFileDTO> entities = baseDao.selectByIds(ids);
+ List<MpkFileDTO> entities = baseDao.selectByProjectId(projectId);
//模板数据
// Map<String, Object> map = new HashMap<>();
@@ -394,7 +409,6 @@
GenUtils.drawTemplate("menu.xml.vm", new VelocityContext(map1), xmlFile);
List<String> javaFilePaths = new ArrayList<>();
- List<String> cppFilePaths = new ArrayList<>();
//生成java文件
for (MpkFileDTO entity : entities) {
@@ -406,24 +420,22 @@
data.put("pyModule", entity.getPyModule());
VelocityContext dataContext = new VelocityContext(data);
//生成java文件
- File javaFile = new File(dirPath.getAbsolutePath() + File.separator + "IAILMDK" + File.separator + entity.getPkgName().replace(".", File.separator) + File.separator + entity.getPyName() + ".java");
+ File javaFile = new File(dirPath.getAbsolutePath() + File.separator + MdkConstant.IAILMDK + File.separator + entity.getPkgName().replace(".", File.separator) + File.separator + entity.getPyName() + ".java");
GenUtils.drawTemplate("abstract.java.vm", dataContext, javaFile);
javaFilePaths.add(javaFile.getAbsolutePath());
//生成Impl.java文件
- File implJavaFile = new File(dirPath.getAbsolutePath() + File.separator + "IAILMDK" + File.separator + entity.getPkgName().replace(".", File.separator) + File.separator + MdkConstant.IMPL + File.separator + entity.getPyName() + "Impl.java");
+ File implJavaFile = new File(dirPath.getAbsolutePath() + File.separator + MdkConstant.IAILMDK + File.separator + entity.getPkgName().replace(".", File.separator) + File.separator + MdkConstant.IMPL + File.separator + entity.getPyName() + "Impl.java");
GenUtils.drawTemplate("impl.java.vm", dataContext, implJavaFile);
javaFilePaths.add(implJavaFile.getAbsolutePath());
//生成.cpp文件
File cppFile = new File(dirPath.getAbsolutePath() + File.separator + MdkConstant.C + File.separator + entity.getPyName() + ".cpp");
GenUtils.drawTemplate("cpp.vm", dataContext, cppFile);
- cppFilePaths.add(cppFile.getAbsolutePath());
//生成Jni.cpp文件
File jniCppFile = new File(dirPath.getAbsolutePath() + File.separator + MdkConstant.C + File.separator + entity.getPyName() + "Jni.cpp");
GenUtils.drawTemplate("Jni.cpp.vm", dataContext, jniCppFile);
- cppFilePaths.add(jniCppFile.getAbsolutePath());
//生成.h文件
File hFile = new File(dirPath.getAbsolutePath() + File.separator + MdkConstant.C + File.separator + entity.getPyName() + ".h");
@@ -444,7 +456,7 @@
//生成dll文件
String dllSavePath = dirPath.getAbsolutePath() + File.separator + MdkConstant.LIBS + File.separator + "IAIL.MDK.Mid.Jni.dll";
- createDllFile(dirPath.getAbsolutePath(),cppFilePaths,dllSavePath);
+ createDllFile(dirPath.getAbsolutePath(),dllSavePath);
// 打包历史id
String historyId = UUID.randomUUID().toString();
//备份dll，发布时使用
@@ -457,16 +469,16 @@
}
//utils + env java文件
- File utilsJavaFile = new File(dirPath.getAbsolutePath() + File.separator + "IAILMDK" + File.separator + "iail" + File.separator + "mdk" + File.separator + "model" + File.separator + "utils" + File.separator + "AlgsUtils.java");
+ File utilsJavaFile = new File(dirPath.getAbsolutePath() + File.separator + MdkConstant.IAILMDK + File.separator + "iail" + File.separator + "mdk" + File.separator + "model" + File.separator + "utils" + File.separator + "AlgsUtils.java");
FileUtil.mkParentDirs(utilsJavaFile);
FileUtil.copy(mpkResources + File.separator +"IAILMDK/utils/AlgsUtils.java", utilsJavaFile.getAbsolutePath(), true);
javaFilePaths.add(utilsJavaFile.getAbsolutePath());
- File envJavaFile = new File(dirPath.getAbsolutePath() + File.separator + "IAILMDK" + File.separator + "iail" + File.separator + "mdk" + File.separator + "model" + File.separator + "common" + File.separator + "Environment.java");
+ File envJavaFile = new File(dirPath.getAbsolutePath() + File.separator + MdkConstant.IAILMDK + File.separator + "iail" + File.separator + "mdk" + File.separator + "model" + File.separator + "common" + File.separator + "Environment.java");
FileUtil.mkParentDirs(envJavaFile);
FileUtil.copy(mpkResources + File.separator +"IAILMDK/common/Environment.java", envJavaFile.getAbsolutePath(), true);
javaFilePaths.add(envJavaFile.getAbsolutePath());
// 生成class文件
- createClassFile(javaFilePaths);
+ createClassFile(dirPath.getAbsolutePath(),javaFilePaths);
// 删除java源文件
deleteJavaFile(javaFilePaths);
// 打jar包
@@ -522,23 +534,10 @@
return bytes;
}
- private void createDllFile(String dirPath, List<String> cppFilePaths, String dllSavePath) {
+ private void createDllFile(String dirPath, String dllSavePath) {
try {
- String paths = cppFilePaths.stream().collect(Collectors.joining(" "));
- String command = "cl /LD " + paths + " /EHsc /o " + dllSavePath + " /link " + "IAIL.MDK.Mid.Windows.lib";
- log.info("执行cmd命令生成dll：" + command);
- ProcessBuilder builder = new ProcessBuilder("cmd", "/c", command);
- builder.directory(new File(dirPath + File.separator + MdkConstant.LIBS));
- Process process = builder.start();
- // 获取命令输出结果
- InputStream inputStream = process.getInputStream();
- BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, "GBK")); // 设置编码为GBK
- String line;
- while ((line = reader.readLine()) != null) {
- log.info(line);
- }
- // 等待命令执行完成
- process.waitFor();
+ String command = "cl /LD *.cpp /EHsc /o " + dllSavePath + " /link " + "IAIL.MDK.Mid.Windows.lib";
+ CmdUtils.exec(command,dirPath + File.separator + MdkConstant.C);
} catch (Exception e) {
throw new RuntimeException("执行cmd命令生成dll异常",e);
}
@@ -604,19 +603,7 @@
try {
String command = "python setup.py build_ext --inplace";
log.info("执行cmd命令编译pyd：" + command);
- ProcessBuilder builder = new ProcessBuilder("cmd", "/c", command);
- builder.directory(new File(dir)); // 设置工作目录
- builder.redirectErrorStream(true); // 将错误输出和标准输出合并
- Process process = builder.start();
- // 获取命令输出结果
- InputStream inputStream = process.getInputStream();
- BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, "GBK")); // 设置编码为GBK
- String line;
- while ((line = reader.readLine()) != null) {
- log.info(line);
- }
- // 等待命令执行完成
- process.waitFor();
+ CmdUtils.exec(command,dir);
} catch (Exception e) {
throw new RuntimeException("执行cmd命令生成dll异常", e);
}
@@ -708,12 +695,10 @@
StringBuilder sb = new StringBuilder();
sb.append("jar -cvf");
sb.append(" ").append(jarSavePath);
- sb.append(" -C ").append(dirPath).append(File.separator).append("IAILMDK").append(File.separator).append(" .");
- log.info("执行cmd命令打jar包：" + sb);
- Process process = RuntimeUtil.exec(sb.toString());
- process.waitFor();
+ sb.append(" -C ").append(dirPath).append(File.separator).append(MdkConstant.IAILMDK).append(File.separator).append(" .");
+ CmdUtils.exec(sb.toString());
return jarSavePath;
- } catch (InterruptedException e) {
+ } catch (Exception e) {
throw new RuntimeException("执行cmd命令打jar包异常",e);
}
}
@@ -724,17 +709,23 @@
}
}
- private void createClassFile(List<String> javaFilePaths){
+ private void createClassFile(String dirPath,List<String> javaFilePaths){
try {
+ Set<String> pathSet = javaFilePaths.stream().map(path -> {
+ // 替换为相对路径
+ path = path.replace(dirPath + File.separator + MdkConstant.IAILMDK, ".");
+ // 删除文件名，替换为通配符*
+ path = path.substring(0, path.lastIndexOf("\\") + 1) + "*.java";
+ return path;
+ }).collect(Collectors.toSet());
+
StringBuilder sb = new StringBuilder();
sb.append("javac -encoding utf-8");
- for (String path : javaFilePaths) {
+ for (String path : pathSet) {
sb.append(" ").append(path);
}
- log.info("执行cmd命令生成class：" + sb);
- Process process = RuntimeUtil.exec(sb.toString());
- process.waitFor();
- } catch (InterruptedException e) {
+ CmdUtils.exec(sb.toString(),dirPath + File.separator + MdkConstant.IAILMDK);
+ } catch (Exception e) {
throw new RuntimeException("执行cmd命令生成class异常",e);
}
}
--
Gitblit v1.9.3