iailab-module-model/iailab-module-model-biz/src/main/java/com/iailab/module/model/mpk/controller/admin/MdkController.java

@@ -1,25 +1,42 @@
package com.iailab.module.model.mpk.controller.admin;

import cn.hutool.cache.CacheUtil;
import cn.hutool.cache.impl.FIFOCache;
import cn.hutool.core.io.FileUtil;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.iail.IAILMDK;
import com.iail.bean.FieldSet;
import com.iail.bean.Property;
import com.iail.bean.SelectItem;
import com.iail.model.IAILModel;
import com.iail.utils.RSAUtils;
import com.iailab.framework.common.exception.enums.GlobalErrorCodeConstants;
import com.iailab.framework.common.pojo.CommonResult;
import com.iailab.framework.tenant.core.context.TenantContextHolder;
import com.iailab.module.model.mpk.common.MdkConstant;
import com.iailab.module.model.mpk.common.utils.DllUtils;
import com.iailab.module.model.mpk.common.utils.Readtxt;
import com.iailab.module.model.mpk.dto.MdkDTO;
import com.iailab.module.model.mpk.dto.MdkRunDTO;
import com.iailab.module.model.mpk.dto.MethodSettingDTO;
import io.swagger.v3.oas.annotations.Operation;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.IOUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.util.CollectionUtils;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;

import javax.servlet.http.HttpServletResponse;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Method;
import java.net.URLClassLoader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.net.URLEncoder;
import java.nio.file.Files;
import java.util.*;
import java.util.stream.Collectors;

import static com.iailab.framework.common.pojo.CommonResult.error;
import static com.iailab.framework.common.pojo.CommonResult.success;
@@ -36,17 +53,28 @@
    @Value("${mpk.bak-file-path}")
    private String mpkBakFilePath;

    // 先进先出缓存 临时保存导入的数据
    private static FIFOCache<String, String> cache = CacheUtil.newFIFOCache(100);

    @PostMapping("run")
    public CommonResult<String> run(@RequestBody MdkDTO dto) {
    /**
     * @description: 模型测试运行
     * @author: dzd
     * @date: 2024/10/14 15:26
     **/
    @PostMapping("test")
    public CommonResult<String> test(@RequestBody MdkDTO dto) {
        Long tenantId = TenantContextHolder.getTenantId();
        // 备份文件 租户隔离
        String mpkTenantBakFilePath = mpkBakFilePath + File.separator + tenantId;

        Class<?> clazz;
        URLClassLoader classLoader;
        try {
            File jarFile = new File(mpkBakFilePath + File.separator + MdkConstant.JAR + File.separator + dto.getPyName() + ".jar");
            File jarFile = new File(mpkTenantBakFilePath + File.separator + MdkConstant.JAR + File.separator + dto.getPyName() + ".jar");
            if (!jarFile.exists()) {
                throw new RuntimeException("jar包不存在，请先生成代码。jarPath:" + jarFile.getAbsolutePath());
            }
            File dllFile = new File(mpkBakFilePath + File.separator + MdkConstant.DLL + File.separator + dto.getPyName() + ".dll");
            File dllFile = new File(mpkTenantBakFilePath + File.separator + MdkConstant.DLL + File.separator + dto.getPyName() + ".dll");
            if (!dllFile.exists()) {
                throw new RuntimeException("dll文件不存在，请先生成代码。dllPath:" + dllFile.getAbsolutePath());
            }
@@ -63,16 +91,19 @@

        System.out.println("runTime=" + System.currentTimeMillis());
        try {
            List<String> datas = dto.getDatas();
            List<String> uuids = dto.getUuids();

            int paramLength = dto.getHasModel() ? datas.size() + 2 : datas.size() + 1;
            int paramLength = dto.getHasModel() ? uuids.size() + 2 : uuids.size() + 1;
            Object[] paramsValueArray = new Object[paramLength];
            Class<?>[] paramsArray = new Class[paramLength];

            try {
                for (int i = 0; i < datas.size(); i++) {
                    String json = datas.get(i);
                    JSONArray jsonArray = JSON.parseArray(json);
                for (int i = 0; i < uuids.size(); i++) {
                    String uuid = uuids.get(i);
                    if (!cache.containsKey(uuid)) {
                        return error(GlobalErrorCodeConstants.BAD_REQUEST.getCode(),"请重新导入模型参数");
                    }
                    JSONArray jsonArray = JSON.parseArray(cache.get(uuid));
                    double[][] data = new double[jsonArray.size()][jsonArray.getJSONArray(0).size()];
                    for (int j = 0; j < jsonArray.size(); j++) {
                        for (int k = 0; k < jsonArray.getJSONArray(j).size(); k++) {
@@ -84,17 +115,26 @@
                }
            } catch (Exception e) {
                e.printStackTrace();
                return error(GlobalErrorCodeConstants.BAD_REQUEST.getCode(),"参数错误，请检查！");
                return error(GlobalErrorCodeConstants.BAD_REQUEST.getCode(),"模型参数错误，请检查！");
            }

            if (dto.getHasModel()) {
                paramsValueArray[datas.size()] = dto.getModel();
                paramsValueArray[datas.size() + 1] = dto.getModelSettings();
                paramsArray[datas.size()] = HashMap.class;
                paramsArray[datas.size() + 1] = HashMap.class;
            }else {
                paramsValueArray[datas.size()] = dto.getModelSettings();
                paramsArray[datas.size()] = HashMap.class;
            try {
                if (dto.getModelSettings().stream().noneMatch(e -> e.getSettingKey().equals(MdkConstant.PY_FILE_KEY))) {
                    return error(GlobalErrorCodeConstants.BAD_REQUEST.getCode(),"模型设置参数缺少必要信息【" + MdkConstant.PY_FILE_KEY +  "】,请重新上传模型！");
                }

                if (dto.getHasModel()) {
                    paramsValueArray[uuids.size()] = dto.getModel();
                    paramsValueArray[uuids.size() + 1] = handleModelSettings(dto.getModelSettings());
                    paramsArray[uuids.size()] = HashMap.class;
                    paramsArray[uuids.size() + 1] = HashMap.class;
                }else {
                    paramsValueArray[uuids.size()] = handleModelSettings(dto.getModelSettings());
                    paramsArray[uuids.size()] = HashMap.class;
                }
            } catch (Exception e) {
                e.printStackTrace();
                return error(GlobalErrorCodeConstants.BAD_REQUEST.getCode(),"模型设置错误，请检查！");
            }

            HashMap result = (HashMap) clazz.getDeclaredMethod(dto.getMethodName(), paramsArray).invoke(clazz.newInstance(), paramsValueArray);
@@ -104,22 +144,196 @@
            return error(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"运行异常");
        } finally {
            if (classLoader != null) {
                DllUtils.uploadDll(classLoader);
                DllUtils.uploadJar(classLoader);
                DllUtils.unloadDll(classLoader);
                DllUtils.unloadJar(classLoader);
            }
            System.gc();
        }
    }

    private IAILModel createModelBean(MdkDTO dto) {
        IAILModel modelBean = new IAILModel();

        //ParamPathList
        List<String> paramPathList = new ArrayList<>();
        List<String> paramNameList = new ArrayList<>();

        for (Map.Entry<String, Object> entry : dto.getModel().entrySet()) {
            paramNameList.add(entry.getKey());
            paramPathList.add(entry.getValue().toString());
        }
        modelBean.setParamNameList(paramNameList);
        modelBean.setParamPathList(paramPathList);
        //ClassName MethodName
        modelBean.setClassName(dto.getClassName());
        modelBean.setMethodName(dto.getMethodName());
        //ParamsArray
        int paramLength = dto.getHasModel() ? dto.getDataLength() + 2 : dto.getDataLength() + 1;
        Class<?>[] paramsArray = new Class[paramLength];

        for (int i = 0; i < dto.getDataLength(); i++) {
            paramsArray[i] = double[][].class;
        }

        if (dto.getHasModel()) {
            paramsArray[dto.getDataLength()] = HashMap.class;
            paramsArray[dto.getDataLength() + 1] = HashMap.class;
        }else {
            paramsArray[dto.getDataLength()] = HashMap.class;
        }
        modelBean.setParamsArray(paramsArray);
        //LoadFieldSetList
        List<FieldSet> loadFieldSetList = new ArrayList<>();
        FieldSet fieldSet = new FieldSet();
        fieldSet.setFieldName("");
        List<Property> propertyList = new ArrayList<>();
        for (MethodSettingDTO modelSetting : dto.getPredModelSettings()) {
            Property property = new Property();
            property.setKey(modelSetting.getSettingKey());
            property.setName(modelSetting.getName());
            property.setType(modelSetting.getType());
            property.setValueType(modelSetting.getValueType());
            property.setMin(modelSetting.getMin() == null ? "" : modelSetting.getMin().toString());
            property.setMax(modelSetting.getMax() == null ? "" : modelSetting.getMax().toString());
            property.setSelectItemList(CollectionUtils.isEmpty(modelSetting.getSettingSelects()) ? null : modelSetting.getSettingSelects().stream().map(e -> new SelectItem(e.getSelectKey(),e.getName())).collect(Collectors.toList()));
            property.setValue(modelSetting.getValue());
            property.setFlow(false);
            propertyList.add(property);
        }
        fieldSet.setPropertyList(propertyList);
        loadFieldSetList.add(fieldSet);
        modelBean.setLoadFieldSetList(loadFieldSetList);
        //SettingConfigMap
        Map<String, Object> settingConfigMap = new HashMap<String, Object>();
        List<com.iail.bean.Value> settingKeyList = new ArrayList<com.iail.bean.Value>();
        Map<String, Object> settingMap = new HashMap<String, Object>();
        for (MethodSettingDTO modelSetting : dto.getModelSettings()) {
            settingKeyList.add(new com.iail.bean.Value(modelSetting.getSettingKey(),modelSetting.getSettingKey()));
            settingConfigMap.put("settingKeyList", settingKeyList);
            settingConfigMap.put("settingMap", handleModelSettings(dto.getModelSettings()));
        }
        modelBean.setSettingConfigMap(settingConfigMap);
        //DataMap
        modelBean.setDataMap(dto.getModelResult());
        //ResultKey
        modelBean.setResultKey(dto.getResultKey());
        //ResultKey
        modelBean.setVersion("1.0.0");


        return modelBean;
    }

    @PostMapping("saveModel")
    public void saveModel(@RequestBody MdkDTO dto, HttpServletResponse response) {
        IAILModel modelBean = createModelBean(dto);

        try {
            //临时文件夹
            File tempFile = null;
            try {
                tempFile = Files.createTempFile(dto.getPyName(),".miail").toFile();
                log.info("生成临时文件，" + tempFile.getAbsolutePath());
            } catch (IOException e) {
                throw new RuntimeException("创建临时文件异常",e);
            }



            try {
                IAILMDK.saveModel(tempFile, modelBean);
            } catch (Exception e) {
                throw new RuntimeException("IAILMDK.saveModel异常",e);
            }

            byte[] data = FileUtil.readBytes(tempFile);
            response.reset();
            response.setHeader("Content-Disposition", "attachment; filename=\"" + URLEncoder.encode(tempFile.getName(), "UTF-8") + "\"");
            response.addHeader("Content-Length", "" + data.length);
            response.setContentType("application/octet-stream; charset=UTF-8");

            IOUtils.write(data, response.getOutputStream());
        } catch (Exception e) {
            throw new RuntimeException("代码生成异常",e);
        }
    }

    private HashMap<String, Object> handleModelSettings(List<MethodSettingDTO> modelSettings) {
        HashMap<String, Object> resultMap = null;
        try {
            resultMap = new HashMap<>(modelSettings.size());
            for (MethodSettingDTO modelSetting : modelSettings) {
                switch (modelSetting.getValueType()) {
                    case "int":
                        resultMap.put(modelSetting.getSettingKey(), Integer.valueOf(modelSetting.getSettingValue()));
                        break;
                    case "string":
                        resultMap.put(modelSetting.getSettingKey(), modelSetting.getSettingValue());
                        break;
                    case "decimal":
                        resultMap.put(modelSetting.getSettingKey(), Double.valueOf(modelSetting.getSettingValue()));
                        break;
                    case "decimalArray":
                        JSONArray jsonArray = JSON.parseArray(modelSetting.getSettingValue());
                        double[] doubles = new double[jsonArray.size()];
                        for (int i = 0; i < jsonArray.size(); i++) {
                            doubles[i] = Double.valueOf(String.valueOf(jsonArray.get(i)));
                        }
                        resultMap.put(modelSetting.getSettingKey(), doubles);
                        break;
                }
            }
        } catch (NumberFormatException e) {
            throw new RuntimeException("模型参数有误，请检查！！！");
        }
        return resultMap;
    }

    /**
     * @description: 模型运行
     * @author: dzd
     * @date: 2024/10/14 15:26
     **/
    @PostMapping("run")
    public CommonResult<String> run(@RequestBody MdkRunDTO dto) {
        if (RSAUtils.checkLisenceBean().getCode() != 1) {
            log.error("Lisence 不可用！");
            return CommonResult.error(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"Lisence 不可用！");
        } else {
            try {
                URLClassLoader classLoader = DllUtils.getClassLoader(dto.getProjectId());
                if (null == classLoader) {
                    return CommonResult.error(GlobalErrorCodeConstants.ERROR_CONFIGURATION.getCode(),"请先发布项目！");
                }

                Class<?> clazz = classLoader.loadClass(dto.getClassName());
                Method method = clazz.getMethod(dto.getMethodName(), dto.getParamsClassArray());
                HashMap invoke = (HashMap) method.invoke(clazz.newInstance(), dto.getParamsValueArray());

                // todo 将结果存入数据库

            } catch (Exception e) {
                log.error("模型运行失败",e);
                return CommonResult.error(GlobalErrorCodeConstants.INTERNAL_SERVER_ERROR.getCode(),"模型运行失败！");
            }
        }
        return CommonResult.success("");
    }

    @PostMapping("/import")
    @Operation(summary = "导入参数")
    public CommonResult<List<String>> importExcel(@RequestParam("file") MultipartFile file) throws Exception {
    public CommonResult<List<HashMap<String,Object>>> importExcel(@RequestParam("file") MultipartFile file) throws Exception {
        List<double[][]> datas = Readtxt.readMethodExcel(file);
        List<String> result = new ArrayList<>();
        List<HashMap<String,Object>> result = new ArrayList<>();
        if (!CollectionUtils.isEmpty(datas)) {
            for (double[][] data : datas) {
                if (data.length > 0) {
                    result.add(JSON.toJSONString(data));
                    HashMap<String,Object> map = new HashMap<>();
                    String uuid = UUID.randomUUID().toString();
                    map.put("uuid",uuid);
                    map.put("data",JSON.toJSONString(data));
                    cache.put(uuid,JSON.toJSONString(data));
                    result.add(map);
                }
            }
        }