iailab-module-data/iailab-module-data-biz/src/main/java/com/iailab/module/data/channel/http/collector/ihdb/HttpCollectorForIhd.java

@@ -6,18 +6,13 @@
import com.google.gson.Gson;
import com.iailab.framework.common.constant.CommonConstant;
import com.iailab.module.data.channel.http.entity.HttpApiEntity;
import com.iailab.module.data.channel.http.entity.HttpTagEntity;
import com.iailab.module.data.channel.http.service.HttpApiService;
import com.iailab.module.data.channel.http.service.HttpTagService;
import com.iailab.module.data.common.enums.DataSourceType;
import com.iailab.module.data.common.utils.DateUtils;
import com.iailab.module.data.common.utils.HttpRequest;
import com.iailab.module.data.common.utils.TagUtils;
import com.iailab.module.data.influxdb.pojo.InfluxPointValueBoolPOJO;
import com.iailab.module.data.influxdb.pojo.InfluxPointValueDigPOJO;
import com.iailab.module.data.influxdb.pojo.InfluxPointValueSimPOJO;
import com.iailab.module.data.influxdb.pojo.InfluxPointValueStrPOJO;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.BoundHashOperations;
import org.springframework.data.redis.core.RedisTemplate;
@@ -26,8 +21,14 @@

import java.math.BigDecimal;
import java.math.RoundingMode;
import java.sql.Timestamp;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.LocalDateTime;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.*;
import java.util.function.Function;
import java.util.stream.Collectors;

/**
 * iHyperDB采集
@@ -39,7 +40,7 @@
@Slf4j
@Component
public class HttpCollectorForIhd {
    private Map<String, HttpApiEntity> apiMap = new HashMap<>();
    private static Map<String, HttpApiEntity> apiMap = new HashMap<>();

    @Autowired
    private HttpApiService httpApiService;
@@ -47,12 +48,43 @@
    @Autowired
    private RedisTemplate redisTemplate;

    @Autowired
    private HttpTagService httpTagService;
    ThreadPoolExecutor threadPool = new ThreadPoolExecutor(18, 36, 30, TimeUnit.SECONDS,
            new ArrayBlockingQueue<Runnable>(36), new ThreadPoolExecutor.AbortPolicy());

    private static final String STA_TRUE = "true";

    private static final int GROUP_MAX_COUNT = 50;
    private static final int GROUP_MAX_COUNT = 300;

    private static final int MAX_WAIT = 30;

    private static final String pattern = "yyyyMMddHHmm00";

    private static final String IS_SUCCESS = "isSuccess";

    /**
     * tagName
     */
    private static final String N = "n";

    /**
     * dimension
     */
    private static final String D = "d";

    /**
     * 类型
     */
    private static final String P = "p";

    /**
     * dataValue
     */
    private static final String V = "v";

    /**
     * dataTime
     */
    private static final String T = "t";

    private HttpApiEntity getHttpApi(String id) {
        if (apiMap.containsKey(id)) {
@@ -69,30 +101,30 @@
        StringBuilder tagSb = new StringBuilder();
        tagSb.append("[");
        Map<String, Object> queryParams = new HashMap<>();
        queryParams.put("datatype", valueType);
        queryParams.put("dimension", dimension);
        queryParams.put("tagname", tagNo);
        queryParams.put(P, valueType);
        queryParams.put(D, dimension);
        queryParams.put(N, tagNo);
        String jsonString = JSON.toJSONString(queryParams);
        tagSb.append(jsonString);
        tagSb.append("]");
        log.info("body=====" + tagSb.toString());
        String currentDate = DateUtils.format(new Date(), "yyyyMMddHHmm00");
        String currentDate = DateUtils.format(new Date(), pattern);
        String responseStr = HttpRequest.sendPost(httpApi.getUrl() + "/" + currentDate, tagSb.toString());
        JSONObject responseObj = JSON.parseObject(responseStr);
        if (STA_TRUE.equals(responseObj.get("isSuccess").toString())) {
        if (STA_TRUE.equals(responseObj.get(IS_SUCCESS).toString())) {
            JSONArray tagValueList = responseObj.getJSONArray("data");
            if (!CollectionUtils.isEmpty(tagValueList)) {
                for (int i = 0; i < tagValueList.size(); i++) {
                    JSONObject item = tagValueList.getJSONObject(i);
                    value = new BigDecimal(item.get("value").toString());
                    value = new BigDecimal(item.get(V).toString());
                }
            }
        }
        return value;
    }

    public Map<String, Object> getLastValues(List<String> tagNames) {
    public Map<String, Object> getLastValues(String sourceId, List<String> tagNames) {
        Map<String, Object> result = new HashMap<>();
        HttpApiEntity httpApi = this.getHttpApi(sourceId);
        try {
            if (CollectionUtils.isEmpty(tagNames)) {
                return result;
@@ -101,8 +133,8 @@
            for (int i = 0; i < tagNames.size(); i++) {
                //先查缓存
                BoundHashOperations<String, String, Object> ops = redisTemplate.boundHashOps(tagNames.get(i));
                if (ops.get("value") != null) {
                    BigDecimal value = new BigDecimal(ops.get("value").toString());
                if (ops.get(V) != null) {
                    BigDecimal value = new BigDecimal(ops.get(V).toString());
                    result.put(tagNames.get(i), value.setScale(3, RoundingMode.HALF_UP));
                } else {
                    noCacheTagNames.add(tagNames.get(i));
@@ -116,26 +148,26 @@
            Gson gson = new Gson();
            String tagSb = gson.toJson(noCacheTagNames);
            log.info("body=====" + tagSb);
            String currentDate = DateUtils.format(new Date(), "yyyyMMddHHmm00");
            String currentDate = DateUtils.format(new Date(), pattern);
            String responseStr = "";
            responseStr = HttpRequest.sendPost("http://172.16.59.105:9082/api/IHD/getPointslast" + "/" + currentDate, tagSb);
            responseStr = HttpRequest.sendPost(httpApi.getUrl().replace("getPointdatasAvg", "getPointslast") + "/" + currentDate, tagSb);
            JSONObject responseObj = JSON.parseObject(responseStr);
            if (STA_TRUE.equals(responseObj.get("isSuccess").toString())) {
            if (STA_TRUE.equals(responseObj.get(IS_SUCCESS).toString())) {
                JSONArray tagValueList = responseObj.getJSONArray("data");
                if (!CollectionUtils.isEmpty(tagValueList)) {
                    for (int i = 0; i < tagValueList.size(); i++) {
                        JSONObject item = tagValueList.getJSONObject(i);
                        if (item.get("value") != null) {
                        if (item.get(V) != null) {
                            //存缓存
                            BoundHashOperations<String, String, Object> ops = redisTemplate.boundHashOps(item.get("tagname").toString());
                            ops.put("value", item.get("value").toString());
                            BoundHashOperations<String, String, Object> ops = redisTemplate.boundHashOps(item.get(V).toString());
                            ops.put(V, item.get(V).toString());
                            //设置过期时间
                            redisTemplate.expire(item.get("tagname").toString(), 10, TimeUnit.SECONDS);
                            redisTemplate.expire(item.get(N).toString(), 10, TimeUnit.SECONDS);
                            //把查询到的数据插入结果集
                            BigDecimal value = new BigDecimal(item.get("value").toString());
                            result.put(item.get("tagname").toString(), value.setScale(3, RoundingMode.HALF_UP));
                            BigDecimal value = new BigDecimal(item.get(V).toString());
                            result.put(item.get(N).toString(), value.setScale(3, RoundingMode.HALF_UP));
                        } else {
                            result.put(item.get("tagname").toString(), CommonConstant.BAD_VALUE);
                            result.put(item.get(N).toString(), CommonConstant.BAD_VALUE);
                        }
                    }
                }
@@ -149,42 +181,127 @@
        return result;
    }

    public Map<String, Object> getTagValues(List<Object[]> params) {
    public Map<String, Object> getTagValues(List<Object[]> params, Date collectTime) {
        Map<String, Object> result = new HashMap<>();
        if (CollectionUtils.isEmpty(params)) {
            return new HashMap<>();
        }
        try {
            Map<Integer, List<Object[]>> measurePointsCountGroup = new HashMap<>();
            int pointListSize = params.size();
            int groupCount = pointListSize / GROUP_MAX_COUNT + ((pointListSize % GROUP_MAX_COUNT) > 0 ? 1 : 0);
            log.info("groupCount=" + groupCount);
            for (int i = 0; i < groupCount; i++) {
                int end = (i + 1) * GROUP_MAX_COUNT;
                if (end > pointListSize) {
                    end = pointListSize;
                }
                measurePointsCountGroup.put(i, params.subList(i * GROUP_MAX_COUNT, end));
            }
            log.info("measurePointsCountGroup.size()=" + measurePointsCountGroup.size());
            result = new ConcurrentHashMap<>(params.size());
            CountDownLatch countDownLatch = new CountDownLatch(measurePointsCountGroup.size());
            for (Map.Entry<Integer, List<Object[]>> measurePointsItem : measurePointsCountGroup.entrySet()) {
                HttpApiEntity httpApi = this.getHttpApi(measurePointsItem.getValue().get(0)[0].toString());
                // 并发
                Thread.sleep(1000);
                threadPool.submit(new Task(httpApi.getUrl(), httpApi.getCode(), result, measurePointsItem.getValue(),
                        collectTime, countDownLatch));
                // 顺序
                //this.getByHtp(result, measurePointsItem.getValue(), collectTime);
            }
            countDownLatch.await(MAX_WAIT, TimeUnit.SECONDS);

        Map<Integer, List<Object[]>> measurePointsCountGroup = new HashMap<>();
        int pointListSize = params.size();
        int groupCount = pointListSize / GROUP_MAX_COUNT + ((pointListSize % GROUP_MAX_COUNT) > 0 ? 1 : 0);
        log.info("groupCount=" + groupCount);
        for (int i = 0; i < groupCount; i++) {
            int end = (i + 1) * GROUP_MAX_COUNT;
            if (end > pointListSize) {
                end = pointListSize;
            }
            measurePointsCountGroup.put(i, params.subList(i * GROUP_MAX_COUNT, end));
        }
        Map<String, Object> result = new HashMap<>(params.size());
        for (Map.Entry<Integer, List<Object[]>> measurePointsItem : measurePointsCountGroup.entrySet()) {
            try {
                getByHtp(result, measurePointsItem.getValue());
            } catch (Exception ex) {
                ex.printStackTrace();
            }
        } catch (Exception ex) {
            ex.printStackTrace();
        }
        return result;
    }

    private void getByHtp(Map<String, Object> result, List<Object[]> params) {
        HttpApiEntity httpApi = this.getHttpApi(params.get(0)[0].toString());
    /**
     * 异步采集任务
     */
    private class Task implements Runnable {
        String url;
        String sourceName;
        Date collectTime;
        Map<String, Object> result;
        List<Object[]> params;
        CountDownLatch countDownLatch;

        public Task(String url, String sourceName, Map<String, Object> result, List<Object[]> params,
                    Date collectTime, CountDownLatch countDownLatch) {
            this.url = url;
            this.sourceName = sourceName;
            this.result = result;
            this.collectTime = collectTime;
            this.params = params;
            this.countDownLatch = countDownLatch;
        }

        @Override
        public void run() {
            try {
                log.info("请求的Tag数量：" + params.size());
                this.getByHtp(url, sourceName, result, params, collectTime);
                log.info("请求结束：url=" + url);
            } catch (Exception ex) {
                log.info("获取采集值失败," + ex.getMessage());
                ex.printStackTrace();
            } finally {
                countDownLatch.countDown();
            }
        }

        /**
         * getTagDataByHttp
         *
         * @param url
         * @param sourceName
         * @param result
         * @param params
         * @param collectTime
         */
        private void getByHtp(String url, String sourceName, Map<String, Object> result, List<Object[]> params, Date collectTime) {
            StringBuilder tagSb = new StringBuilder();
            tagSb.append("[");
            for (int i = 0; i < params.size(); i++) {
                Map<String, Object> queryParams = new HashMap<>(3);
                queryParams.put(N, params.get(i)[1]);
                queryParams.put(D, params.get(i)[2]);
                queryParams.put(P, params.get(i)[3]);
                String jsonString = JSON.toJSONString(queryParams);
                tagSb.append(jsonString);
                if (i < params.size() - 1) {
                    tagSb.append(",");
                }
            }
            tagSb.append("]");
            log.info("body=====" + tagSb);
            String currentDate = DateUtils.format(collectTime, pattern);
            String responseStr = HttpRequest.sendPost(url + "/" + currentDate, tagSb.toString());
            JSONObject responseObj = JSON.parseObject(responseStr);
            log.info("responseObj=====" + responseObj.toJSONString());
            if (STA_TRUE.equals(responseObj.get(IS_SUCCESS).toString())) {
                JSONArray tagValueList = responseObj.getJSONArray("data");
                if (!CollectionUtils.isEmpty(tagValueList)) {
                    for (int i = 0; i < tagValueList.size(); i++) {
                        JSONObject item = tagValueList.getJSONObject(i);
                        result.put(TagUtils.genTagId(DataSourceType.HTTP.getCode(), sourceName, item.get(N).toString()), item.get(V));
                    }
                }
            }
        }
    }

    private void getByHtp(String url, String sourceName, Map<String, Object> result, List<Object[]> params, Date collectTime) {
        StringBuilder tagSb = new StringBuilder();
        tagSb.append("[");
        for (int i = 0; i < params.size(); i++) {
            Map<String, Object> queryParams = new HashMap<>();
            queryParams.put("tagname", params.get(i)[1]);
            queryParams.put("dimension", params.get(i)[2]);
            queryParams.put("datatype", params.get(i)[3]);
            queryParams.put(N, params.get(i)[1]);
            queryParams.put(D, params.get(i)[2]);
            queryParams.put(P, params.get(i)[3]);
            String jsonString = JSON.toJSONString(queryParams);
            tagSb.append(jsonString);
            if (i < params.size() - 1) {
@@ -192,17 +309,17 @@
            }
        }
        tagSb.append("]");
        log.info("body=====" + tagSb.toString());
        String currentDate = DateUtils.format(new Date(), "yyyyMMddHHmm00");
        String responseStr = HttpRequest.sendPost(httpApi.getUrl() + "/" + currentDate, tagSb.toString());
        log.info("body=====" + tagSb);
        String currentDate = DateUtils.format(collectTime, pattern);
        String responseStr = HttpRequest.sendPost(url + "/" + currentDate, tagSb.toString());
        JSONObject responseObj = JSON.parseObject(responseStr);
        log.info("responseObj=====" + responseObj.toJSONString());
        if (STA_TRUE.equals(responseObj.get("isSuccess").toString())) {
        if (STA_TRUE.equals(responseObj.get(IS_SUCCESS).toString())) {
            JSONArray tagValueList = responseObj.getJSONArray("data");
            if (!CollectionUtils.isEmpty(tagValueList)) {
                for (int i = 0; i < tagValueList.size(); i++) {
                    JSONObject item = tagValueList.getJSONObject(i);
                    result.put(TagUtils.genTagId(DataSourceType.HTTP.getCode(), httpApi.getCode(), item.get("tagname").toString()), item.get("value"));
                    result.put(TagUtils.genTagId(DataSourceType.HTTP.getCode(), sourceName, item.get(N).toString()), item.get(V));
                }
            }
        }