package com.iailab.module.data.channel.http.collector.asdb; import com.alibaba.fastjson.JSON; import com.iailab.framework.common.constant.CommonConstant; import com.iailab.framework.common.util.http.HttpUtils; import com.iailab.module.data.channel.http.collector.asdb.vo.HttpAsdbRespDataVO; import com.iailab.module.data.channel.http.entity.HttpApiEntity; import com.iailab.module.data.channel.http.service.HttpApiService; import com.iailab.module.data.common.enums.DataSourceType; import com.iailab.module.data.common.utils.TagUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.redis.core.BoundHashOperations; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; import java.math.BigDecimal; import java.math.RoundingMode; import java.util.*; import java.util.concurrent.*; import java.util.stream.Collectors; /** * AnSteelDB采集 * * @author Jay */ @Slf4j @Component public class HttpCollectorForAsdb { private static Map apiMap = new HashMap<>(); @Autowired private HttpApiService httpApiService; @Autowired private RedisTemplate redisTemplate; ThreadPoolExecutor threadPool = new ThreadPoolExecutor(18, 36, 30, TimeUnit.SECONDS, new ArrayBlockingQueue(36), new ThreadPoolExecutor.AbortPolicy()); private static final String STA_TRUE = "true"; private static final int GROUP_MAX_COUNT = 300; private static final int MAX_WAIT = 30; private static final String pattern = "yyyyMMddHHmm00"; private static final String IS_SUCCESS = "isSuccess"; /** * tagName */ private static final String N = "n"; /** * dimension */ private static final String D = "d"; /** * 类型 */ private static final String P = "p"; /** * dataValue */ private static final String V = "v"; /** * dataTime */ private static final String T = "t"; /** * 数据质量G:good,B:bad */ private static final String Q = "q"; private HttpApiEntity getHttpApi(String id) { if (apiMap.containsKey(id)) { return apiMap.get(id); } HttpApiEntity httpApi = httpApiService.info(id); apiMap.put(id, httpApi); return httpApi; } public BigDecimal getTagValue(String sourceId, String tagNo, Integer dimension, String valueType) { BigDecimal value = CommonConstant.BAD_VALUE; HttpApiEntity httpApi = this.getHttpApi(sourceId); String responseStr = HttpUtils.sendGet(httpApi.getUrl(), null, ""); if (responseStr == null) { return value; } List dataList = JSON.parseArray(responseStr, HttpAsdbRespDataVO.class); value = Objects.requireNonNull(dataList.stream().filter(data -> tagNo.equals(data.getPoint())).findFirst().orElse(null)).getValue(); return value; } public Map getLastValues(String sourceId, List tagNames) { Map result = new HashMap<>(); HttpApiEntity httpApi = this.getHttpApi(sourceId); try { if (CollectionUtils.isEmpty(tagNames)) { return result; } List noCacheTagNames = new ArrayList<>();//未缓存的tag for (int i = 0; i < tagNames.size(); i++) { //先查缓存 BoundHashOperations ops = redisTemplate.boundHashOps(tagNames.get(i)); if (ops.get(V) != null) { BigDecimal value = new BigDecimal(ops.get(V).toString()); result.put(tagNames.get(i), value.setScale(3, RoundingMode.HALF_UP)); } else { noCacheTagNames.add(tagNames.get(i)); } } if (CollectionUtils.isEmpty(noCacheTagNames)) { log.info("全部读取缓存"); return result; } log.info("查询未缓存的数据"); String responseStr = HttpUtils.sendGet(httpApi.getUrl(), null, ""); List dataList = JSON.parseArray(responseStr, HttpAsdbRespDataVO.class); List noCacheDataList = dataList.stream().filter(data -> noCacheTagNames.contains(data.getPoint())).collect(Collectors.toList()); for (HttpAsdbRespDataVO data : noCacheDataList ){ if (data.getValue() != null){ //存缓存 BoundHashOperations ops = redisTemplate.boundHashOps(data.getValue().toString()); ops.put(V, data.getValue().toString()); //设置过期时间 redisTemplate.expire(data.getValue().toString(), 10, TimeUnit.SECONDS); //把查询到的数据插入结果集 BigDecimal value = new BigDecimal(data.getValue().toString()); result.put(data.getPoint(), value.setScale(3, RoundingMode.HALF_UP)); } else { result.put(data.getPoint(), CommonConstant.BAD_VALUE); } } } catch (Exception ex) { log.info("getCurrentValue异常"); ex.printStackTrace(); throw ex; } return result; } public Map getTagValues(List params, Date collectTime) { Map result = new HashMap<>(); if (CollectionUtils.isEmpty(params)) { return new HashMap<>(); } try { Map> measurePointsCountGroup = new HashMap<>(); int pointListSize = params.size(); int groupCount = pointListSize / GROUP_MAX_COUNT + ((pointListSize % GROUP_MAX_COUNT) > 0 ? 1 : 0); log.info("groupCount=" + groupCount); for (int i = 0; i < groupCount; i++) { int end = (i + 1) * GROUP_MAX_COUNT; if (end > pointListSize) { end = pointListSize; } measurePointsCountGroup.put(i, params.subList(i * GROUP_MAX_COUNT, end)); } log.info("measurePointsCountGroup.size()=" + measurePointsCountGroup.size()); result = new ConcurrentHashMap<>(params.size()); CountDownLatch countDownLatch = new CountDownLatch(measurePointsCountGroup.size()); for (Map.Entry> measurePointsItem : measurePointsCountGroup.entrySet()) { HttpApiEntity httpApi = this.getHttpApi(measurePointsItem.getValue().get(0)[0].toString()); // 并发 Thread.sleep(1000); threadPool.submit(new Task(httpApi.getUrl(), httpApi.getCode(), result, measurePointsItem.getValue(), collectTime, countDownLatch)); // 顺序 //this.getByHtp(result, measurePointsItem.getValue(), collectTime); } countDownLatch.await(MAX_WAIT, TimeUnit.SECONDS); } catch (Exception ex) { ex.printStackTrace(); } return result; } /** * 异步采集任务 */ private class Task implements Runnable { String url; String sourceName; Date collectTime; Map result; List params; CountDownLatch countDownLatch; public Task(String url, String sourceName, Map result, List params, Date collectTime, CountDownLatch countDownLatch) { this.url = url; this.sourceName = sourceName; this.result = result; this.collectTime = collectTime; this.params = params; this.countDownLatch = countDownLatch; } @Override public void run() { try { log.info("请求的Tag数量:" + params.size()); this.getByHtp(url, sourceName, result, params, collectTime); log.info("请求结束:url=" + url); } catch (Exception ex) { log.info("获取采集值失败," + ex.getMessage()); ex.printStackTrace(); } finally { countDownLatch.countDown(); } } /** * getTagDataByHttp * * @param url * @param sourceName * @param result * @param params * @param collectTime */ private void getByHtp(String url, String sourceName, Map result, List params, Date collectTime) { String responseStr = HttpUtils.sendGet(url, null, ""); List dataList = JSON.parseArray(responseStr, HttpAsdbRespDataVO.class); for (HttpAsdbRespDataVO data : dataList){ result.put(TagUtils.genTagId(DataSourceType.HTTP.getCode(), sourceName, data.getPoint()), data.getValue()); } } } private void getByHtp(String url, String sourceName, Map result, List params, Date collectTime) {String responseStr = HttpUtils.sendGet(url, null, ""); List dataList = JSON.parseArray(responseStr, HttpAsdbRespDataVO.class); for (HttpAsdbRespDataVO data : dataList){ result.put(TagUtils.genTagId(DataSourceType.HTTP.getCode(), sourceName, data.getPoint()), data.getValue()); }} }