From c47a8644f0b9911fcdc7d0fbca3fda1c610c150e Mon Sep 17 00:00:00 2001
From: dengzedong <dengzedong@email>
Date: 星期三, 15 一月 2025 17:37:29 +0800
Subject: [PATCH] point数据采集数据质量判断
---
iailab-module-data/iailab-module-data-biz/src/main/java/com/iailab/module/data/channel/http/collector/ihdb/HttpCollectorForIhd.java | 25 +++++++++++++++++--------
1 files changed, 17 insertions(+), 8 deletions(-)
diff --git a/iailab-module-data/iailab-module-data-biz/src/main/java/com/iailab/module/data/channel/http/collector/ihdb/HttpCollectorForIhd.java b/iailab-module-data/iailab-module-data-biz/src/main/java/com/iailab/module/data/channel/http/collector/ihdb/HttpCollectorForIhd.java
index ce58758..ca01dca 100644
--- a/iailab-module-data/iailab-module-data-biz/src/main/java/com/iailab/module/data/channel/http/collector/ihdb/HttpCollectorForIhd.java
+++ b/iailab-module-data/iailab-module-data-biz/src/main/java/com/iailab/module/data/channel/http/collector/ihdb/HttpCollectorForIhd.java
@@ -5,6 +5,7 @@
import com.alibaba.fastjson.JSONObject;
import com.google.gson.Gson;
import com.iailab.framework.common.constant.CommonConstant;
+import com.iailab.framework.common.util.http.HttpUtils;
import com.iailab.module.data.channel.http.entity.HttpApiEntity;
import com.iailab.module.data.channel.http.service.HttpApiService;
import com.iailab.module.data.common.enums.DataSourceType;
@@ -12,6 +13,7 @@
import com.iailab.module.data.common.utils.HttpRequest;
import com.iailab.module.data.common.utils.TagUtils;
import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.BoundHashOperations;
import org.springframework.data.redis.core.RedisTemplate;
@@ -52,7 +54,7 @@
private static final String STA_TRUE = "true";
- private static final int GROUP_MAX_COUNT = 90;
+ private static final int GROUP_MAX_COUNT = 300;
private static final int MAX_WAIT = 30;
@@ -85,6 +87,11 @@
*/
private static final String T = "t";
+ /**
+ * 数据质量G：good,B:bad
+ */
+ private static final String Q = "q";
+
private HttpApiEntity getHttpApi(String id) {
if (apiMap.containsKey(id)) {
return apiMap.get(id);
@@ -107,7 +114,7 @@
tagSb.append(jsonString);
tagSb.append("]");
String currentDate = DateUtils.format(new Date(), pattern);
- String responseStr = HttpRequest.sendPost(httpApi.getUrl() + "/" + currentDate, tagSb.toString());
+ String responseStr = HttpUtils.sendPost(httpApi.getUrl() + "/" + currentDate, tagSb.toString());
JSONObject responseObj = JSON.parseObject(responseStr);
if (STA_TRUE.equals(responseObj.get(IS_SUCCESS).toString())) {
JSONArray tagValueList = responseObj.getJSONArray("data");
@@ -149,7 +156,7 @@
log.info("body=====" + tagSb);
String currentDate = DateUtils.format(new Date(), pattern);
String responseStr = "";
- responseStr = HttpRequest.sendPost(httpApi.getUrl().replace("getPointdatasAvg", "getPointslast") + "/" + currentDate, tagSb);
+ responseStr = HttpUtils.sendPost(httpApi.getUrl().replace("getPointdatasAvg", "getPointslast") + "/" + currentDate, tagSb);
JSONObject responseObj = JSON.parseObject(responseStr);
if (STA_TRUE.equals(responseObj.get(IS_SUCCESS).toString())) {
JSONArray tagValueList = responseObj.getJSONArray("data");
@@ -203,7 +210,7 @@
for (Map.Entry<Integer, List<Object[]>> measurePointsItem : measurePointsCountGroup.entrySet()) {
HttpApiEntity httpApi = this.getHttpApi(measurePointsItem.getValue().get(0)[0].toString());
// 并发
- Thread.sleep(200);
+ Thread.sleep(1000);
threadPool.submit(new Task(httpApi.getUrl(), httpApi.getCode(), result, measurePointsItem.getValue(),
collectTime, countDownLatch));
// 顺序
@@ -265,7 +272,7 @@
StringBuilder tagSb = new StringBuilder();
tagSb.append("[");
for (int i = 0; i < params.size(); i++) {
- Map<String, Object> queryParams = new HashMap<>();
+ Map<String, Object> queryParams = new HashMap<>(3);
queryParams.put(N, params.get(i)[1]);
queryParams.put(D, params.get(i)[2]);
queryParams.put(P, params.get(i)[3]);
@@ -278,7 +285,7 @@
tagSb.append("]");
log.info("body=====" + tagSb);
String currentDate = DateUtils.format(collectTime, pattern);
- String responseStr = HttpRequest.sendPost(url + "/" + currentDate, tagSb.toString());
+ String responseStr = HttpUtils.sendPost(url + "/" + currentDate, tagSb.toString());
JSONObject responseObj = JSON.parseObject(responseStr);
log.info("responseObj=====" + responseObj.toJSONString());
if (STA_TRUE.equals(responseObj.get(IS_SUCCESS).toString())) {
@@ -286,7 +293,9 @@
if (!CollectionUtils.isEmpty(tagValueList)) {
for (int i = 0; i < tagValueList.size(); i++) {
JSONObject item = tagValueList.getJSONObject(i);
- result.put(TagUtils.genTagId(DataSourceType.HTTP.getCode(), sourceName, item.get(N).toString()), item.get(V));
+ if (item.get(Q).toString().equals("G")) {
+ result.put(TagUtils.genTagId(DataSourceType.HTTP.getCode(), sourceName, item.get(N).toString()), item.get(V));
+ }
}
}
}
@@ -310,7 +319,7 @@
tagSb.append("]");
log.info("body=====" + tagSb);
String currentDate = DateUtils.format(collectTime, pattern);
- String responseStr = HttpRequest.sendPost(url + "/" + currentDate, tagSb.toString());
+ String responseStr = HttpUtils.sendPost(url + "/" + currentDate, tagSb.toString());
JSONObject responseObj = JSON.parseObject(responseStr);
log.info("responseObj=====" + responseObj.toJSONString());
if (STA_TRUE.equals(responseObj.get(IS_SUCCESS).toString())) {
--
Gitblit v1.9.3