|
|
@@ -9,7 +9,9 @@ import cn.com.ctop.job.adq.data.service.IAdqCreativeReportService;
|
|
|
import cn.com.ctop.job.adq.data.utils.Check;
|
|
|
import cn.com.ctop.job.adq.data.utils.FieldsConfig;
|
|
|
import cn.com.ctop.job.adq.data.utils.HttpUtils;
|
|
|
+import cn.com.ctop.job.adq.data.utils.JsonUtils;
|
|
|
import cn.com.ctop.job.adq.utils.RandomStringGenerator;
|
|
|
+import cn.com.ctop.job.adq.utils.doris.sink.DorisMultiTableStreamLoadManager;
|
|
|
import com.alibaba.fastjson.JSONArray;
|
|
|
import com.alibaba.fastjson.JSONObject;
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
@@ -33,14 +35,8 @@ public class AdqCreativeCreativeReportServiceImpl implements IAdqCreativeReportS
|
|
|
@Autowired
|
|
|
private AdqErrorLogMapper adqErrorLogMapper;
|
|
|
|
|
|
-
|
|
|
- /*private static final int MAX_BATCH_SIZE = 5000;
|
|
|
- private static final long MAX_WAIT_TIME_MS = 2 * 60 * 1000;
|
|
|
-
|
|
|
- private final List<AdqCreativeReportHour> creativeReportHourBuffer = Collections.synchronizedList(new ArrayList<>());
|
|
|
- private final List<AdqCreativeReportDaily> creativeReportDailyBuffer = Collections.synchronizedList(new ArrayList<>());
|
|
|
- private long creativeReportHourLastFlushTime = System.currentTimeMillis();
|
|
|
- private long creativeReportDailyLastFlushTime = System.currentTimeMillis();*/
|
|
|
+ @Autowired
|
|
|
+ private DorisMultiTableStreamLoadManager dorisLoadManager;
|
|
|
|
|
|
|
|
|
@Override
|
|
|
@@ -84,6 +80,10 @@ public class AdqCreativeCreativeReportServiceImpl implements IAdqCreativeReportS
|
|
|
hourReport.setAdvertiserId(advertiserId);
|
|
|
hourReport.setStatTimeDay(Long.valueOf(startDate.replace("-", "")));
|
|
|
hourList.add(hourReport);
|
|
|
+
|
|
|
+ JSONObject filterJson = JSONObject.parseObject(JsonUtils.toFilterJSON(hourReport));
|
|
|
+ filterJson.put("video_outer_play100_count",hourReport.getVideoOuterPlay100Count());
|
|
|
+ dorisLoadManager.add("media_api","adq_creative_report_hour", filterJson);
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
@@ -104,9 +104,8 @@ public class AdqCreativeCreativeReportServiceImpl implements IAdqCreativeReportS
|
|
|
log.info("hourList.size:{}", hourList.size());
|
|
|
if (!Check.isNull(hourList)) {
|
|
|
adqReportMapper.addsHourList(hourList);
|
|
|
- adqReportMapper.addsHourListSD(hourList);
|
|
|
+ //adqReportMapper.addsHourListSD(hourList);
|
|
|
log.info("新增数据完成account:id:{}", advertiserId);
|
|
|
- //creativeReportHourBuffer.addAll(hourList);
|
|
|
}
|
|
|
if (hourList.size() >= 1000) {
|
|
|
getAdqCreativeHour(advertiserId, accessToken, startDate, endDate, page + 1, isLog);
|
|
|
@@ -165,6 +164,11 @@ public class AdqCreativeCreativeReportServiceImpl implements IAdqCreativeReportS
|
|
|
hourReport.setAdvertiserId(advertiserId);
|
|
|
hourReport.setStatTimeDay(Long.valueOf(dataJson.getString("date").replace("-", "")));
|
|
|
dailies.add(hourReport);
|
|
|
+
|
|
|
+ JSONObject filterJson = JSONObject.parseObject(JsonUtils.toFilterJSON(hourReport));
|
|
|
+ filterJson.put("video_outer_play100_count",hourReport.getVideoOuterPlay100Count());
|
|
|
+ dorisLoadManager.add("media_api","adq_creative_report_daily", filterJson);
|
|
|
+
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
@@ -184,8 +188,7 @@ public class AdqCreativeCreativeReportServiceImpl implements IAdqCreativeReportS
|
|
|
}
|
|
|
if (!Check.isNull(dailies)) {
|
|
|
adqReportMapper.addsDaily(dailies);
|
|
|
- adqReportMapper.addsDailySD(dailies);
|
|
|
- //creativeReportDailyBuffer.addAll(dailies);
|
|
|
+ //adqReportMapper.addsDailySD(dailies);
|
|
|
}
|
|
|
if (dailies.size() >= 1000) {
|
|
|
getAdqCreativeDaily(advertiserId, accessToken, startDate, endDate, page + 1, isLog);
|
|
|
@@ -206,85 +209,4 @@ public class AdqCreativeCreativeReportServiceImpl implements IAdqCreativeReportS
|
|
|
public List<JSONObject> getAccountListByDate(Integer startDate, Integer endDate) {
|
|
|
return adqErrorLogMapper.getAccountListByDate(startDate, endDate);
|
|
|
}
|
|
|
-
|
|
|
-
|
|
|
- /*@Scheduled(fixedRate = 10000) // 每30秒检查一次
|
|
|
- public void flushIfNeeded() {
|
|
|
- log.info("开始执行创意报表定时批处理任务....");
|
|
|
- synchronized (creativeReportHourBuffer) {
|
|
|
- long now = System.currentTimeMillis();
|
|
|
- // creativeReportHourBuffer.size() >= MAX_BATCH_SIZE || (now - creativeReportHourLastFlushTime) >= MAX_WAIT_TIME_MS &&
|
|
|
- if (!creativeReportHourBuffer.isEmpty()) {
|
|
|
- try {
|
|
|
- long startTime = System.currentTimeMillis();
|
|
|
- adqReportMapper.addsHourListSD(new ArrayList<>(creativeReportHourBuffer));
|
|
|
- long endTime = System.currentTimeMillis();
|
|
|
- long diffInMilliseconds = endTime - startTime;
|
|
|
- double diffInSeconds = diffInMilliseconds / 1000.0;
|
|
|
- DecimalFormat df = new DecimalFormat("#.00");
|
|
|
- String seconds = df.format(diffInSeconds);
|
|
|
- log.info("Flushed creativeReportHour batch size: {},执行时间:{}", creativeReportHourBuffer.size(), seconds);
|
|
|
- creativeReportHourBuffer.clear();
|
|
|
- creativeReportHourLastFlushTime = now;
|
|
|
- } catch (Exception e) {
|
|
|
- log.error("Error flushing creativeReportHour batch", e);
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- synchronized (creativeReportDailyBuffer) {
|
|
|
- long now = System.currentTimeMillis();
|
|
|
- // creativeReportDailyBuffer.size() >= MAX_BATCH_SIZE || (now - creativeReportDailyLastFlushTime) >= MAX_WAIT_TIME_MS &&
|
|
|
- if ( !creativeReportDailyBuffer.isEmpty()) {
|
|
|
- try {
|
|
|
- long startTime = System.currentTimeMillis();
|
|
|
- adqReportMapper.addsDailySD(new ArrayList<>(creativeReportDailyBuffer));
|
|
|
- long endTime = System.currentTimeMillis();
|
|
|
- long diffInMilliseconds = endTime - startTime;
|
|
|
- double diffInSeconds = diffInMilliseconds / 1000.0;
|
|
|
- DecimalFormat df = new DecimalFormat("#.00");
|
|
|
- String seconds = df.format(diffInSeconds);
|
|
|
- log.info("Flushed creativeReportDaily batch size: {},执行时间:{}", creativeReportDailyBuffer.size(),seconds);
|
|
|
-
|
|
|
- creativeReportDailyBuffer.clear();
|
|
|
- creativeReportDailyLastFlushTime = now;
|
|
|
- } catch (Exception e) {
|
|
|
- log.error("Error flushing creativeReportDaily batch", e);
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- @PreDestroy
|
|
|
- public void flushPreDestroy() {
|
|
|
- log.info("项目重启,开始执行创意报表批处理任务....");
|
|
|
- synchronized (creativeReportHourBuffer) {
|
|
|
- if (!creativeReportHourBuffer.isEmpty()) {
|
|
|
- try {
|
|
|
- log.info("Flushed advertiserReportHour batch size: {}", creativeReportHourBuffer.size());
|
|
|
- adqReportMapper.addsHourListSD(new ArrayList<>(creativeReportHourBuffer));
|
|
|
- creativeReportHourBuffer.clear();
|
|
|
- } catch (Exception e) {
|
|
|
- log.error("Error flushing advertiserReportHour batch", e);
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- synchronized (creativeReportDailyBuffer) {
|
|
|
- if (!creativeReportDailyBuffer.isEmpty()) {
|
|
|
- try {
|
|
|
- log.info("Flushed creativeReportDaily batch size: {}", creativeReportDailyBuffer.size());
|
|
|
- adqReportMapper.addsDailySD(new ArrayList<>(creativeReportDailyBuffer));
|
|
|
- creativeReportDailyBuffer.clear();
|
|
|
- } catch (Exception e) {
|
|
|
- log.error("Error flushing creativeReportDaily batch", e);
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- }*/
|
|
|
}
|