|
|
@@ -1,8 +1,6 @@
|
|
|
package cn.com.ctop.job.adq.data.service.impl;
|
|
|
|
|
|
-import cn.com.ctop.job.adq.data.entity.AdqAdvertiserReportDaily;
|
|
|
-import cn.com.ctop.job.adq.data.entity.AdqAdvertiserReportHour;
|
|
|
-import cn.com.ctop.job.adq.data.entity.AdqErrorLog;
|
|
|
+import cn.com.ctop.job.adq.data.entity.*;
|
|
|
import cn.com.ctop.job.adq.data.mapper.AdqAccountReportMapper;
|
|
|
import cn.com.ctop.job.adq.data.mapper.AdqErrorLogMapper;
|
|
|
import cn.com.ctop.job.adq.data.service.IAdqAccountReportService;
|
|
|
@@ -14,13 +12,12 @@ import com.alibaba.fastjson.JSONArray;
|
|
|
import com.alibaba.fastjson.JSONObject;
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
|
+import org.springframework.scheduling.annotation.Scheduled;
|
|
|
import org.springframework.stereotype.Service;
|
|
|
|
|
|
+import javax.annotation.PreDestroy;
|
|
|
import java.time.Instant;
|
|
|
-import java.util.ArrayList;
|
|
|
-import java.util.HashMap;
|
|
|
-import java.util.List;
|
|
|
-import java.util.Map;
|
|
|
+import java.util.*;
|
|
|
|
|
|
@Service
|
|
|
@Slf4j
|
|
|
@@ -29,7 +26,13 @@ public class AdqAccountReportServiceImpl implements IAdqAccountReportService {
|
|
|
private AdqErrorLogMapper adqErrorLogMapper;
|
|
|
@Autowired
|
|
|
private AdqAccountReportMapper accountReportMapper;
|
|
|
+ private static final int MAX_BATCH_SIZE = 5000;
|
|
|
+ private static final long MAX_WAIT_TIME_MS = 2 * 60 * 1000;
|
|
|
|
|
|
+ private final List<AdqAdvertiserReportHour> advertiserReportHourBuffer = Collections.synchronizedList(new ArrayList<>());
|
|
|
+ private final List<AdqAdvertiserReportDaily> advertiserReportDailyBuffer = Collections.synchronizedList(new ArrayList<>());
|
|
|
+ private long advertiserReportHourLastFlushTime = System.currentTimeMillis();
|
|
|
+ private long advertiserReportDailyLastFlushTime = System.currentTimeMillis();
|
|
|
|
|
|
@Override
|
|
|
public void getAdqAccountHour(Long advertiserId, String accessToken, String startDate, String endDate, Integer page, Integer isLog) {
|
|
|
@@ -92,7 +95,7 @@ public class AdqAccountReportServiceImpl implements IAdqAccountReportService {
|
|
|
if (!Check.isNull(hourList)) {
|
|
|
accountReportMapper.addAccountHourList(hourList);
|
|
|
try {
|
|
|
- accountReportMapper.addAccountHourListSD(hourList);
|
|
|
+ advertiserReportHourBuffer.addAll(hourList);
|
|
|
} catch (Exception e) {
|
|
|
log.error("addAccountHourListSD", e.getMessage());
|
|
|
}
|
|
|
@@ -167,7 +170,7 @@ public class AdqAccountReportServiceImpl implements IAdqAccountReportService {
|
|
|
if (!Check.isNull(dailyList)) {
|
|
|
accountReportMapper.addAccountDailyList(dailyList);
|
|
|
try {
|
|
|
- accountReportMapper.addAccountDailyListSD(dailyList);
|
|
|
+ advertiserReportDailyBuffer.addAll(dailyList);
|
|
|
} catch (Exception e) {
|
|
|
log.error("addAccountDailyListSD", e);
|
|
|
}
|
|
|
@@ -181,4 +184,73 @@ public class AdqAccountReportServiceImpl implements IAdqAccountReportService {
|
|
|
e.printStackTrace();
|
|
|
}
|
|
|
}
|
|
|
+
|
|
|
+
|
|
|
+ @Scheduled(cron = "*/20 * * * * ?") // 每30秒检查一次
|
|
|
+ public void flushIfNeeded() {
|
|
|
+ log.info("开始执行账户报表定时批处理任务....");
|
|
|
+ synchronized (advertiserReportHourBuffer) {
|
|
|
+ long now = System.currentTimeMillis();
|
|
|
+ if (advertiserReportHourBuffer.size() >= MAX_BATCH_SIZE ||
|
|
|
+ (now - advertiserReportHourLastFlushTime) >= MAX_WAIT_TIME_MS && !advertiserReportHourBuffer.isEmpty()) {
|
|
|
+ try {
|
|
|
+ log.info("Flushed advertiserReportHour batch size: {}", advertiserReportHourBuffer.size());
|
|
|
+ accountReportMapper.addAccountHourListSD(new ArrayList<>(advertiserReportHourBuffer));
|
|
|
+ advertiserReportHourBuffer.clear();
|
|
|
+ advertiserReportHourLastFlushTime = now;
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error("Error flushing advertiserReportHour batch", e);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+ synchronized (advertiserReportDailyBuffer) {
|
|
|
+ long now = System.currentTimeMillis();
|
|
|
+ if (advertiserReportDailyBuffer.size() >= MAX_BATCH_SIZE ||
|
|
|
+ (now - advertiserReportDailyLastFlushTime) >= MAX_WAIT_TIME_MS && !advertiserReportDailyBuffer.isEmpty()) {
|
|
|
+ try {
|
|
|
+ log.info("Flushed advertiserReportDaily batch size: {}", advertiserReportDailyBuffer.size());
|
|
|
+ accountReportMapper.addAccountDailyListSD(new ArrayList<>(advertiserReportDailyBuffer));
|
|
|
+ advertiserReportDailyBuffer.clear();
|
|
|
+ advertiserReportDailyLastFlushTime = now;
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error("Error flushing advertiserReportDaily batch", e);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+ @PreDestroy
|
|
|
+ public void flushPreDestroy() {
|
|
|
+ log.info("项目重启,开始执行账户报表批处理任务....");
|
|
|
+ synchronized (advertiserReportHourBuffer) {
|
|
|
+ if (!advertiserReportHourBuffer.isEmpty()) {
|
|
|
+ try {
|
|
|
+ log.info("Flushed advertiserReportHour batch size: {}", advertiserReportHourBuffer.size());
|
|
|
+ accountReportMapper.addAccountHourListSD(new ArrayList<>(advertiserReportHourBuffer));
|
|
|
+ advertiserReportHourBuffer.clear();
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error("Error flushing advertiserReportHour batch", e);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+ synchronized (advertiserReportDailyBuffer) {
|
|
|
+ if (!advertiserReportDailyBuffer.isEmpty()) {
|
|
|
+ try {
|
|
|
+ log.info("Flushed advertiserReportDaily batch size: {}", advertiserReportDailyBuffer.size());
|
|
|
+ accountReportMapper.addAccountDailyListSD(new ArrayList<>(advertiserReportDailyBuffer));
|
|
|
+ advertiserReportDailyBuffer.clear();
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error("Error flushing advertiserReportDaily batch", e);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
}
|