|
@@ -14,8 +14,10 @@ import com.alibaba.fastjson.JSONArray;
|
|
|
import com.alibaba.fastjson.JSONObject;
|
|
import com.alibaba.fastjson.JSONObject;
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
|
|
|
+import org.springframework.scheduling.annotation.Scheduled;
|
|
|
import org.springframework.stereotype.Service;
|
|
import org.springframework.stereotype.Service;
|
|
|
|
|
|
|
|
|
|
+import javax.annotation.PreDestroy;
|
|
|
import java.time.Instant;
|
|
import java.time.Instant;
|
|
|
import java.util.*;
|
|
import java.util.*;
|
|
|
|
|
|
|
@@ -28,6 +30,16 @@ public class AdqCreativeCreativeReportServiceImpl implements IAdqCreativeReportS
|
|
|
@Autowired
|
|
@Autowired
|
|
|
private AdqErrorLogMapper adqErrorLogMapper;
|
|
private AdqErrorLogMapper adqErrorLogMapper;
|
|
|
|
|
|
|
|
|
|
+
|
|
|
|
|
+ private static final int MAX_BATCH_SIZE = 5000;
|
|
|
|
|
+ private static final long MAX_WAIT_TIME_MS = 2 * 60 * 1000;
|
|
|
|
|
+
|
|
|
|
|
+ private final List<AdqCreativeReportHour> creativeReportHourBuffer = Collections.synchronizedList(new ArrayList<>());
|
|
|
|
|
+ private final List<AdqCreativeReportDaily> creativeReportDailyBuffer = Collections.synchronizedList(new ArrayList<>());
|
|
|
|
|
+ private long creativeReportHourLastFlushTime = System.currentTimeMillis();
|
|
|
|
|
+ private long creativeReportDailyLastFlushTime = System.currentTimeMillis();
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
@Override
|
|
@Override
|
|
|
public void getAdqCreativeHour(Long advertiserId, String accessToken, String startDate, String endDate, Integer page, Integer isLog) {
|
|
public void getAdqCreativeHour(Long advertiserId, String accessToken, String startDate, String endDate, Integer page, Integer isLog) {
|
|
|
try {
|
|
try {
|
|
@@ -55,7 +67,6 @@ public class AdqCreativeCreativeReportServiceImpl implements IAdqCreativeReportS
|
|
|
paramsMap.put("fields", fieldsList);
|
|
paramsMap.put("fields", fieldsList);
|
|
|
List<AdqCreativeReportHour> hourList = new ArrayList<>();
|
|
List<AdqCreativeReportHour> hourList = new ArrayList<>();
|
|
|
String s = HttpUtils.httpGet("https://api.e.qq.com/v3.0/hourly_reports/get", paramsMap, null);
|
|
String s = HttpUtils.httpGet("https://api.e.qq.com/v3.0/hourly_reports/get", paramsMap, null);
|
|
|
- // System.err.println(s);
|
|
|
|
|
JSONObject jsonObject = JSONObject.parseObject(s);
|
|
JSONObject jsonObject = JSONObject.parseObject(s);
|
|
|
if (!Check.isNull(jsonObject)) {
|
|
if (!Check.isNull(jsonObject)) {
|
|
|
Integer code = jsonObject.getInteger("code");
|
|
Integer code = jsonObject.getInteger("code");
|
|
@@ -93,7 +104,8 @@ public class AdqCreativeCreativeReportServiceImpl implements IAdqCreativeReportS
|
|
|
adqReportMapper.addsHourList(hourList);
|
|
adqReportMapper.addsHourList(hourList);
|
|
|
log.info("新增数据完成account:id:{}", advertiserId);
|
|
log.info("新增数据完成account:id:{}", advertiserId);
|
|
|
try {
|
|
try {
|
|
|
- adqReportMapper.addsHourListSD(hourList);
|
|
|
|
|
|
|
+ creativeReportHourBuffer.addAll(hourList);
|
|
|
|
|
+
|
|
|
} catch (Exception e) {
|
|
} catch (Exception e) {
|
|
|
e.printStackTrace();
|
|
e.printStackTrace();
|
|
|
}
|
|
}
|
|
@@ -176,7 +188,7 @@ public class AdqCreativeCreativeReportServiceImpl implements IAdqCreativeReportS
|
|
|
if (!Check.isNull(dailies)) {
|
|
if (!Check.isNull(dailies)) {
|
|
|
adqReportMapper.addsDaily(dailies);
|
|
adqReportMapper.addsDaily(dailies);
|
|
|
try {
|
|
try {
|
|
|
- adqReportMapper.addsDailySD(dailies);
|
|
|
|
|
|
|
+ creativeReportDailyBuffer.addAll(dailies);
|
|
|
} catch (Exception e) {
|
|
} catch (Exception e) {
|
|
|
e.printStackTrace();
|
|
e.printStackTrace();
|
|
|
}
|
|
}
|
|
@@ -200,4 +212,73 @@ public class AdqCreativeCreativeReportServiceImpl implements IAdqCreativeReportS
|
|
|
public List<JSONObject> getAccountListByDate(Integer startDate, Integer endDate) {
|
|
public List<JSONObject> getAccountListByDate(Integer startDate, Integer endDate) {
|
|
|
return adqErrorLogMapper.getAccountListByDate(startDate, endDate);
|
|
return adqErrorLogMapper.getAccountListByDate(startDate, endDate);
|
|
|
}
|
|
}
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ @Scheduled(cron = "*/30 * * * * ?") // 每30秒检查一次
|
|
|
|
|
+ public void flushIfNeeded() {
|
|
|
|
|
+ log.info("开始执行创意报表定时批处理任务....");
|
|
|
|
|
+ synchronized (creativeReportHourBuffer) {
|
|
|
|
|
+ long now = System.currentTimeMillis();
|
|
|
|
|
+ if (creativeReportHourBuffer.size() >= MAX_BATCH_SIZE ||
|
|
|
|
|
+ (now - creativeReportHourLastFlushTime) >= MAX_WAIT_TIME_MS && !creativeReportHourBuffer.isEmpty()) {
|
|
|
|
|
+ try {
|
|
|
|
|
+ log.info("Flushed creativeReportHour batch size: {}", creativeReportHourBuffer.size());
|
|
|
|
|
+ adqReportMapper.addsHourListSD(new ArrayList<>(creativeReportHourBuffer));
|
|
|
|
|
+ creativeReportHourBuffer.clear();
|
|
|
|
|
+ creativeReportHourLastFlushTime = now;
|
|
|
|
|
+ } catch (Exception e) {
|
|
|
|
|
+ log.error("Error flushing creativeReportHour batch", e);
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ synchronized (creativeReportDailyBuffer) {
|
|
|
|
|
+ long now = System.currentTimeMillis();
|
|
|
|
|
+ if (creativeReportDailyBuffer.size() >= MAX_BATCH_SIZE ||
|
|
|
|
|
+ (now - creativeReportDailyLastFlushTime) >= MAX_WAIT_TIME_MS && !creativeReportDailyBuffer.isEmpty()) {
|
|
|
|
|
+ try {
|
|
|
|
|
+ log.info("Flushed creativeReportDaily batch size: {}", creativeReportDailyBuffer.size());
|
|
|
|
|
+ adqReportMapper.addsDailySD(new ArrayList<>(creativeReportDailyBuffer));
|
|
|
|
|
+ creativeReportDailyBuffer.clear();
|
|
|
|
|
+ creativeReportDailyLastFlushTime = now;
|
|
|
|
|
+ } catch (Exception e) {
|
|
|
|
|
+ log.error("Error flushing creativeReportDaily batch", e);
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ @PreDestroy
|
|
|
|
|
+ public void flushPreDestroy() {
|
|
|
|
|
+ log.info("项目重启,开始执行创意报表批处理任务....");
|
|
|
|
|
+ synchronized (creativeReportHourBuffer) {
|
|
|
|
|
+ if (!creativeReportHourBuffer.isEmpty()) {
|
|
|
|
|
+ try {
|
|
|
|
|
+ log.info("Flushed advertiserReportHour batch size: {}", creativeReportHourBuffer.size());
|
|
|
|
|
+ adqReportMapper.addsHourListSD(new ArrayList<>(creativeReportHourBuffer));
|
|
|
|
|
+ creativeReportHourBuffer.clear();
|
|
|
|
|
+ } catch (Exception e) {
|
|
|
|
|
+ log.error("Error flushing advertiserReportHour batch", e);
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ synchronized (creativeReportDailyBuffer) {
|
|
|
|
|
+ if (!creativeReportDailyBuffer.isEmpty()) {
|
|
|
|
|
+ try {
|
|
|
|
|
+ log.info("Flushed creativeReportDaily batch size: {}", creativeReportDailyBuffer.size());
|
|
|
|
|
+ adqReportMapper.addsDailySD(new ArrayList<>(creativeReportDailyBuffer));
|
|
|
|
|
+ creativeReportDailyBuffer.clear();
|
|
|
|
|
+ } catch (Exception e) {
|
|
|
|
|
+ log.error("Error flushing creativeReportDaily batch", e);
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
}
|
|
}
|