Merge remote-tracking branch 'origin/master'

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/controller/BytedanceController.java

@@ -209,8 +209,8 @@ public class BytedanceController {
         for (JSONObject jsonObject : costAccount) {
             executor10Service1.submit(() -> {
                 Long adverId = jsonObject.getLong("advertiserId");
-                cdpProjectService.getProjectList(adverId, null, null, null,null,1);
-                cdpProjectService.getProjectList(adverId, null, "del", null,null,1);
+                cdpProjectService.getProjectList(adverId, null, null, null, null, 1);
+                cdpProjectService.getProjectList(adverId, null, "del", null, null, 1);
             });
         }
     }
@@ -221,7 +221,7 @@ public class BytedanceController {
         for (JSONObject jsonObject : costAccount) {
             executor10Service2.submit(() -> {
                 Long adverId = jsonObject.getLong("advertiserId");
-                cdpProjectService.getPromotionList(adverId,null,null,null,1);
+                cdpProjectService.getPromotionList(adverId, null, null, null, 1);
             });
         }
     }
@@ -247,8 +247,8 @@ public class BytedanceController {
                 log.info("项目ID数不能超过100");
                 return;
             }
-            cdpProjectService.getProjectList(advertiserId, ids, null, null,null,1);
-            cdpProjectService.getProjectList(advertiserId, ids, "del", null,null,1);
+            cdpProjectService.getProjectList(advertiserId, ids, null, null, null, 1);
+            cdpProjectService.getProjectList(advertiserId, ids, "del", null, null, 1);
         }
     }
 
@@ -273,7 +273,7 @@ public class BytedanceController {
                 log.info("项目ID数不能超过100");
                 return;
             }
-            cdpProjectService.getPromotionList(advertiserId, ids, null,null,1);
+            cdpProjectService.getPromotionList(advertiserId, ids, null, null, 1);
         }
     }
 
@@ -313,7 +313,19 @@ public class BytedanceController {
             executor10Service1.submit(() -> {
                 Long adverId = jsonObject.getLong("advertiserId");
                 String accessToken = jsonObject.getString("accessToken");
-                bytedanceMaterialReportService.getBytedanceMaterialDailyReport(adverId, accessToken, startDate, endDate, 1, null, 1, 0);
+                bytedanceMaterialReportService.getBytedanceMaterialDailyReport(adverId, accessToken, startDate, endDate, 1, null, 1, 0, 1);
+            });
+        }
+    }
+
+    @GetMapping("/materialVideoGet")
+    public void materialVideoGet(@RequestParam(required = false) Long advertiserId, String startDate, String endDate) {
+        List<JSONObject> costAccount = productReportService.getCostAccount(advertiserId, startDate, endDate);
+        for (JSONObject jsonObject : costAccount) {
+            executor10Service1.submit(() -> {
+                Long adverId = jsonObject.getLong("advertiserId");
+                String accessToken = jsonObject.getString("accessToken");
+                bytedanceMaterialReportService.getBytedanceMaterialVideo(adverId, null, accessToken, null, null, 1, null, 1, 0,1);
             });
         }
     }

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/mapper/xml/BytedancePromoterReportMapper.xml

@@ -8,7 +8,7 @@
     </delete>
 
     <select id="getAccountList" resultType="com.alibaba.fastjson.JSONObject">
-        SELECT t1.is_send AS 'isSend', t1.id AS 'id', t1.advertiser_id AS 'advertiserId', t1.start_date AS 'startDate', t1.end_date AS 'endDate', t1.data_dimension AS 'dataDimension', t1.data_type AS 'dataType', t3.access_token AS 'accessToken'
+        SELECT t1.msg,t1.is_send AS 'isSend', t1.id AS 'id', t1.advertiser_id AS 'advertiserId', t1.start_date AS 'startDate', t1.end_date AS 'endDate', t1.data_dimension AS 'dataDimension', t1.data_type AS 'dataType', t3.access_token AS 'accessToken'
         FROM bytedance_error_log t1
                  LEFT JOIN deepthink.bytedance_advertiser_list t2 ON t1.advertiser_id = t2.advertiser_id
                  LEFT JOIN deepthink.bytedance_auth_list t3 ON t2.admin_advertiser_id = t3.admin_advertiser_id

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/IBytedanceMaterialReportService.java

@@ -1,6 +1,10 @@
 package cn.com.ctop.job.bytedance.data.service;
 
+import java.util.Set;
+
 public interface IBytedanceMaterialReportService {
     void getBytedanceMaterialDailyReport(Long advertiserId, String accessToken, String startDate, String endDate, Integer page,
-                                        Long errorId, Integer isLog, Integer isSend);
+                                         Long errorId, Integer isLog, Integer isSend, int retry);
+
+    void getBytedanceMaterialVideo(Long advertiserId, Set<Long> materialIds, String accessToken, String startTime, String endTime, Integer page, Long errorId, Integer isLog, Integer isSend, int retry);
 }

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/impl/BytedanceMaterialReportServiceImpl.java

@@ -12,10 +12,14 @@ import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.data.redis.core.RedisTemplate;
 import org.springframework.stereotype.Service;
 
+import java.time.Duration;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 
 @Service
 @Slf4j
@@ -24,9 +28,11 @@ public class BytedanceMaterialReportServiceImpl implements IBytedanceMaterialRep
     private BytedancePromoterReportMapper promoterReportMapper;
     @Autowired
     private DorisMultiTableStreamLoadManager dorisLoadManager;
+    @Autowired
+    private RedisTemplate redisTemplate_db2;
 
     @Override
-    public void getBytedanceMaterialDailyReport(Long advertiserId, String accessToken, String startDate, String endDate, Integer page, Long errorId, Integer isLog, Integer isSend) {
+    public void getBytedanceMaterialDailyReport(Long advertiserId, String accessToken, String startDate, String endDate, Integer page, Long errorId, Integer isLog, Integer isSend, int retry) {
         try {
             String url = "https://api.oceanengine.com/open_api/v3.0/report/custom/get/";
             Map<String, Object> paramsMap = new HashMap<>();
@@ -45,6 +51,7 @@ public class BytedanceMaterialReportServiceImpl implements IBytedanceMaterialRep
             headers.put("Content-Type", "application/json");
             String s = HttpUtils.httpGet(url, paramsMap, headers);
             JSONObject result = JSONObject.parseObject(s);
+            Set<Long> materialIds = new HashSet<>();
             if (Check.isNull(result)) {
                 log.info("账户id:{}，素材日报请求结果返回为空", advertiserId);
                 if (isLog == 1) {
@@ -76,10 +83,24 @@ public class BytedanceMaterialReportServiceImpl implements IBytedanceMaterialRep
                             dimensionsJson.put("stat_time_day", dimensionsJson.getString("stat_time_day").replace("-", ""));
                             dimensionsJson.put("advertiser_id", advertiserId);
                             dorisLoadManager.add("media_api", "bytedance_material_report_daily", dimensionsJson);
+
+                            Long materialId = dimensionsJson.getLong("material_id");
+                            String imageMode = dimensionsJson.getString("image_mode");
+                            if (!materialIds.contains(materialId) && !redisTemplate_db2.hasKey("mvid:1:" + materialId) && imageMode.contains("视频")) {
+                                materialIds.add(materialId);
+                            }
                         }
                     }
                 }
             } else {
+                if (retry <= 5) {
+                    try {
+                        Thread.sleep(800);
+                    } catch (Exception ignored) {
+                    }
+                    getBytedanceMaterialDailyReport(advertiserId, accessToken, startDate, endDate, page, errorId, isLog, isSend, retry + 1);
+                    return;
+                }
                 log.info("账户id:{}，日期:{}素材日报请求结果失败，返回数据:{}", advertiserId, startDate, result);
                 if (isLog == 1) {
                     BytedanceErrorLog errorInfo = new BytedanceErrorLog();
@@ -93,8 +114,13 @@ public class BytedanceMaterialReportServiceImpl implements IBytedanceMaterialRep
                 }
                 return;
             }
+
+            if (!Check.isNull(materialIds)) {
+                getBytedanceMaterialVideo(advertiserId, materialIds, accessToken, null, null, 1, null, 1, null, 1);
+            }
+
             if (page < totalPage) {
-                getBytedanceMaterialDailyReport(advertiserId, accessToken, startDate, endDate, page + 1, errorId, isLog, isSend);
+                getBytedanceMaterialDailyReport(advertiserId, accessToken, startDate, endDate, page + 1, errorId, isLog, isSend, 1);
             } else {
                 log.info("账户ID：{}, 开始日期:{}, 结束日期:{}, totalPage:{}, 素材日报数据请求成功", advertiserId, startDate, endDate, totalPage);
                 if (errorId != null) {
@@ -109,4 +135,94 @@ public class BytedanceMaterialReportServiceImpl implements IBytedanceMaterialRep
 
         }
     }
+
+    public void getBytedanceMaterialVideo(Long advertiserId, Set<Long> materialIds, String accessToken, String startTime, String endTime, Integer page, Long errorId, Integer isLog, Integer isSend, int retry) {
+        try {
+            String url = "https://api.oceanengine.com/open_api/2/file/video/get/";
+            Map<String, Object> paramsMap = new HashMap<>();
+            paramsMap.put("advertiser_id", advertiserId);
+            JSONObject filtering = new JSONObject();
+            if (!Check.isNull(startTime) && !Check.isNull(endTime)) {
+                filtering.put("start_time", startTime);
+                filtering.put("end_time", endTime);
+            }
+            if (!Check.isNull(materialIds)) {
+                filtering.put("material_ids", materialIds);
+            }
+            paramsMap.put("filtering", filtering);
+            paramsMap.put("page", page);
+            paramsMap.put("page_size", 100);
+            Map<String, String> headers = new HashMap<>();
+            headers.put("Access-Token", accessToken);
+            headers.put("Content-Type", "application/json");
+            String s = HttpUtils.httpGet(url, paramsMap, headers);
+            JSONObject result = JSONObject.parseObject(s);
+            if (Check.isNull(result)) {
+                log.info("账户id:{}，获取视频素材结果返回为空", advertiserId);
+                if (isLog == 1) {
+                    BytedanceErrorLog errorInfo = new BytedanceErrorLog();
+                    errorInfo.setAdvertiserId(advertiserId);
+                    errorInfo.setStartDate(Check.isNull(startTime) ? "" : startTime);
+                    errorInfo.setEndDate(Check.isNull(endTime) ? "" : endTime);
+                    errorInfo.setDataDimension("material_video_get");
+                    errorInfo.setDataType("daily");
+                    errorInfo.setMsg(new JSONObject().fluentPut("msg", "结果为空").fluentPut("material_ids", materialIds).toJSONString());
+                    promoterReportMapper.replaceErrorInfo(errorInfo);
+                }
+                return;
+            }
+            Integer code = result.getInteger("code");
+            int totalPage = 1;
+            if (code == 0) {
+                JSONObject data = result.getJSONObject("data");
+                if (!Check.isNull(data)) {
+                    totalPage = data.getJSONObject("page_info").getIntValue("total_page");
+                    JSONArray rows = data.getJSONArray("list");
+                    if (!Check.isNull(rows)) {
+                        for (int i = 0; i < rows.size(); i++) {
+                            JSONObject rowJson = rows.getJSONObject(i);
+                            String materialId = rowJson.getString("material_id");
+                            redisTemplate_db2.opsForValue().set("mvid:1:" + materialId, materialId, Duration.ofSeconds(30 * 24 * 3600 + (int) (Math.random() * 7200)));
+                            rowJson.put("advertiser_id", advertiserId);
+                            dorisLoadManager.add("media_api", "bytedance_video_list", rowJson);
+                        }
+                    }
+                }
+            } else {
+                if (retry <= 5) {
+                    try {
+                        Thread.sleep(800);
+                    } catch (Exception ignored) {
+                    }
+                    getBytedanceMaterialVideo(advertiserId, materialIds, accessToken, startTime, endTime, page, errorId, isLog, isSend, retry + 1);
+                    return;
+                }
+                log.info("账户id:{}，日期:{}获取视频素材失败，返回数据:{}", advertiserId, startTime, result);
+                if (isLog == 1) {
+                    BytedanceErrorLog errorInfo = new BytedanceErrorLog();
+                    errorInfo.setAdvertiserId(advertiserId);
+                    errorInfo.setStartDate(Check.isNull(startTime) ? "" : startTime);
+                    errorInfo.setEndDate(Check.isNull(endTime) ? "" : endTime);
+                    errorInfo.setDataDimension("material_video_get");
+                    errorInfo.setDataType("daily");
+                    errorInfo.setMsg(result.fluentPut("material_ids", materialIds).toJSONString());
+                    promoterReportMapper.replaceErrorInfo(errorInfo);
+                }
+                return;
+            }
+            if (page < totalPage) {
+                getBytedanceMaterialVideo(advertiserId, materialIds, accessToken, startTime, endTime, page + 1, errorId, isLog, isSend, 1);
+            } else {
+                log.info("账户ID：{}, 开始日期:{}, 结束日期:{}, totalPage:{}, 获取视频素材成功", advertiserId, startTime, endTime, totalPage);
+                if (errorId != null) {
+                    promoterReportMapper.deleteInfoByErrorId(errorId);
+                }
+                if (isSend != null && 1 == isSend) {
+                    AlertUtils.sendMessage(advertiserId, startTime, endTime, 1, "material", 1, null);
+                }
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
 }

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedancePromoterReportJob.java

@@ -4,7 +4,6 @@ package cn.com.ctop.job.bytedance.handler;
 import cn.com.ctop.job.bytedance.data.service.IBytedanceMaterialReportService;
 import cn.com.ctop.job.bytedance.data.service.IBytedanceProductReportService;
 import cn.com.ctop.job.bytedance.data.service.IBytedancePromoterReportService;
-import cn.com.ctop.job.bytedance.data.service.IChannelService;
 import cn.com.ctop.job.bytedance.data.utils.Check;
 import cn.com.ctop.job.bytedance.data.utils.DateUtils;
 import com.alibaba.fastjson.JSONObject;
@@ -107,7 +106,6 @@ public class BytedancePromoterReportJob {
     }
 
 
-
     @XxlJob("ltv3ReportPromoterReport")
     public void ltv3ReportPromoterReport() throws Exception {
         String param = XxlJobHelper.getJobParam(); // 执行参数
@@ -146,10 +144,24 @@ public class BytedancePromoterReportJob {
         dailyExecutorService.submit(new Runnable() {
             @Override
             public void run() {
-                bytedanceMaterialReportService.getBytedanceMaterialDailyReport(advertiserId, accessToken, date, date, 1, null, 1, 0);
+                bytedanceMaterialReportService.getBytedanceMaterialDailyReport(advertiserId, accessToken, date, date, 1, null, 1, 0, 1);
             }
         });
     }
 
+    @XxlJob("todayMaterialVideoGet")
+    public void materialVideoGet() throws Exception {
+        String nowDate = DateUtils.getNowDate("yyyy-MM-dd");
+        String date = DateUtils.getAnotherDay("yyyy-MM-dd", nowDate, -1);
+        List<JSONObject> costAccount = productReportService.getCostAccount(null, nowDate, nowDate);
+        for (JSONObject jsonObject : costAccount) {
+            dailyExecutorService.submit(() -> {
+                Long adverId = jsonObject.getLong("advertiserId");
+                String accessToken = jsonObject.getString("accessToken");
+                bytedanceMaterialReportService.getBytedanceMaterialVideo(adverId, null, accessToken, date, nowDate, 1, null, 1, 0, 1);
+            });
+        }
+    }
+
 
 }

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedanceReportRetryJob.java

@@ -12,6 +12,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
 import javax.annotation.Resource;
+import java.util.HashSet;
 import java.util.List;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -85,6 +86,7 @@ public class BytedanceReportRetryJob {
                 String dataType = jsonObject.getString("dataType");
                 String accessToken = jsonObject.getString("accessToken");
                 Integer isSend = jsonObject.getInteger("isSend");
+                String msg = jsonObject.getString("msg");
                 if ("promoter".equals(dataDimension)) {
                     if (dataType.equals("hour")) { // 广告时报
                         hourPromoterRetrExecutorService.submit(new Runnable() {
@@ -152,7 +154,21 @@ public class BytedanceReportRetryJob {
                         @Override
                         public void run() {
                             if (!Check.isNull(accessToken)) {
-                                bytedanceMaterialReportService.getBytedanceMaterialDailyReport(advertiserId, accessToken, startDate, endDate, 1, id, null, isSend);
+                                bytedanceMaterialReportService.getBytedanceMaterialDailyReport(advertiserId, accessToken, startDate, endDate, 1, id, null, isSend, 1);
+                            }
+                        }
+                    });
+                } else if ("material_video_get".equals(dataDimension)) {
+                    HashSet<Long> materialIds = new HashSet<>();
+                    JSONObject msgObject = JSONObject.parseObject(msg);
+                    if (msgObject != null && msgObject.getJSONArray("material_ids") != null) {
+                        materialIds.addAll(msgObject.getJSONArray("material_ids").toJavaList(Long.class));
+                    }
+                    productDailyRetrExecutor.submit(new Runnable() {
+                        @Override
+                        public void run() {
+                            if (!Check.isNull(accessToken)) {
+                                bytedanceMaterialReportService.getBytedanceMaterialVideo(advertiserId, materialIds, accessToken, startDate, endDate, 1, id, null, isSend, 1);
                             }
                         }
                     });

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/utils/RedisConfig.java

@@ -29,10 +29,15 @@ public class RedisConfig {
     }
 
     @Bean(name = "redisTemplate_db10")
-    public RedisTemplate<String, Object> redisTemplate1() {
+    public RedisTemplate<String, Object> redisTemplate10() {
         return createTemplate(10);
     }
 
+    @Bean(name = "redisTemplate_db2")
+    public RedisTemplate<String, Object> redisTemplate2() {
+        return createTemplate(2);
+    }
+
     private RedisTemplate<String, Object> createTemplate(int dbIndex) {
         RedisStandaloneConfiguration config = new RedisStandaloneConfiguration();
         config.setHostName(host);