Merge #44 into master from pingkang

素材日报

* pingkang: (3 commits)
  fix bytedance_wx_app_info
  Merge branch 'master' of codeup.aliyun.com:60c06368dbbed34fe0f0c263/job/hr_job-...
  素材日报

Signed-off-by: 1129979389@qq.com <accounts_6361eaad7b63b44dd7833e63@mail.teambition.com>
Merged-by: 1129979389@qq.com <accounts_6361eaad7b63b44dd7833e63@mail.teambition.com>

CR-link: https://codeup.aliyun.com/60c06368dbbed34fe0f0c263/job/hr_job-bytedance/change/44

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/controller/BytedanceController.java

@@ -50,6 +50,9 @@ public class BytedanceController {
     @Autowired
     private IBytedanceCdpProjectService cdpProjectService;
 
+    @Autowired
+    private IBytedanceMaterialReportService bytedanceMaterialReportService;
+
 
     @GetMapping("/accountReport")
     public void getChannel() {
@@ -219,4 +222,17 @@ public class BytedanceController {
     public void bytedanceAccountHour(Long advertiserId, String accessToken, String date) {
         accountReportService.getBytedanceAccountHour(advertiserId, accessToken, date, date, 1, 0);
     }
+
+    @GetMapping("/materialDaily")
+    public void materialDaily(@RequestParam(required = false) Long advertiserId, String startDate, String endDate) {
+        List<JSONObject> costAccount = productReportService.getCostAccount(advertiserId, startDate, endDate);
+        for (JSONObject jsonObject : costAccount) {
+            dailyRetrExecutorService.submit(() -> {
+                Long adverId = jsonObject.getLong("advertiserId");
+                String accessToken = jsonObject.getString("accessToken");
+                bytedanceMaterialReportService.getBytedanceMaterialDailyReport(adverId, accessToken, startDate, endDate, 1, null, 1, 0);
+            });
+        }
+    }
+
 }

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/IBytedanceMaterialReportService.java

@@ -0,0 +1,6 @@
+package cn.com.ctop.job.bytedance.data.service;
+
+public interface IBytedanceMaterialReportService {
+    void getBytedanceMaterialDailyReport(Long advertiserId, String accessToken, String startDate, String endDate, Integer page,
+                                        Long errorId, Integer isLog, Integer isSend);
+}

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/impl/BytedanceMaterialReportServiceImpl.java

@@ -0,0 +1,112 @@
+package cn.com.ctop.job.bytedance.data.service.impl;
+
+import cn.com.ctop.job.bytedance.data.entity.BytedanceErrorLog;
+import cn.com.ctop.job.bytedance.data.mapper.BytedancePromoterReportMapper;
+import cn.com.ctop.job.bytedance.data.service.IBytedanceMaterialReportService;
+import cn.com.ctop.job.bytedance.data.utils.AlertUtils;
+import cn.com.ctop.job.bytedance.data.utils.Check;
+import cn.com.ctop.job.bytedance.data.utils.FieldsConfig;
+import cn.com.ctop.job.bytedance.data.utils.HttpUtils;
+import cn.com.ctop.job.bytedance.utils.doris.sink.DorisMultiTableStreamLoadManager;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.HashMap;
+import java.util.Map;
+
+@Service
+@Slf4j
+public class BytedanceMaterialReportServiceImpl implements IBytedanceMaterialReportService {
+    @Autowired
+    private BytedancePromoterReportMapper promoterReportMapper;
+    @Autowired
+    private DorisMultiTableStreamLoadManager dorisLoadManager;
+
+    @Override
+    public void getBytedanceMaterialDailyReport(Long advertiserId, String accessToken, String startDate, String endDate, Integer page, Long errorId, Integer isLog, Integer isSend) {
+        try {
+            String url = "https://api.oceanengine.com/open_api/v3.0/report/custom/get/";
+            Map<String, Object> paramsMap = new HashMap<>();
+            paramsMap.put("advertiser_id", advertiserId);
+            paramsMap.put("start_time", startDate);
+            paramsMap.put("end_time", endDate);
+            paramsMap.put("data_topic", "BASIC_DATA");
+            paramsMap.put("metrics", FieldsConfig.getMaterialMetrics());
+            paramsMap.put("dimensions", FieldsConfig.getMaterialDailyDimensions());
+            paramsMap.put("order_by", new JSONArray());
+            paramsMap.put("filters", new JSONArray());
+            paramsMap.put("page", page);
+            paramsMap.put("page_size", 100);
+            Map<String, String> headers = new HashMap<>();
+            headers.put("Access-Token", accessToken);
+            headers.put("Content-Type", "application/json");
+            String s = HttpUtils.httpGet(url, paramsMap, headers);
+            JSONObject result = JSONObject.parseObject(s);
+            if (Check.isNull(result)) {
+                log.info("账户id:{}，素材日报请求结果返回为空", advertiserId);
+                if (isLog == 1) {
+                    BytedanceErrorLog errorInfo = new BytedanceErrorLog();
+                    errorInfo.setAdvertiserId(advertiserId);
+                    errorInfo.setStartDate(startDate);
+                    errorInfo.setEndDate(endDate);
+                    errorInfo.setDataDimension("material");
+                    errorInfo.setDataType("daily");
+                    errorInfo.setMsg("请求结果返回为空");
+                    promoterReportMapper.replaceErrorInfo(errorInfo);
+                }
+                return;
+            }
+            Integer code = result.getInteger("code");
+            int totalPage = 1;
+            if (code == 0) {
+                JSONObject data = result.getJSONObject("data");
+                if (!Check.isNull(data)) {
+                    totalPage = data.getJSONObject("page_info").getIntValue("total_page");
+                    JSONArray rows = data.getJSONArray("rows");
+                    if (!Check.isNull(rows)) {
+                        for (int i = 0; i < rows.size(); i++) {
+                            JSONObject rowJson = rows.getJSONObject(i);
+                            JSONObject metricsJson = rowJson.getJSONObject("metrics");
+                            JSONObject dimensionsJson = rowJson.getJSONObject("dimensions");
+
+                            dimensionsJson.putAll(metricsJson);
+                            dimensionsJson.put("stat_time_day", dimensionsJson.getString("stat_time_day").replace("-", ""));
+                            dimensionsJson.put("advertiser_id", advertiserId);
+                            dorisLoadManager.add("media_api", "bytedance_material_report_daily", dimensionsJson);
+                        }
+                    }
+                }
+            } else {
+                log.info("账户id:{}，日期:{}素材日报请求结果失败，返回数据:{}", advertiserId, startDate, result);
+                if (isLog == 1) {
+                    BytedanceErrorLog errorInfo = new BytedanceErrorLog();
+                    errorInfo.setAdvertiserId(advertiserId);
+                    errorInfo.setStartDate(startDate);
+                    errorInfo.setEndDate(endDate);
+                    errorInfo.setDataDimension("material");
+                    errorInfo.setDataType("daily");
+                    errorInfo.setMsg(result.getString("message"));
+                    promoterReportMapper.replaceErrorInfo(errorInfo);
+                }
+                return;
+            }
+            if (page < totalPage) {
+                getBytedanceMaterialDailyReport(advertiserId, accessToken, startDate, endDate, page + 1, errorId, isLog, isSend);
+            } else {
+                log.info("账户ID：{}, 开始日期:{}, 结束日期:{}, totalPage:{}, 素材日报数据请求成功", advertiserId, startDate, endDate, totalPage);
+                if (errorId != null) {
+                    promoterReportMapper.deleteInfoByErrorId(errorId);
+                }
+                if (isSend != null && 1 == isSend) {
+                    AlertUtils.sendMessage(advertiserId, startDate, endDate, 1, "material", 1, null);
+                }
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+
+        }
+    }
+}

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/utils/FieldsConfig.java

@@ -44,6 +44,21 @@ public class FieldsConfig {
 
     }
 
+    public static JSONArray getMaterialDailyDimensions() {
+        JSONArray dimensions = new JSONArray();
+        dimensions.add("stat_time_day");
+        dimensions.add("cdp_project_id");
+        dimensions.add("cdp_project_name");
+        dimensions.add("cdp_promotion_id");
+        dimensions.add("cdp_promotion_name");
+        dimensions.add("material_id");
+        dimensions.add("image_mode");
+        dimensions.add("ad_platform_material_name");
+        dimensions.add("ad_platform_material_content");
+        return dimensions;
+
+    }
+
     public static JSONArray getMetrics() {
         JSONArray metricslds = new JSONArray();
         metricslds.add("stat_pay_amount");
@@ -72,6 +87,33 @@ public class FieldsConfig {
 
     }
 
+    public static JSONArray getMaterialMetrics() {
+        JSONArray metricslds = new JSONArray();
+        metricslds.add("stat_pay_amount");
+        metricslds.add("stat_cost");
+        metricslds.add("active");
+        metricslds.add("download_finish_cnt");
+        metricslds.add("attribution_next_day_open_cnt");
+        metricslds.add("game_addiction");
+        metricslds.add("attribution_game_in_app_ltv_1day");
+        metricslds.add("attribution_deep_convert_cnt");
+        metricslds.add("game_pay_count");
+        metricslds.add("convert_cnt");
+        metricslds.add("show_cnt");
+        metricslds.add("active_pay");
+        metricslds.add("click_cnt");
+        metricslds.add("click_start_cnt");
+        metricslds.add("attribution_convert_cnt");
+        metricslds.add("deep_convert_cnt");
+        metricslds.add("valid_play");
+        metricslds.add("attribution_billing_game_in_app_ltv_1day");
+        metricslds.add("attribution_micro_game_0d_ltv");
+        metricslds.add("attribution_micro_game_3d_ltv");
+        metricslds.add("attribution_micro_game_7d_ltv");
+        return metricslds;
+
+    }
+
     public static JSONArray getProductMetrics() {
         JSONArray metricslds = new JSONArray();
         metricslds.add("stat_cost");

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedancePromoterReportJob.java

@@ -1,6 +1,7 @@
 package cn.com.ctop.job.bytedance.handler;
 
 
+import cn.com.ctop.job.bytedance.data.service.IBytedanceMaterialReportService;
 import cn.com.ctop.job.bytedance.data.service.IBytedanceProductReportService;
 import cn.com.ctop.job.bytedance.data.service.IBytedancePromoterReportService;
 import cn.com.ctop.job.bytedance.data.service.IChannelService;
@@ -28,20 +29,14 @@ import java.util.concurrent.Executors;
 public class BytedancePromoterReportJob {
     @Autowired
     private IBytedanceProductReportService productReportService;
-
     @Autowired
     private IBytedancePromoterReportService promoterReportService;
     @Autowired
-    private IChannelService channelService;
+    private IBytedanceMaterialReportService bytedanceMaterialReportService;
 
     private static ExecutorService dailyExecutorService = Executors.newFixedThreadPool(10);
     private static ExecutorService hourExecutorService = Executors.newFixedThreadPool(4);
-
-    private static ExecutorService dailyRetrExecutorService = Executors.newFixedThreadPool(2);
-    private static ExecutorService hourRetrExecutorService = Executors.newFixedThreadPool(2);
-
     private static ExecutorService productDailyExecutor = Executors.newFixedThreadPool(2);
-    private static ExecutorService productDailyRetrExecutor = Executors.newFixedThreadPool(2);
 
 
     @XxlJob("yesHourPromoterReport")
@@ -134,6 +129,27 @@ public class BytedancePromoterReportJob {
         });
     }
 
+    @XxlJob("yesMaterialReportDaily")
+    public void yesMaterialReportDaily() throws Exception {
+        String param = XxlJobHelper.getJobParam(); // 执行参数
+        if (Check.isNull(param)) {
+            log.error("入参为空");
+            return;
+        }
+
+        String nowDate = DateUtils.getNowDate("yyyy-MM-dd");
+        String date = DateUtils.getAnotherDay("yyyy-MM-dd", nowDate, -1);
+        String[] s = param.split("_");
+        Long advertiserId = Long.valueOf(s[0]);
+        String accessToken = s[1];
+
+        dailyExecutorService.submit(new Runnable() {
+            @Override
+            public void run() {
+                bytedanceMaterialReportService.getBytedanceMaterialDailyReport(advertiserId, accessToken, date, date, 1, null, 1, 0);
+            }
+        });
+    }
 
 
 }

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedanceReportRetryJob.java

@@ -44,6 +44,9 @@ public class BytedanceReportRetryJob {
     @Autowired
     private IBytedanceXingTuService bytedanceXingTuService;
 
+    @Autowired
+    private IBytedanceMaterialReportService bytedanceMaterialReportService;
+
 
     private static ExecutorService hourAccountRetrExecutorService = Executors.newFixedThreadPool(2);
     private static ExecutorService dailyAccountRetrExecutorService = Executors.newFixedThreadPool(2);
@@ -133,6 +136,15 @@ public class BytedanceReportRetryJob {
                             byteDanceFlowDetailsReportService.getBytedanceFlowDetailsDailyReport(advertiserId, accessToken, AccountTypeEnum.AD.getValue(), startDate, endDate, 1, id);
                         }
                     });
+                } else if ("material".equals(dataDimension)) {
+                    productDailyRetrExecutor.submit(new Runnable() {
+                        @Override
+                        public void run() {
+                            if (!Check.isNull(accessToken)) {
+                                bytedanceMaterialReportService.getBytedanceMaterialDailyReport(advertiserId, accessToken, startDate, endDate, 1, id, null, isSend);
+                            }
+                        }
+                    });
                 }