Merge #2 into master from pingkang

巨量产品数据报表

* pingkang: (2 commits)
  巨量产品数据报表
  巨量产品数据报表

Signed-off-by: 1129979389@qq.com <accounts_6361eaad7b63b44dd7833e63@mail.teambition.com>
Merged-by: 1129979389@qq.com <accounts_6361eaad7b63b44dd7833e63@mail.teambition.com>

CR-link: https://codeup.aliyun.com/60c06368dbbed34fe0f0c263/job/hr_job-bytedance/change/2

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/controller/BytedanceController.java

@@ -1,25 +1,17 @@
 package cn.com.ctop.job.bytedance.controller;
 
 import cn.com.ctop.job.bytedance.data.service.IAdminAdvertiserListService;
-import cn.com.ctop.job.bytedance.data.service.IBytedancePromoterReportService;
-import cn.com.ctop.job.bytedance.data.service.IBytedanceRefreshTokenService;
-import cn.com.ctop.job.bytedance.data.service.IChannelService;
+import cn.com.ctop.job.bytedance.data.service.IBytedanceProductReportService;
 import cn.com.ctop.job.bytedance.data.utils.Check;
 import cn.com.ctop.job.bytedance.data.utils.DateUtils;
 import com.alibaba.fastjson.JSONObject;
-import com.xxl.job.core.context.XxlJobHelper;
 import io.swagger.annotations.Api;
 import lombok.extern.slf4j.Slf4j;
-
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import java.util.List;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 
@@ -34,17 +26,17 @@ import java.util.concurrent.Executors;
 @Api(tags = "头条")
 @Slf4j
 public class BytedanceController {
+    @Autowired
+    private IBytedanceProductReportService productReportService;
 
-    private static ExecutorService dailyRetrExecutorService = Executors.newFixedThreadPool(5);
+    private static final ExecutorService dailyRetrExecutorService = Executors.newFixedThreadPool(5);
 
     @Autowired
     private IAdminAdvertiserListService adminAdvertiserListService;
 
 
-
     @GetMapping("/getChannel")
     public void getChannel(String param) {
-
         String[] s = param.split("_");
         Long advertiserId = Long.valueOf(s[0]);
         String accessToken = s[1];
@@ -58,9 +50,16 @@ public class BytedanceController {
                 log.info("字节多线程获取账户返回结果：{}", result);
             }
         }
+    }
 
-
-
+    @GetMapping("/productData")
+    public void productData(String param) {
+        String[] s = param.split("_");
+        Long advertiserId = Long.valueOf(s[0]);
+        String accessToken = s[1];
+        String nowDate = DateUtils.getNowDate("yyyy-MM-dd");
+        String data = DateUtils.getAnotherDay("yyyy-MM-dd", nowDate, -1);
+        productReportService.getBytedanceProductDailyReport(advertiserId, accessToken, data, data, 1, null, 1, null);
 
     }
 

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/entity/BytedanceProductReportDaily.java

@@ -0,0 +1,23 @@
+package cn.com.ctop.job.bytedance.data.entity;
+
+import lombok.Data;
+
+@Data
+public class BytedanceProductReportDaily {
+    private Long advertiserId;
+    private Integer statTimeDay;
+    private Long productId;
+    private Long cdpProjectId;
+    private Long cdpPromotionId;
+    private String adPlatformProductName;
+    private Long productPlatformId;
+    private String adPlatformProductPlatformName;
+    private String adPlatformProductFirstCategoryName;
+    private String adPlatformProductSubCategoryName;
+    private String adPlatformProductThirdCategoryName;
+    private Double statCost;
+    private Long showCnt;
+    private Long clickCnt;
+    private Long convertCnt;
+    private Long deepConvertCnt;
+}

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/mapper/BytedanceProductReportMapper.java

@@ -0,0 +1,10 @@
+package cn.com.ctop.job.bytedance.data.mapper;
+
+
+import cn.com.ctop.job.bytedance.data.entity.BytedanceProductReportDaily;
+
+import java.util.List;
+
+public interface BytedanceProductReportMapper {
+    void replaceProductReportDailies(List<BytedanceProductReportDaily> reports);
+}

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/mapper/xml/BytedanceProductReportMapper.xml

@@ -0,0 +1,47 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
+<mapper namespace="cn.com.ctop.job.bytedance.data.mapper.BytedanceProductReportMapper">
+
+    <insert id="replaceProductReportDailies">
+        replace into media_api.bytedance_product_report_daily (
+        advertiser_id,
+        stat_time_day,
+        product_id,
+        cdp_project_id,
+        cdp_promotion_id,
+        ad_platform_product_name,
+        product_platform_id,
+        ad_platform_product_platform_name,
+        ad_platform_product_first_category_name,
+        ad_platform_product_sub_category_name,
+        ad_platform_product_third_category_name,
+        stat_cost,
+        show_cnt,
+        click_cnt,
+        convert_cnt,
+        deep_convert_cnt
+        )
+        values
+        <foreach collection="list" item="item" separator=",">
+            (
+            #{item.advertiserId},
+            #{item.statTimeDay},
+            #{item.productId},
+            #{item.cdpProjectId},
+            #{item.cdpPromotionId},
+            #{item.adPlatformProductName},
+            #{item.productPlatformId},
+            #{item.adPlatformProductPlatformName},
+            #{item.adPlatformProductFirstCategoryName},
+            #{item.adPlatformProductSubCategoryName},
+            #{item.adPlatformProductThirdCategoryName},
+            #{item.statCost},
+            #{item.showCnt},
+            #{item.clickCnt},
+            #{item.convertCnt},
+            #{item.deepConvertCnt}
+            )
+        </foreach>
+    </insert>
+
+</mapper>

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/IBytedanceProductReportService.java

@@ -0,0 +1,6 @@
+package cn.com.ctop.job.bytedance.data.service;
+
+public interface IBytedanceProductReportService {
+    void getBytedanceProductDailyReport(Long advertiserId, String accessToken, String startDate, String endDate, Integer page,
+                                        Long errorId, Integer isLog, Integer isSend);
+}

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/impl/BytedanceProductReportServiceImpl.java

@@ -0,0 +1,122 @@
+package cn.com.ctop.job.bytedance.data.service.impl;
+
+import cn.com.ctop.job.bytedance.data.entity.BytedanceErrorLog;
+import cn.com.ctop.job.bytedance.data.entity.BytedanceProductReportDaily;
+import cn.com.ctop.job.bytedance.data.mapper.BytedanceProductReportMapper;
+import cn.com.ctop.job.bytedance.data.mapper.BytedancePromoterReportMapper;
+import cn.com.ctop.job.bytedance.data.service.IBytedanceProductReportService;
+import cn.com.ctop.job.bytedance.data.utils.AlertUtils;
+import cn.com.ctop.job.bytedance.data.utils.Check;
+import cn.com.ctop.job.bytedance.data.utils.FieldsConfig;
+import cn.com.ctop.job.bytedance.data.utils.HttpUtils;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@Service
+@Slf4j
+public class BytedanceProductReportServiceImpl implements IBytedanceProductReportService {
+    @Autowired
+    private BytedancePromoterReportMapper promoterReportMapper;
+
+    @Autowired
+    private BytedanceProductReportMapper productReportMapper;
+
+    @Override
+    public void getBytedanceProductDailyReport(Long advertiserId, String accessToken, String startDate, String endDate, Integer page,
+                                               Long errorId, Integer isLog, Integer isSend) {
+        try {
+            String url = "https://api.oceanengine.com/open_api/v3.0/report/custom/get/";
+            Map<String, Object> paramsMap = new HashMap<>();
+            paramsMap.put("advertiser_id", advertiserId);
+            paramsMap.put("start_time", startDate);
+            paramsMap.put("end_time", endDate);
+            paramsMap.put("data_topic", "PRODUCT_DATA");
+            paramsMap.put("metrics", FieldsConfig.getProductMetrics());
+            paramsMap.put("dimensions", FieldsConfig.getProductDailyDimensions());
+            paramsMap.put("order_by", new JSONArray());
+            paramsMap.put("filters", new JSONArray());
+            paramsMap.put("page", page);
+            paramsMap.put("page_size", 100);
+            Map<String, String> headers = new HashMap<>();
+            headers.put("Access-Token", accessToken);
+            headers.put("Content-Type", "application/json");
+            String s = HttpUtils.httpGet(url, paramsMap, headers);
+            JSONObject result = JSONObject.parseObject(s);
+            if (Check.isNull(result)) {
+                log.info("账户id:{}，产品数据日报请求结果返回为空", advertiserId);
+                insertErrorLog(isLog, advertiserId, startDate, endDate);
+                return;
+            }
+            List<BytedanceProductReportDaily> reportDailies = new ArrayList<>();
+            Integer code = result.getInteger("code");
+            int totalPage = 1;
+            if (code == 0) {
+                JSONObject data = result.getJSONObject("data");
+                if (!Check.isNull(data)) {
+                    totalPage = data.getJSONObject("page_info").getIntValue("total_page");
+                    JSONArray rows = data.getJSONArray("rows");
+                    if (!Check.isNull(rows)) {
+                        for (int i = 0; i < rows.size(); i++) {
+                            JSONObject rowJson = rows.getJSONObject(i);
+                            JSONObject metricsJson = rowJson.getJSONObject("metrics");
+                            JSONObject dimensionsJson = rowJson.getJSONObject("dimensions");
+                            metricsJson.putAll(dimensionsJson);
+                            metricsJson.put("stat_time_day", dimensionsJson.getString("stat_time_day").replace("-", ""));
+                            BytedanceProductReportDaily reportDaily = JSONObject.parseObject(metricsJson.toJSONString(), BytedanceProductReportDaily.class);
+                            reportDaily.setAdvertiserId(advertiserId);
+                            reportDailies.add(reportDaily);
+                        }
+                    }
+                }
+            } else {
+                log.info("账户id:{}，日期:{}产品数据日报请求结果失败，返回数据:{}", advertiserId, startDate, result);
+                insertErrorLog(isLog, advertiserId, startDate, endDate);
+                return;
+            }
+            if (!Check.isNull(reportDailies)) {
+                productReportMapper.replaceProductReportDailies(reportDailies);
+                log.info("账户ID：{}，产品数据日报请求成功,开始日期：{}，结束日期：{}，page:{}", advertiserId, startDate, endDate, page);
+            }
+            if (page < totalPage) {
+                try {
+                    Thread.sleep(500L);
+                } catch (InterruptedException e) {
+                    throw new RuntimeException(e);
+                }
+                getBytedanceProductDailyReport(advertiserId, accessToken, startDate, endDate, page + 1, errorId, isLog, isSend);
+            } else {
+                log.info("账户ID：{}，日期:{},产品数据日报请求成功", advertiserId, startDate);
+                if (errorId != null) {
+                    promoterReportMapper.deleteInfoByErrorId(errorId);
+                }
+                if (isSend != null && 1 == isSend) {
+                    AlertUtils.sendMessage(advertiserId, startDate, endDate, 1, "product_daily", 1);
+                }
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    private void insertErrorLog(Integer isLog, Long advertiserId, String startDate, String endDate) {
+        if (isLog != null && 1 == isLog) {
+            BytedanceErrorLog errorInfo = new BytedanceErrorLog();
+            errorInfo.setAdvertiserId(advertiserId);
+            errorInfo.setStartDate(startDate);
+            errorInfo.setEndDate(endDate);
+            errorInfo.setDataDimension("product");
+            errorInfo.setDataType("daily");
+            promoterReportMapper.replaceErrorInfo(errorInfo);
+        }
+    }
+
+
+}

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/impl/BytedancePromoterReportServiceImpl.java

@@ -8,6 +8,7 @@ import cn.com.ctop.job.bytedance.data.service.IBytedancePromoterReportService;
 import cn.com.ctop.job.bytedance.data.utils.Check;
 import cn.com.ctop.job.bytedance.data.utils.FieldsConfig;
 import cn.com.ctop.job.bytedance.data.utils.HttpUtils;
+import cn.com.ctop.job.bytedance.data.utils.AlertUtils;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import lombok.extern.slf4j.Slf4j;
@@ -18,16 +19,12 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
 
 @Service
 @Slf4j
 public class BytedancePromoterReportServiceImpl implements IBytedancePromoterReportService {
     @Autowired
     private BytedancePromoterReportMapper promoterReportMapper;
-    private static String sendUrl = "http://172.16.0.108:8765/webhook/media_report";
-    private static ExecutorService sendExecutorService = Executors.newFixedThreadPool(5);
 
     @Override
     public List<JSONObject> getAccountList() {
@@ -106,7 +103,7 @@ public class BytedancePromoterReportServiceImpl implements IBytedancePromoterRep
                 log.info("账户ID：{}，日期:{},广告时报（最细粒度）数据请求成功", advertiserId, startDate);
                 promoterReportMapper.deleteInfoByErrorId(errorId);
                 if (isSend == 1) {
-                    sendMessage(advertiserId, startDate, endDate, 1, "hour", 1);
+                    AlertUtils.sendMessage(advertiserId, startDate, endDate, 1, "hour", 1);
                 }
             }
         } catch (Exception e) {
@@ -186,7 +183,7 @@ public class BytedancePromoterReportServiceImpl implements IBytedancePromoterRep
                 log.info("账户ID：{}，日期:{},广告日报（最细粒度）数据请求成功", advertiserId, startDate);
                 promoterReportMapper.deleteInfoByErrorId(errorId);
                 if (isSend == 1) {
-                    sendMessage(advertiserId, startDate, endDate, 1, "daily", 1);
+                    AlertUtils.sendMessage(advertiserId, startDate, endDate, 1, "daily", 1);
                 }
             }
         } catch (Exception e) {
@@ -400,33 +397,4 @@ public class BytedancePromoterReportServiceImpl implements IBytedancePromoterRep
         return promoterReportMapper.getAllList();
     }
 
-    private static void sendMessage(Long account_id, String start_date, String end_date, Integer media_type, String type, Integer count) {
-        if (count > 5) {
-            log.error("超过最大次数，account_id：{}", count);
-            return;
-        }
-        sendExecutorService.submit(new Runnable() {
-            @Override
-            public void run() {
-                Map<String, Object> param = new HashMap<>();
-                param.put("account_id", account_id);
-                param.put("start_date", start_date);
-                param.put("end_date", end_date);
-                param.put("media_type", media_type);
-                param.put("type", type);
-                String s = HttpUtils.httpPostRequest(sendUrl, param, new HashMap<>());
-                JSONObject jsonObject = JSONObject.parseObject(s);
-                if (!Check.isNull(jsonObject)) {
-                    Integer code = jsonObject.getInteger("code");
-                    if (code == 200) {
-                        log.info("请求数据清洗服务成功,account_id:{}", account_id);
-
-                    } else {
-                        sendMessage(account_id, start_date, end_date, media_type, type, count + 1);
-                    }
-                }
-            }
-        });
-    }
-
 }

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/utils/AlertUtils.java

@@ -0,0 +1,43 @@
+package cn.com.ctop.job.bytedance.data.utils;
+
+import com.alibaba.fastjson.JSONObject;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+@Slf4j
+public class AlertUtils {
+    private static final String sendUrl = "http://172.16.0.108:8765/webhook/media_report";
+    private static final ExecutorService sendExecutorService = Executors.newFixedThreadPool(5);
+
+    public static void sendMessage(Long account_id, String start_date, String end_date, Integer media_type, String type, Integer count) {
+        if (count > 5) {
+            log.error("超过最大次数，account_id：{}", count);
+            return;
+        }
+        sendExecutorService.submit(new Runnable() {
+            @Override
+            public void run() {
+                Map<String, Object> param = new HashMap<>();
+                param.put("account_id", account_id);
+                param.put("start_date", start_date);
+                param.put("end_date", end_date);
+                param.put("media_type", media_type);
+                param.put("type", type);
+                String s = HttpUtils.httpPostRequest(sendUrl, param, new HashMap<>());
+                JSONObject jsonObject = JSONObject.parseObject(s);
+                if (!Check.isNull(jsonObject)) {
+                    Integer code = jsonObject.getInteger("code");
+                    if (code == 200) {
+                        log.info("请求数据清洗服务成功,account_id:{}", account_id);
+                    } else {
+                        sendMessage(account_id, start_date, end_date, media_type, type, count + 1);
+                    }
+                }
+            }
+        });
+    }
+}

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/utils/FieldsConfig.java

@@ -54,4 +54,30 @@ public class FieldsConfig {
         return metricslds;
 
     }
+
+    public static JSONArray getProductMetrics() {
+        JSONArray metricslds = new JSONArray();
+        metricslds.add("stat_cost");
+        metricslds.add("show_cnt");
+        metricslds.add("click_cnt");
+        metricslds.add("convert_cnt");
+        metricslds.add("deep_convert_cnt");
+        return metricslds;
+    }
+
+    public static JSONArray getProductDailyDimensions() {
+        JSONArray houtDimensions = new JSONArray();
+        houtDimensions.add("product_id");
+        houtDimensions.add("ad_platform_product_name");
+        houtDimensions.add("stat_time_day");
+        houtDimensions.add("cdp_project_id");
+        houtDimensions.add("cdp_promotion_id");
+        houtDimensions.add("product_platform_id");
+        houtDimensions.add("ad_platform_product_platform_name");
+        houtDimensions.add("ad_platform_product_first_category_name");
+        houtDimensions.add("ad_platform_product_sub_category_name");
+        houtDimensions.add("ad_platform_product_third_category_name");
+        return houtDimensions;
+
+    }
 }

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedancePromoterReportJob.java

@@ -1,6 +1,7 @@
 package cn.com.ctop.job.bytedance.handler;
 
 
+import cn.com.ctop.job.bytedance.data.service.IBytedanceProductReportService;
 import cn.com.ctop.job.bytedance.data.service.IBytedancePromoterReportService;
 import cn.com.ctop.job.bytedance.data.service.IChannelService;
 import cn.com.ctop.job.bytedance.data.utils.Check;
@@ -26,6 +27,9 @@ import java.util.concurrent.Executors;
 @Component
 public class BytedancePromoterReportJob {
     @Autowired
+    private IBytedanceProductReportService productReportService;
+
+    @Autowired
     private IBytedancePromoterReportService promoterReportService;
     @Autowired
     private IChannelService channelService;
@@ -36,6 +40,10 @@ public class BytedancePromoterReportJob {
     private static ExecutorService dailyRetrExecutorService = Executors.newFixedThreadPool(2);
     private static ExecutorService hourRetrExecutorService = Executors.newFixedThreadPool(2);
 
+    private static ExecutorService productDailyExecutor = Executors.newFixedThreadPool(2);
+    private static ExecutorService productDailyRetrExecutor = Executors.newFixedThreadPool(2);
+
+
     @XxlJob("promoterReport")
     public void promoterReport() throws Exception {
         List<JSONObject> accountList = promoterReportService.getAccountList();
@@ -57,7 +65,7 @@ public class BytedancePromoterReportJob {
                             @Override
                             public void run() {
                                 if (!Check.isNull(accessToken)) {
-                                    promoterReportService.getBytedanceRetryPromoterHourReport(id, advertiserId, accessToken, startDate, endDate, 1,isSend);
+                                    promoterReportService.getBytedanceRetryPromoterHourReport(id, advertiserId, accessToken, startDate, endDate, 1, isSend);
                                 }
                             }
                         });
@@ -67,15 +75,22 @@ public class BytedancePromoterReportJob {
                             @Override
                             public void run() {
                                 if (!Check.isNull(accessToken)) {
-                                    promoterReportService.getBytedancePromoterRetryDailyReport(id, advertiserId, accessToken, startDate, endDate, 1,isSend);
+                                    promoterReportService.getBytedancePromoterRetryDailyReport(id, advertiserId, accessToken, startDate, endDate, 1, isSend);
 
                                 }
                             }
                         });
                     }
+                } else if ("product".equals(dataDimension)) {
+                    productDailyRetrExecutor.submit(new Runnable() {
+                        @Override
+                        public void run() {
+                            if (!Check.isNull(accessToken)) {
+                                productReportService.getBytedanceProductDailyReport(advertiserId, accessToken, startDate, endDate, 1, id, null, isSend);
+                            }
+                        }
+                    });
                 }
-
-
             }
 
         } else {
@@ -96,7 +111,7 @@ public class BytedancePromoterReportJob {
             JSONObject jsonObject = channelList.get(i);
             String channelAccountId = jsonObject.getString("channelAccountId");
             String secretKey = jsonObject.getString("secretKey");
-            String startDate = DateUtils.getNowAddDate("yyyy-MM-dd",-1);
+            String startDate = DateUtils.getNowAddDate("yyyy-MM-dd", -1);
             String endDate = DateUtils.getNowDate("yyyy-MM-dd");
             channelService.getPromotionIdList(startDate, endDate, channelAccountId, secretKey, 0);
         }
@@ -118,15 +133,14 @@ public class BytedancePromoterReportJob {
         String accessToken = s[1];
 
         hourExecutorService.submit(new Runnable() {
-                    @Override
-                    public void run() {
-                        promoterReportService.getBytedancePromoterHour(advertiserId, accessToken, date, date, 1, 1);
-                    }
-                });
+            @Override
+            public void run() {
+                promoterReportService.getBytedancePromoterHour(advertiserId, accessToken, date, date, 1, 1);
+            }
+        });
     }
 
 
-
     @XxlJob("yesReportPromoterReport")
     public void yesReportPromoterReport() throws Exception {
         String param = XxlJobHelper.getJobParam(); // 执行参数
@@ -149,5 +163,27 @@ public class BytedancePromoterReportJob {
         });
     }
 
+    @XxlJob("productReport")
+    public void productReport() throws Exception {
+        String param = XxlJobHelper.getJobParam(); // 执行参数
+        if (Check.isNull(param)) {
+            log.error("入参为空");
+            return;
+        }
+
+        String nowDate = DateUtils.getNowDate("yyyy-MM-dd");
+        String data = DateUtils.getAnotherDay("yyyy-MM-dd", nowDate, -1);
+        String[] s = param.split("_");
+        Long advertiserId = Long.valueOf(s[0]);
+        String accessToken = s[1];
+
+        productDailyExecutor.submit(new Runnable() {
+            @Override
+            public void run() {
+                productReportService.getBytedanceProductDailyReport(advertiserId, accessToken, data, data, 1, null, 1, null);
+            }
+        });
+    }
+
 
 }