Parcourir la source

Merge #18 into master from pingkang

素材日报

* pingkang: (2 commits)
  素材日报
  素材日报

Signed-off-by: 1129979389@qq.com <accounts_6361eaad7b63b44dd7833e63@mail.teambition.com>
Merged-by: 1129979389@qq.com <accounts_6361eaad7b63b44dd7833e63@mail.teambition.com>

CR-link: https://codeup.aliyun.com/60c06368dbbed34fe0f0c263/job/hr_job-adq/change/18
huangpingkang il y a 10 mois
Parent
commit
d668fc24fc

+ 23 - 1
job-adq/src/main/java/cn/com/ctop/job/adq/controller/AdqController.java

@@ -7,7 +7,6 @@ import cn.com.ctop.job.adq.data.utils.DateUtils;
 import com.alibaba.fastjson.JSONObject;
 import io.swagger.annotations.Api;
 import lombok.extern.slf4j.Slf4j;
-
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.RequestMapping;
@@ -47,6 +46,8 @@ public class AdqController {
     @Autowired
     private IAppBanWarnService appBanWarnService;
 
+    @Autowired
+    private IAdqMaterialReportService adqMaterialReportService;
 
 
     @GetMapping("/getReport")
@@ -163,4 +164,25 @@ public class AdqController {
     }
 
 
+    @GetMapping("/materialDaily")
+    public void materialDaily(String startDate, String endDate) {
+        Integer startDateInt = Integer.valueOf(startDate.replace("-", ""));
+        Integer endDateInt = Integer.valueOf(endDate.replace("-", ""));
+        List<JSONObject> dataJson = adqReportService.getAccountListByDate(startDateInt, endDateInt);
+        if (!Check.isNull(dataJson)) {
+            for (int i = 0; i < dataJson.size(); i++) {
+                JSONObject jsonObject = dataJson.get(i);
+                Long advertiserId = jsonObject.getLong("advertiserId");
+                String accessToken = jsonObject.getString("accessToken");
+                dailyExecutorService.submit(new Runnable() {
+                    @Override
+                    public void run() {
+                        adqMaterialReportService.getAdqMaterialDailyReport(advertiserId, accessToken, startDate, endDate, 1, 1, null, 0);
+                    }
+                });
+            }
+        }
+    }
+
+
 }

+ 2 - 0
job-adq/src/main/java/cn/com/ctop/job/adq/data/mapper/AdqErrorLogMapper.java

@@ -2,6 +2,7 @@ package cn.com.ctop.job.adq.data.mapper;
 
 import cn.com.ctop.job.adq.data.entity.AdqErrorLog;
 import com.alibaba.fastjson.JSONObject;
+import com.baomidou.dynamic.datasource.annotation.DS;
 import org.apache.ibatis.annotations.Param;
 
 import java.util.List;
@@ -11,5 +12,6 @@ public interface AdqErrorLogMapper {
     void deleteInfoByErrorId(@Param("errorId") Long errorId);
     List<JSONObject> getErrorLogs();
 
+    @DS("selectdb")
     List<JSONObject> getAccountListByDate(@Param("startDate") Integer startDate, @Param("endDate") Integer endDate);
 }

+ 15 - 10
job-adq/src/main/java/cn/com/ctop/job/adq/data/mapper/xml/AdqErrorLogMapper.xml

@@ -34,17 +34,22 @@
                  LEFT JOIN deepthink.adq_advertiser_list t2 ON t1.advertiser_id = t2.advertiser_id
                  LEFT JOIN deepthink.adq_auth_list t3 ON t2.admin_advertiser_id = t3.admin_advertiser_id
     </select>
+
     <select id="getAccountListByDate" resultType="com.alibaba.fastjson.JSONObject">
-        select t1.advertiser_id as 'advertiserId', t3.access_token as 'accessToken'
-        from media_api.adq_advertiser_report_daily t1
-                 left join deepthink.adq_advertiser_list t2
-                           on t1.advertiser_id = t2.advertiser_id
-                 left join deepthink.adq_auth_list t3
-                           on t2.admin_advertiser_id = t3.admin_advertiser_id
-        where t1.stat_time_day >= #{startDate}
-          and t1.stat_time_day &lt;= #{endDate}
-          and t1.cost > 0
-        group by t1.advertiser_id
+        SELECT t1.advertiser_id AS 'advertiserId', t3.access_token AS 'accessToken'
+        FROM (
+        select advertiser_id from media_api.adq_creative_report_hour
+        where stat_time_day >= #{startDate} and stat_time_day &lt;= #{endDate}
+        group by 1
+
+        union
+
+        select advertiser_id from media_api.adq_creative_report_daily
+        where stat_time_day >= #{startDate} and stat_time_day &lt;= #{endDate}
+        group by 1
+        ) t1
+        LEFT JOIN tidb.deepthink.adq_advertiser_list t2 ON t1.advertiser_id = t2.advertiser_id
+        LEFT JOIN tidb.deepthink.adq_auth_list t3 ON t2.admin_advertiser_id = t3.admin_advertiser_id
     </select>
 
 </mapper>

+ 6 - 0
job-adq/src/main/java/cn/com/ctop/job/adq/data/service/IAdqMaterialReportService.java

@@ -0,0 +1,6 @@
+package cn.com.ctop.job.adq.data.service;
+
+public interface IAdqMaterialReportService {
+    void getAdqMaterialDailyReport(Long advertiserId, String accessToken, String startDate, String endDate, Integer page, Integer isLog, Long errId, Integer isSend);
+    void getAdqMaterialDailyReport(Long advertiserId, String accessToken, String level, String startDate, String endDate, Integer page, Integer isLog, Long errId, Integer isSend);
+}

+ 136 - 0
job-adq/src/main/java/cn/com/ctop/job/adq/data/service/impl/AdqMaterialReportServiceImpl.java

@@ -0,0 +1,136 @@
+package cn.com.ctop.job.adq.data.service.impl;
+
+import cn.com.ctop.job.adq.data.entity.AdqErrorLog;
+import cn.com.ctop.job.adq.data.mapper.AdqErrorLogMapper;
+import cn.com.ctop.job.adq.data.service.IAdqMaterialReportService;
+import cn.com.ctop.job.adq.data.utils.AlertUtils;
+import cn.com.ctop.job.adq.data.utils.Check;
+import cn.com.ctop.job.adq.data.utils.FieldsConfig;
+import cn.com.ctop.job.adq.data.utils.HttpUtils;
+import cn.com.ctop.job.adq.utils.RandomStringGenerator;
+import cn.com.ctop.job.adq.utils.doris.sink.DorisMultiTableStreamLoadManager;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.Map;
+
+@Slf4j
+@Service
+public class AdqMaterialReportServiceImpl implements IAdqMaterialReportService {
+    @Autowired
+    private DorisMultiTableStreamLoadManager dorisLoadManager;
+    @Autowired
+    private AdqErrorLogMapper adqErrorLogMapper;
+
+    @Override
+    public void getAdqMaterialDailyReport(Long advertiserId, String accessToken, String startDate, String endDate, Integer page, Integer isLog, Long errId, Integer isSend) {
+        getAdqMaterialDailyReport(advertiserId, accessToken, "REPORT_LEVEL_MATERIAL_IMAGE", startDate, endDate, page, isLog, errId, isSend);
+        getAdqMaterialDailyReport(advertiserId, accessToken, "REPORT_LEVEL_MATERIAL_VIDEO", startDate, endDate, page, isLog, errId, isSend);
+    }
+
+    /*
+    level  REPORT_LEVEL_MATERIAL_IMAGE REPORT_LEVEL_MATERIAL_VIDEO
+     */
+    @Override
+    public void getAdqMaterialDailyReport(Long advertiserId, String accessToken, String level, String startDate, String endDate, Integer page, Integer isLog, Long errId, Integer isSend) {
+        try {
+            Map<String, Object> paramsMap = new HashMap<>();
+            paramsMap.put("access_token", accessToken);
+            long timestamp = Instant.now().getEpochSecond();
+            paramsMap.put("timestamp", String.valueOf(timestamp));
+            paramsMap.put("nonce", RandomStringGenerator.generateRandomString(32));
+            paramsMap.put("page", page);
+            paramsMap.put("page_size", 1000);
+            paramsMap.put("account_id", advertiserId);
+            paramsMap.put("level", level);
+            JSONObject date_range = new JSONObject();
+            date_range.put("start_date", startDate);
+            date_range.put("end_date", endDate);
+            paramsMap.put("date_range", date_range);
+            paramsMap.put("time_line", "REPORTING_TIME");
+            JSONArray group_by = new JSONArray();
+            group_by.add("date");
+            group_by.add("adgroup_id");
+            JSONArray fieldsList = FieldsConfig.getMaterialDailyFieldsList();
+            if ("REPORT_LEVEL_MATERIAL_IMAGE".equals(level)) {
+                group_by.add("image_id");
+                fieldsList.add("image_id");
+            } else {
+                group_by.add("video_id");
+                fieldsList.add("video_id");
+            }
+            paramsMap.put("group_by", group_by);
+            paramsMap.put("fields", fieldsList);
+            String s = HttpUtils.httpGet("https://api.e.qq.com/v3.0/daily_reports/get", paramsMap, null);
+            JSONObject jsonObject = JSONObject.parseObject(s);
+            int totalPage = 1;
+            if (Check.isNull(jsonObject)) {
+                log.error("请求素材日报失败,account_id:{},返回为空。", advertiserId);
+                if (isLog == 1) {
+                    AdqErrorLog errorInfo = new AdqErrorLog();
+                    errorInfo.setAdvertiserId(advertiserId);
+                    errorInfo.setStartDate(startDate);
+                    errorInfo.setEndDate(endDate);
+                    errorInfo.setDataDimension("material");
+                    errorInfo.setDataType("daily");
+                    adqErrorLogMapper.replaceErrorInfo(errorInfo);
+                }
+                return;
+            } else {
+                Integer code = jsonObject.getInteger("code");
+                if (code == 0) {
+                    JSONObject data = jsonObject.getJSONObject("data");
+                    if (!Check.isNull(data)) {
+                        totalPage = data.getJSONObject("page_info").getIntValue("total_page");
+                        JSONArray list = data.getJSONArray("list");
+                        if (!Check.isNull(list)) {
+                            for (int i = 0; i < list.size(); i++) {
+                                JSONObject dataJson = list.getJSONObject(i);
+                                dataJson.put("stat_time_day", dataJson.getString("date").replace("-", ""));
+                                dataJson.put("advertiser_id", advertiserId);
+                                if ("REPORT_LEVEL_MATERIAL_IMAGE".equals(level)) {
+                                    dataJson.put("material_id", dataJson.getString("image_id"));
+                                    dataJson.put("material_type", "IMAGE");
+                                } else {
+                                    dataJson.put("material_id", dataJson.getString("video_id"));
+                                    dataJson.put("material_type", "VIDEO");
+                                }
+                                dorisLoadManager.add("media_api", "adq_material_report_daily", dataJson);
+                            }
+                        }
+                    }
+                } else {
+                    log.error("请求素材日报失败,account_id:{},返回信息:{}", advertiserId, jsonObject);
+                    if (isLog == 1) {
+                        AdqErrorLog errorInfo = new AdqErrorLog();
+                        errorInfo.setAdvertiserId(advertiserId);
+                        errorInfo.setStartDate(startDate);
+                        errorInfo.setEndDate(endDate);
+                        errorInfo.setDataDimension("material");
+                        errorInfo.setDataType("daily");
+                        adqErrorLogMapper.replaceErrorInfo(errorInfo);
+                    }
+                    return;
+                }
+            }
+            if (page < totalPage) {
+                getAdqMaterialDailyReport(advertiserId, accessToken, level, startDate, endDate, page + 1, isLog, errId, isSend);
+            } else {
+                log.info("请求创意日报完成,account_id:{},开始日期:{},结束日期:{} totalPage:{}", advertiserId, startDate, endDate, totalPage);
+                if (errId != null) {
+                    adqErrorLogMapper.deleteInfoByErrorId(errId);
+                }
+                if (isSend != null && isSend == 1) {
+                    AlertUtils.sendMessage(advertiserId, startDate, endDate, 3, "daily", 1, "asset");
+                }
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+}

+ 30 - 0
job-adq/src/main/java/cn/com/ctop/job/adq/data/utils/FieldsConfig.java

@@ -130,6 +130,36 @@ public class FieldsConfig {
 
     }
 
+    public static JSONArray getMaterialDailyFieldsList(){
+        JSONArray fields = new JSONArray();
+        fields.add("account_id");
+        fields.add("adgroup_id");
+        fields.add("adgroup_name");
+        fields.add("date");
+        fields.add("cost");
+        fields.add("view_count");
+        fields.add("valid_click_count");
+        fields.add("conversions_count");
+        fields.add("deep_conversions_count");
+        fields.add("activated_count");
+        fields.add("reg_pv");
+        fields.add("retention_count");
+        fields.add("video_outer_play_count");
+        fields.add("video_outer_play100_count");
+        fields.add("key_behavior_conversions_count");
+        fields.add("purchase_pv");
+        fields.add("purchase_amount");
+        fields.add("first_pay_count");
+        fields.add("ad_monetization_amount"); //广告变现金额。
+        fields.add("app_ad_paying_users"); //广告变现人数
+        fields.add("income_pv_pla"); //广告变现次数
+        fields.add("income_val_1"); //激活首日广告变现金额
+        fields.add("ad_paying_users_d1"); //激活首日广告变现人数
+        fields.add("income_pv_1d_pla"); //注册首日广告变现次数
+        fields.add("income_val_3"); //注册首日广告变现次数
+        return fields;
+
+    }
 
 
 }

+ 22 - 0
job-adq/src/main/java/cn/com/ctop/job/adq/handler/AdqAccountReportJob.java

@@ -4,6 +4,7 @@ package cn.com.ctop.job.adq.handler;
 import cn.com.ctop.job.adq.data.service.IAdqAccountReportService;
 import cn.com.ctop.job.adq.data.service.IAdqAdIncomeService;
 import cn.com.ctop.job.adq.data.service.IAdqAssetReportService;
+import cn.com.ctop.job.adq.data.service.IAdqMaterialReportService;
 import cn.com.ctop.job.adq.data.utils.Check;
 import cn.com.ctop.job.adq.data.utils.DateUtils;
 import com.alibaba.fastjson.JSONObject;
@@ -32,6 +33,8 @@ public class AdqAccountReportJob {
     private IAdqAccountReportService accountReportService;
     @Autowired
     private IAdqAdIncomeService adqAdIncomeService;
+    @Autowired
+    private IAdqMaterialReportService adqMaterialReportService;
     private static ExecutorService nowDateHourExecutorService = Executors.newFixedThreadPool(5);
     private static ExecutorService yesDateHourExecutorService = Executors.newFixedThreadPool(5);
     private static ExecutorService dailyExecutorService = Executors.newFixedThreadPool(5);
@@ -134,4 +137,23 @@ public class AdqAccountReportJob {
         }
     }
 
+    @XxlJob("yesMaterialDaily")
+    public void yesMaterialDaily() throws Exception {
+        String param = XxlJobHelper.getJobParam(); // 执行参数
+        if (Check.isNull(param)) {
+            log.error("入参为空");
+            return;
+        }
+        String nowDate = DateUtils.getNowDate("yyyy-MM-dd");
+        String yesDate = DateUtils.getAnotherDay("yyyy-MM-dd", nowDate, -1);
+        String[] s = param.split("_");
+        Long advertiserId = Long.valueOf(s[0]);
+        String accessToken = s[1];
+        dailyExecutorService.submit(new Runnable() {
+            @Override
+            public void run() {
+                adqMaterialReportService.getAdqMaterialDailyReport(advertiserId, accessToken, yesDate, yesDate, 1, 1, null, 0);
+            }
+        });
+    }
 }

+ 11 - 1
job-adq/src/main/java/cn/com/ctop/job/adq/handler/AdqRetryReportJob.java

@@ -10,7 +10,6 @@ import com.xxl.job.core.handler.annotation.XxlJob;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
-import org.springframework.web.bind.annotation.GetMapping;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -50,6 +49,8 @@ public class AdqRetryReportJob {
     private IAdqFlowDetailsReportService adqFlowDetailsReportService;
     @Autowired
     private IAdqAccountReportService accountReportService;
+    @Autowired
+    private IAdqMaterialReportService adqMaterialReportService;
 
     @XxlJob("adqRetryReport")
     public void promoterReport() throws Exception {
@@ -147,6 +148,15 @@ public class AdqRetryReportJob {
 
                         }
                     });
+                } else if ("material".equals(dataDimension)) {
+                    assetExecutorService.submit(new Runnable() {
+                        @Override
+                        public void run() {
+                            if (!Check.isNull(accessToken)) {
+                                adqMaterialReportService.getAdqMaterialDailyReport(advertiserId, accessToken, startDate, endDate, 1, null, id, isSend);
+                            }
+                        }
+                    });
                 }