Merge #30 into master from pingkang

Merge branch 'master' of codeup.aliyun.com:60c06368dbbed34fe0f0c263/job/hr_job-...

* pingkang: (4 commits)
  广告组列表
  广告组列表
  广告组列表
  Merge branch 'master' of codeup.aliyun.com:60c06368dbbed34fe0f0c263/job/hr_job-...

Signed-off-by: 1129979389@qq.com <accounts_6361eaad7b63b44dd7833e63@mail.teambition.com>
Merged-by: 1129979389@qq.com <accounts_6361eaad7b63b44dd7833e63@mail.teambition.com>

CR-link: https://codeup.aliyun.com/60c06368dbbed34fe0f0c263/job/hr_job-kuaishou/change/30

job-kuaishou/src/main/java/cn/com/ctop/job/kuaishou/controller/KuaiShouController.java

@@ -88,12 +88,28 @@ public class KuaiShouController {
             for (JSONObject jsonObject : jsonObjects) {
                 Long advertiserId = jsonObject.getLong("advertiser_id");
                 String accessToken = jsonObject.getString("access_token");
-                accountReportService.queryAccountDailyFlows(accessToken, advertiserId, day, day, 1,1,null,2);
+                accountReportService.queryAccountDailyFlows(accessToken, advertiserId, day, day, 1, 1, null, 2);
 
             }
         }
 
     }
 
+    @GetMapping("/groupList")
+    public void groupList(String startDate, String endDate) {
+        Integer intStartDate = Integer.valueOf(startDate.replace("-", ""));
+        Integer intEndDate = Integer.valueOf(endDate.replace("-", ""));
+        List<JSONObject> jsonObjects = kuaiShouAccountReportMapper.queryCostAccount(intStartDate, intEndDate);
+        for (JSONObject jsonObject : jsonObjects) {
+            executorService.submit(new Runnable() {
+                @Override
+                public void run() {
+                    Long advertiserId = jsonObject.getLong("advertiser_id");
+                    String accessToken = jsonObject.getString("access_token");
+                    groupReportService.queryGroupList(accessToken, advertiserId, startDate, endDate, 1, 1, null, 2);
+                }
+            });
+        }
+    }
 
 }

job-kuaishou/src/main/java/cn/com/ctop/job/kuaishou/data/constant/KuaishouLinkConstants.java

@@ -23,4 +23,5 @@ public class KuaishouLinkConstants {
     public static final String AD_SOCIAL_CONTRACTOR_MISSION_LIST = "/rest/openapi/gw/ad_social/v1/transaction/contractor/mission/missionDetailList";
     public static final String AD_SOCIAL_CONTRACTOR_MISSION_INFO = "/rest/openapi/gw/ad_social/v1/transaction/contractor/mission/get";
     public static final String AD_SOCIAL_CONTRACTOR_MISSION_DATA = "/rest/openapi/gw/ad_social/v1/transaction/contractor/order/list";
+    public static final String GROUP_LIST = "/rest/openapi/gw/dsp/unit/list";
 }

job-kuaishou/src/main/java/cn/com/ctop/job/kuaishou/data/mapper/KuaiShouGroupReportMapper.java

@@ -21,5 +21,4 @@ public interface KuaiShouGroupReportMapper {
     List<Long> getAdvertiserIdList(@Param("type") Integer type);
 
     List<JSONObject> getAllAccount();
-
 }

job-kuaishou/src/main/java/cn/com/ctop/job/kuaishou/data/mapper/xml/KuaiShouAccountReportMapper.xml

@@ -283,13 +283,20 @@
     </update>
 
     <select id="queryCostAccount" resultType="com.alibaba.fastjson.JSONObject">
-        select r.advertiser_id,a.access_token
-        from media_api.kuaishou_advertiser_report_daily r
-                 join tidb.deepthink.kuaishou_advertiser_list l
-                      on r.advertiser_id = l.advertiser_id
-                 join tidb.deepthink.kuaishou_auth_list a
-                      on a.admin_advertiser_id = l.admin_advertiser_id
-        where stat_date >= #{startDate} and stat_date &lt;= #{endDate}
-        group by 1,2
+        SELECT t1.advertiser_id , t3.access_token
+        FROM (
+                 select advertiser_id from media_api.kuaishou_adgroup_report_hourly karh
+                 where stat_date >= #{startDate} and stat_date &lt;= #{endDate}
+                 group by 1
+
+                 union
+
+                 select advertiser_id from media_api.kuaishou_adgroup_report_daily kard
+                 where stat_date >= #{startDate} and stat_date &lt;= #{endDate}
+                 group by 1
+             ) t1
+                 LEFT JOIN tidb.deepthink.kuaishou_advertiser_list t2 ON t1.advertiser_id = t2.advertiser_id
+                 LEFT JOIN tidb.deepthink.kuaishou_auth_list t3 ON t2.admin_advertiser_id = t3.admin_advertiser_id
+
     </select>
 </mapper>

job-kuaishou/src/main/java/cn/com/ctop/job/kuaishou/data/service/IKuaiShouGroupReportService.java

@@ -13,4 +13,6 @@ public interface IKuaiShouGroupReportService {
     List<JSONObject> getAllAccount();
 
     void queryGroupReportDaily(String accessToken, Long advertiserId, String startDate, String endDate, int page);
+
+    void queryGroupList(String accessToken, Long advertiserId, String startDate, String endDate, int page, Integer isLog, Long errId, Integer isSend);
 }

job-kuaishou/src/main/java/cn/com/ctop/job/kuaishou/data/service/impl/KuaiShouGroupReportServiceImpl.java

@@ -4,8 +4,9 @@ import cn.com.ctop.job.kuaishou.data.constant.KuaishouLinkConstants;
 import cn.com.ctop.job.kuaishou.data.entity.KuaishouAdgroupReportDaily;
 import cn.com.ctop.job.kuaishou.data.entity.KuaishouAdgroupReportHourly;
 import cn.com.ctop.job.kuaishou.data.mapper.KuaiShouGroupReportMapper;
-import cn.com.ctop.job.kuaishou.data.service.IKuaiShouGroupReportService;
 import cn.com.ctop.job.kuaishou.data.service.IKuaiShouErrorLogService;
+import cn.com.ctop.job.kuaishou.data.service.IKuaiShouGroupReportService;
+import cn.com.ctop.job.kuaishou.data.utils.AlertUtils;
 import cn.com.ctop.job.kuaishou.data.utils.Check;
 import cn.com.ctop.job.kuaishou.data.utils.DateUtils;
 import cn.com.ctop.job.kuaishou.data.utils.HttpUtils;
@@ -15,12 +16,12 @@ import com.alibaba.fastjson.JSONObject;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.scheduling.annotation.EnableScheduling;
-import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Service;
 
-import javax.annotation.PreDestroy;
-import java.text.DecimalFormat;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 @Slf4j
 @Service
@@ -36,7 +37,6 @@ public class KuaiShouGroupReportServiceImpl implements IKuaiShouGroupReportServi
     private DorisMultiTableStreamLoadManager dorisLoadManager;
 
 
-
     @Override
     public void queryGroupReportHourly(String accessToken, Long advertiserId, String startDate, String endDate, int page) {
         try {
@@ -168,4 +168,64 @@ public class KuaiShouGroupReportServiceImpl implements IKuaiShouGroupReportServi
         }
     }
 
+    public void queryGroupList(String accessToken, Long advertiserId, String startDate, String endDate, int page, Integer isLog, Long errId, Integer isSend) {
+        String url = KuaishouLinkConstants.KS_PREFIX + KuaishouLinkConstants.GROUP_LIST;
+        Map<String, String> headers = new HashMap<>();
+        headers.put("Content-Type", "application/json");
+        headers.put("Access-Token", accessToken);
+        Map<String, Object> param = new HashMap<>();
+        if (!Check.isNull(startDate)) {
+            param.put("start_date", startDate);
+        }
+        if (!Check.isNull(endDate)) {
+            param.put("end_date", endDate);
+        }
+        int pageSize = 500;
+        param.put("advertiser_id", advertiserId);
+        param.put("page_size", pageSize);
+        param.put("page", page);
+        param.put("status", -2);
+        String result = HttpUtils.httpPostRequest(url, param, headers);
+        if (Check.isNull(result)) {
+            log.error("获取快手广告组异常，返回为空, accountId:{}", advertiserId);
+            errorLogService.addError(advertiserId, startDate, endDate, "group_list", "daily", "返回为空", 2);
+            return;
+        }
+        JSONObject resultJson = JSONObject.parseObject(result);
+        Integer code = resultJson.getInteger("code");
+        String message = resultJson.getString("message");
+        if (null == code || code != 0) {
+            log.error("获取快手广告组异常:{},accountId:{}", message, advertiserId);
+            errorLogService.addError(advertiserId, startDate, endDate, "group_list", "daily", message, 2);
+            return;
+        }
+        JSONArray details = resultJson.getJSONObject("data").getJSONArray("details");
+        if (!Check.isNull(details)) {
+            for (int i = 0; i < details.size(); i++) {
+                JSONObject detailJson = details.getJSONObject(i);
+                JSONObject data = new JSONObject();
+                data.put("advertiser_id", advertiserId);
+                data.put("unit_id", detailJson.getLong("unit_id"));
+                data.put("campaign_id", detailJson.getString("campaign_id"));
+                data.put("series_id", detailJson.getString("series_id"));
+                data.put("episode_id", detailJson.getString("episode_id"));
+                data.put("create_time", detailJson.getString("create_time"));
+                data.put("update_time", detailJson.getString("update_time"));
+                dorisLoadManager.add("media_api", "kuaishou_adgroup_list", data);
+            }
+        }
+
+        if (details.size() >= pageSize) {
+            queryGroupList(accessToken, advertiserId, startDate, endDate, page + 1, isLog, errId, isSend);
+        } else {
+            log.info("【{}】录入广告组完成 startDate:{}, endDate:{}", advertiserId, startDate, endDate);
+            if (errId != null) {
+                errorLogService.deleteInfoByErrorId(errId);
+            }
+            if (isSend != null && isSend == 1) {
+                AlertUtils.sendMessage(advertiserId, startDate, endDate, 2, "daily", 1, "group_list");
+            }
+        }
+    }
+
 }

job-kuaishou/src/main/java/cn/com/ctop/job/kuaishou/handler/KuaiShouGroupReportJob.java

@@ -91,4 +91,24 @@ public class KuaiShouGroupReportJob {
         });
     }
 
+    @XxlJob("queryGroupList")
+    public void queryGroupList() {
+        String param = XxlJobHelper.getJobParam(); // 执行参数
+        if (Check.isNull(param)) {
+            log.error("入参为空");
+            return;
+        }
+        String[] s = param.split("_");
+        Long advertiserId = Long.valueOf(s[0]);
+        String accessToken = s[1];
+        String nowDate = DateUtils.getNowDate("yyyy-MM-dd");
+        String yesDate = DateUtils.getAnotherDay("yyyy-MM-dd", nowDate, -1);
+        yesGroupexecutorService.submit(new Runnable() {
+            @Override
+            public void run() {
+                groupReportService.queryGroupList(accessToken, advertiserId, yesDate, nowDate, 1, 1, null, 2);
+            }
+        });
+    }
+
 }

job-kuaishou/src/main/java/cn/com/ctop/job/kuaishou/handler/KuaiShouRetryReportJob.java

@@ -1,10 +1,7 @@
 package cn.com.ctop.job.kuaishou.handler;
 
 
-import cn.com.ctop.job.kuaishou.data.service.IKuaiShouAccountReportService;
-import cn.com.ctop.job.kuaishou.data.service.IKuaiShouAdSocialService;
-import cn.com.ctop.job.kuaishou.data.service.IKuaiShouDramaService;
-import cn.com.ctop.job.kuaishou.data.service.IKuaiShouRetryReportService;
+import cn.com.ctop.job.kuaishou.data.service.*;
 import cn.com.ctop.job.kuaishou.data.utils.Check;
 import com.alibaba.fastjson.JSONObject;
 import com.xxl.job.core.handler.annotation.XxlJob;
@@ -45,6 +42,9 @@ public class KuaiShouRetryReportJob {
     @Autowired
     private IKuaiShouAccountReportService accountReportService;
 
+    @Autowired
+    private IKuaiShouGroupReportService groupReportService;
+
     @XxlJob("kuaiShouRetryReport")
     public void promoterReport() throws Exception {
         List<JSONObject> accountList = retryReportService.getErrorLogs();
@@ -105,12 +105,21 @@ public class KuaiShouRetryReportJob {
                         });
                     }
 
-                }else if ("daily_flows".equals(dataDimension)) {
+                } else if ("daily_flows".equals(dataDimension)) {
+                    dailyAccountExecutorService.submit(new Runnable() {
+                        @Override
+                        public void run() {
+                            if (!Check.isNull(accessToken)) {
+                                accountReportService.queryAccountDailyFlows(accessToken, advertiserId, startDate, endDate, 1, null, id, isSend);
+                            }
+                        }
+                    });
+                } else if ("group_list".equals(dataDimension)) {
                     dailyAccountExecutorService.submit(new Runnable() {
                         @Override
                         public void run() {
                             if (!Check.isNull(accessToken)) {
-                                accountReportService.queryAccountDailyFlows(accessToken, advertiserId, startDate, endDate, 1,null,id, isSend);
+                                groupReportService.queryGroupList(accessToken, advertiserId, startDate, endDate, 1, null, id, isSend);
                             }
                         }
                     });