Browse Source

组,计划-遗漏数据补充

yangzian 3 years ago
parent
commit
5912875b86

+ 9 - 3
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/impl/ByteDanceAdvertiserDataServiceImpl.java

@@ -76,7 +76,7 @@ public class ByteDanceAdvertiserDataServiceImpl implements IByteDanceAdvertiserD
      */
 
     public void getAd(OauthToken token, int pageNum, String ids, String date, String updateDate) {
-        log.info("开始获取广告计划信息 accountID:{},page:{}", token.getAccountId(), pageNum);
+        log.info("开始获取广告计划信息 accountID:{},日期{},page:{}", token.getAccountId(), updateDate,pageNum);
 
         List<Object> adAccountList = redisUtil.getList("AD"+updateDate,null);
 
@@ -115,9 +115,9 @@ public class ByteDanceAdvertiserDataServiceImpl implements IByteDanceAdvertiserD
         Integer code = resultObject.getInteger("code");
         if (null == code || !code.equals(0)) {
             log.error(
-                    "获取广告计划信息接口异常==》accountId:{},message:{}",
+                    "获取广告计划信息接口异常==》accountId:{},message:{},code{}",
                     token.getAccountId(),
-                    resultObject.getString("message"));
+                    resultObject.getString("message"),code);
             //请求过于频繁
             if (40100 == code){
                 log.info("广告计划请求频繁,code:{},准备重试,accountId:{},错误消息:{}",code,token.getAccountId(),resultObject.getString("message"));
@@ -128,6 +128,12 @@ public class ByteDanceAdvertiserDataServiceImpl implements IByteDanceAdvertiserD
                     adAccountList.add(token.getAccountId());
                     redisUtil.add("AD"+updateDate,adAccountList, 1, TimeUnit.HOURS);
                 }
+            }else {
+                if (adAccountList.contains(token.getAccountId())){
+                    adAccountList.remove(token.getAccountId());
+                    redisUtil.add("AD"+updateDate,adAccountList, 1, TimeUnit.HOURS);
+                }
+
             }
             return;
         }

+ 8 - 2
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/impl/BytedanceCampaignGetServiceImpl.java

@@ -42,7 +42,7 @@ public class BytedanceCampaignGetServiceImpl implements IBytedanceCampaignGetSer
 
     @Override
     public void getCampaignData(OauthToken oauthToken, String startDate, String endDate, int pageNumber, int pageSize) {
-        log.info("开始获取广告组全量数据 accountIDL{},page:{}", oauthToken.getAccountId(), pageNumber);
+        log.info("开始获取广告组全量数据 accountIDL{},日期{},page:{}", oauthToken.getAccountId(), startDate,pageNumber);
         List<Object> campAccountList = redisUtil.getList("CAMP"+startDate,null);
 
         // 请求地址
@@ -64,7 +64,7 @@ public class BytedanceCampaignGetServiceImpl implements IBytedanceCampaignGetSer
 
         Integer code = resultObject.getInteger("code");
         if (null == code || !code.equals(0)) {
-            log.error("获取广告组接口异常==》accountId:{},message:{}", oauthToken.getAccountId(), resultObject.getString("message"));
+            log.error("获取广告组接口异常==》accountId:{},message:{},code{}", oauthToken.getAccountId(), resultObject.getString("message"),code);
 
             //请求过于频繁
             if (40100 == code){
@@ -76,6 +76,12 @@ public class BytedanceCampaignGetServiceImpl implements IBytedanceCampaignGetSer
                     campAccountList.add(oauthToken.getAccountId());
                     redisUtil.add("CAMP"+startDate,campAccountList, 1, TimeUnit.HOURS);
                 }
+            }else {
+                if (campAccountList.contains(oauthToken.getAccountId())){
+                    campAccountList.remove(oauthToken.getAccountId());
+                    redisUtil.add("CAMP"+startDate,campAccountList, 1, TimeUnit.HOURS);
+                }
+
             }
             return;
         }

+ 64 - 29
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedanceCampaignLoadJob.java

@@ -16,9 +16,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
 import java.text.SimpleDateFormat;
-import java.util.Arrays;
-import java.util.Date;
-import java.util.List;
+import java.util.*;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 
@@ -54,32 +52,43 @@ public class BytedanceCampaignLoadJob {
     @XxlJob("bytedanceCampaignLoadJobSupplement")
     public void bytedanceCampaignLoadJobSupplement() throws Exception {
         String nowDate = DateUtils.getNowDate("yyyy-MM-dd");
-        String yesterda = DateUtils.getAnotherDay("yyyy-MM-dd", nowDate, -1);
-
-        //添加重试
-        List<Object> campAccountList;
-        campAccountList = redisUtil.getList("CAMP"+yesterda,null);
-        for (Object accountId : campAccountList) {
-            log.info("广告组遗漏数据补充,日期:{},accountId:{}",yesterda,campAccountList.toString());
-            String token = tokenService.getByAccountId(Long.valueOf(String.valueOf(accountId)));
-            if (Check.isNull(token)) {
-                log.error("此账户未获取到相关token,accountId:{}", accountId);
-                return;
-            }
-
-            OauthToken oauthToken = new OauthToken();
-            oauthToken.setAccessToken(token);
-            oauthToken.setAccountId(Long.valueOf(String.valueOf(accountId)));
-            campaignDataService.getCampaignData(oauthToken, yesterda, yesterda, 1, 1000);
-            campAccountList = redisUtil.getList("CAMP"+yesterda,null);
-            if (campAccountList.size() == 0){
-                redisUtil.delete("CAMP"+yesterda);
-            }
+        //String yesterda = DateUtils.getAnotherDay("yyyy-MM-dd", nowDate, -1);
+
+       Set<String> set = redisUtil.kesy("CAMP*");
+       if (Check.isNull(set)){
+           return;
+       }
+
+       List<String> list = new ArrayList<>(set);
+
+        for (String yesterda : list) {
+
+            //添加重试
+            List<Object> campAccountList;
+            campAccountList = redisUtil.getList(yesterda,null);
+            for (Object accountId : campAccountList) {
+                log.info("广告组遗漏数据补充,日期:{},accountId:{}",yesterda,campAccountList.toString());
+                String token = tokenService.getByAccountId(Long.valueOf(String.valueOf(accountId)));
+                if (Check.isNull(token)) {
+                    log.error("此账户未获取到相关token,accountId:{}", accountId);
+                    return;
+                }
+
+                OauthToken oauthToken = new OauthToken();
+                oauthToken.setAccessToken(token);
+                oauthToken.setAccountId(Long.valueOf(String.valueOf(accountId)));
+                campaignDataService.getCampaignData(oauthToken, yesterda.replaceAll("CAMP",""), yesterda.replaceAll("CAMP",""), 1, 1000);
+                campAccountList = redisUtil.getList(yesterda,null);
+                if (campAccountList.size() == 0){
+                    redisUtil.delete(yesterda);
+                }
+
+                try {
+                    Thread.sleep(1000*3);
+                } catch (InterruptedException e) {
+                    e.printStackTrace();
+                }
 
-            try {
-                Thread.sleep(1000*10);
-            } catch (InterruptedException e) {
-                e.printStackTrace();
             }
 
         }
@@ -87,6 +96,7 @@ public class BytedanceCampaignLoadJob {
 
 
 
+
     }
 
 
@@ -116,7 +126,32 @@ public class BytedanceCampaignLoadJob {
             OauthToken oauthToken = new OauthToken();
             oauthToken.setAccessToken(token);
             oauthToken.setAccountId(Long.parseLong(param));
-            campaignDataService.getCampaignData(oauthToken, yesterda, yesterda, 1, 1000);
+
+//            List<String> list = new ArrayList<>();
+//            list.add("2022-06-15");
+//            list.add("2022-06-16");
+//            list.add("2022-06-17");
+//            list.add("2022-06-18");
+//            list.add("2022-06-19");
+//            list.add("2022-06-20");
+//            list.add("2022-06-21");
+//            list.add("2022-06-22");
+//            list.add("2022-06-23");
+//            list.add("2022-06-24");
+//            list.add("2022-06-25");
+//            list.add("2022-06-26");
+//            list.add("2022-06-27");
+//            list.add("2022-06-28");
+//            list.add("2022-06-29");
+//
+//            list.forEach(str ->{
+                campaignDataService.getCampaignData(oauthToken, nowDate, nowDate, 1, 1000);
+                campaignDataService.getCampaignData(oauthToken, yesterda, yesterda, 1, 1000);
+
+            //});
+
+
+
 
         //campaignDataService.getCampaignData(oauthToken, yesterda,yesterda,1, 1000);
     }

+ 61 - 25
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedancePlanLoadJob.java

@@ -49,40 +49,52 @@ public class BytedancePlanLoadJob {
     public void bytedancePlanLoadJobSupplement() throws Exception {
 
         String nowDate = DateUtils.getNowDate("yyyy-MM-dd");
-        String yesterda = DateUtils.getAnotherDay("yyyy-MM-dd", nowDate, -1);
-        //添加重试
-        List<Object> adAccountList;
-        adAccountList = redisUtil.getList("AD"+yesterda,null);
-        for (Object accountId : adAccountList) {
+        //String yesterda = DateUtils.getAnotherDay("yyyy-MM-dd", nowDate, -1);
 
-            log.info("广告计划遗漏数据补充,日期:{},accountId:{}",yesterda,adAccountList.toString());
-            String token = tokenService.getByAccountId(Long.valueOf(String.valueOf(accountId)));
-            if (Check.isNull(token)) {
-                log.error("此账户未获取到相关token,accountId:{}", accountId);
-                return;
-            }
+        Set<String> set = redisUtil.kesy("AD*");
+        if (Check.isNull(set)){
+            return;
+        }
 
-            OauthToken oauthToken = new OauthToken();
-            oauthToken.setAccessToken(token);
-            oauthToken.setAccountId(Long.valueOf(String.valueOf(accountId)));
-            advertiserDataService.getAdvertiserPlan(oauthToken, "", null, yesterda);
+        List<String> list = new ArrayList<>(set);
+        for (String yesterda : list) {
 
-            adAccountList = redisUtil.getList("AD"+yesterda,null);
+            //添加重试
+            List<Object> adAccountList;
+            adAccountList = redisUtil.getList(yesterda,null);
+            for (Object accountId : adAccountList) {
 
-            if (adAccountList.size() == 0){
-                redisUtil.delete("AD"+yesterda);
-            }
+                log.info("广告计划遗漏数据补充,日期:{},accountId:{}",yesterda,adAccountList.toString());
+                String token = tokenService.getByAccountId(Long.valueOf(String.valueOf(accountId)));
+                if (Check.isNull(token)) {
+                    log.error("此账户未获取到相关token,accountId:{}", accountId);
+                    return;
+                }
+
+                OauthToken oauthToken = new OauthToken();
+                oauthToken.setAccessToken(token);
+                oauthToken.setAccountId(Long.valueOf(String.valueOf(accountId)));
+                advertiserDataService.getAdvertiserPlan(oauthToken, "", null,  yesterda.replaceAll("AD",""));
+
+                adAccountList = redisUtil.getList(yesterda,null);
+
+                if (adAccountList.size() == 0){
+                    redisUtil.delete(yesterda);
+                }
+
+                try {
+                    Thread.sleep(1000*2);
+                } catch (InterruptedException e) {
+                    e.printStackTrace();
+                }
 
-            try {
-                Thread.sleep(1000*10);
-            } catch (InterruptedException e) {
-                e.printStackTrace();
             }
 
         }
 
 
 
+
     }
 
 
@@ -112,8 +124,32 @@ public class BytedancePlanLoadJob {
             OauthToken oauthToken = new OauthToken();
             oauthToken.setAccessToken(token);
             oauthToken.setAccountId(Long.parseLong(param));
-            advertiserDataService.getAdvertiserPlan(oauthToken, "", null, yesterda);
-            //advertiserDataService.getAdvertiserPlan(oauthToken, "", null, yesterda2);
+
+//        List<String> list = new ArrayList<>();
+//        list.add("2022-06-15");
+//        list.add("2022-06-16");
+//        list.add("2022-06-17");
+//        list.add("2022-06-18");
+//        list.add("2022-06-19");
+//        list.add("2022-06-20");
+//        list.add("2022-06-21");
+//        list.add("2022-06-22");
+//        list.add("2022-06-23");
+//        list.add("2022-06-24");
+//        list.add("2022-06-25");
+//        list.add("2022-06-26");
+//        list.add("2022-06-27");
+//        list.add("2022-06-28");
+//        list.add("2022-06-29");
+//
+//        list.forEach(str ->{
+                    advertiserDataService.getAdvertiserPlan(oauthToken, "", null, nowDate);
+                    advertiserDataService.getAdvertiserPlan(oauthToken, "", null, yesterda);
+                  //  log.info("结束。。。。");
+//                });
+
+
+        //advertiserDataService.getAdvertiserPlan(oauthToken, "", null, yesterda2);
 
         //advertiserDataService.getAdvertiserPlan(oauthToken, "", null, yesterda);
     }

+ 10 - 0
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/utils/RedisUtil.java

@@ -230,4 +230,14 @@ public class RedisUtil {
 		return redisTemplate.getExpire(key);
 	}
 
+
+	/**
+	 *  返回 满足前缀 的key
+	 * @param pattern
+	 * @return
+	 */
+	public Set<String> kesy(String pattern){
+		return redisTemplate.keys(pattern);
+	}
+
 }