限流器

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/controller/BytedanceController.java

@@ -313,7 +313,7 @@ public class BytedanceController {
             executor10Service1.submit(() -> {
                 Long adverId = jsonObject.getLong("advertiserId");
                 String accessToken = jsonObject.getString("accessToken");
-                bytedanceMaterialReportService.getBytedanceMaterialDailyReport(adverId, accessToken, startDate, endDate, 1, null, 1, 0, 1);
+                bytedanceMaterialReportService.getBytedanceMaterialDailyReport(adverId, startDate, endDate, 1, null, 1, 0, 1);
             });
         }
     }

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/controller/TestController.java

@@ -58,7 +58,7 @@ public class TestController {
             executorService.submit(new Runnable() {
                 @Override
                 public void run() {
-                    if(rateLimitUtil.acquire("test1","http://a.com/b/c", 2, 100, RateIntervalUnit.MILLISECONDS)){
+                    if(rateLimitUtil.acquire("test1","http://a.com/b/c")){
                         log.info(Thread.currentThread().getName() + " : " + finalI);
                     }else{
                         log.info(Thread.currentThread().getName() + " : 限流了！！！");

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/IBytedanceMaterialReportService.java

@@ -3,7 +3,7 @@ package cn.com.ctop.job.bytedance.data.service;
 import java.util.Set;
 
 public interface IBytedanceMaterialReportService {
-    void getBytedanceMaterialDailyReport(Long advertiserId, String accessToken, String startDate, String endDate, Integer page,
+    void getBytedanceMaterialDailyReport(Long advertiserId, String startDate, String endDate, Integer page,
                                          Long errorId, Integer isLog, Integer isSend, int retry);
 
     void getBytedanceMaterialVideo(Long advertiserId, Set<Long> materialIds, String startTime, String endTime, Integer page, Long errorId, Integer isLog, Integer isSend, int retry);

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/impl/ByteDanceFlowDetailsReportServiceImpl.java

@@ -5,10 +5,12 @@ import cn.com.ctop.job.bytedance.data.entity.ByteDanceFlowDetailsReportDaily;
 import cn.com.ctop.job.bytedance.data.entity.BytedanceErrorLog;
 import cn.com.ctop.job.bytedance.data.mapper.ByteDanceFlowDetailsReportMapper;
 import cn.com.ctop.job.bytedance.data.mapper.BytedancePromoterReportMapper;
+import cn.com.ctop.job.bytedance.data.mapper.BytedanceRefreshTokenMapper;
 import cn.com.ctop.job.bytedance.data.service.IByteDanceFlowDetailsReportService;
 import cn.com.ctop.job.bytedance.data.utils.AlertUtils;
 import cn.com.ctop.job.bytedance.data.utils.Check;
 import cn.com.ctop.job.bytedance.data.utils.HttpUtils;
+import cn.com.ctop.job.bytedance.utils.RateLimitUtil;
 import cn.com.ctop.job.bytedance.utils.doris.sink.DorisMultiTableStreamLoadManager;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
@@ -41,6 +43,10 @@ public class ByteDanceFlowDetailsReportServiceImpl implements IByteDanceFlowDeta
 
     @Autowired
     private BytedancePromoterReportMapper promoterReportMapper;
+    @Autowired
+    private BytedanceRefreshTokenMapper tokenMapper;
+    @Autowired
+    private RateLimitUtil rateLimitUtil;
 
     @Override
     public List<AdvertiserAccount> getAdAccount() {
@@ -49,6 +55,8 @@ public class ByteDanceFlowDetailsReportServiceImpl implements IByteDanceFlowDeta
 
     @Override
     public void getBytedanceFlowDetailsDailyReport(Long advertiserId, String accessToken, String accountType, String startDate, String endDate, Integer page, Long errorId) {
+        JSONObject authConfig = tokenMapper.getAuthConfig(advertiserId);
+        String devAppId = authConfig.getString("app_id");
         String url = "https://ad.oceanengine.com/open_api/2/advertiser/fund/daily_stat/";
         Map<String, Object> paramsMap = new HashMap<>();
         paramsMap.put("advertiser_id", advertiserId);
@@ -66,10 +74,14 @@ public class ByteDanceFlowDetailsReportServiceImpl implements IByteDanceFlowDeta
         Map<String, String> headers = new HashMap<>();
         headers.put("Access-Token", accessToken);
         headers.put("Content-Type", "application/json");
-        String s = HttpUtils.httpGet(url, paramsMap, headers);
+        String s = null;
+        if (rateLimitUtil.acquire(devAppId, url)) {
+            s = HttpUtils.httpGet(url, paramsMap, headers);
+        }
         JSONObject result = JSONObject.parseObject(s);
         if (Check.isNull(result)) {
             log.info("账户id:{}，广告流水详情请求结果返回为空", advertiserId);
+            insertErrorLog(advertiserId, startDate, endDate);
             return;
         }
 
@@ -128,7 +140,7 @@ public class ByteDanceFlowDetailsReportServiceImpl implements IByteDanceFlowDeta
                 }
 
             }
-        }else{
+        } else {
             //增加错误日志保存
             log.info("账户id:{}，日期:{}流水日报请求结果失败，返回数据:{}", advertiserId, startDate, result);
             insertErrorLog(advertiserId, startDate, endDate);

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/impl/BytedanceAccountReportServiceImpl.java

@@ -2,11 +2,13 @@ package cn.com.ctop.job.bytedance.data.service.impl;
 
 import cn.com.ctop.job.bytedance.data.entity.*;
 import cn.com.ctop.job.bytedance.data.mapper.BytedanceAccountReportMapper;
+import cn.com.ctop.job.bytedance.data.mapper.BytedanceRefreshTokenMapper;
 import cn.com.ctop.job.bytedance.data.service.IBytedanceAccountReportService;
 import cn.com.ctop.job.bytedance.data.utils.AlertUtils;
 import cn.com.ctop.job.bytedance.data.utils.Check;
 import cn.com.ctop.job.bytedance.data.utils.FieldsConfig;
 import cn.com.ctop.job.bytedance.data.utils.HttpUtils;
+import cn.com.ctop.job.bytedance.utils.RateLimitUtil;
 import cn.com.ctop.job.bytedance.utils.doris.sink.DorisMultiTableStreamLoadManager;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
@@ -27,10 +29,17 @@ public class BytedanceAccountReportServiceImpl implements IBytedanceAccountRepor
     @Autowired
     private BytedanceAccountReportMapper accountReportMapper;
 
+    @Autowired
+    private BytedanceRefreshTokenMapper tokenMapper;
+
+    @Autowired
+    private RateLimitUtil rateLimitUtil;
 
     @Override
     public void getBytedanceRetryAccountHourReport(Long errorId, Long advertiserId, String accessToken, String startDate, String endDate, Integer page, Integer isSend) {
         try {
+            JSONObject authConfig = tokenMapper.getAuthConfig(advertiserId);
+            String devAppId = authConfig.getString("app_id");
             String url = "https://api.oceanengine.com/open_api/v3.0/report/custom/get/";
             Map<String, Object> paramsMap = new HashMap<>();
             paramsMap.put("advertiser_id", advertiserId);
@@ -46,7 +55,10 @@ public class BytedanceAccountReportServiceImpl implements IBytedanceAccountRepor
             Map<String, String> headers = new HashMap<>();
             headers.put("Access-Token", accessToken);
             headers.put("Content-Type", "application/json;charset=UTF-8");
-            String s = HttpUtils.httpGet(url, paramsMap, headers);
+            String s = null;
+            if (rateLimitUtil.acquire(devAppId, url)) {
+                s = HttpUtils.httpGet(url, paramsMap, headers);
+            }
             JSONObject result = JSONObject.parseObject(s);
             if (Check.isNull(result)) {
                 log.info("账户id:{}，日期:{},广告时报（账户粒度）请求结果返回为空", advertiserId, startDate);
@@ -107,6 +119,8 @@ public class BytedanceAccountReportServiceImpl implements IBytedanceAccountRepor
     @Override
     public void getBytedanceRetryAccountDailyReport(Long errorId, Long advertiserId, String accessToken, String startDate, String endDate, Integer page, Integer isSend) {
         try {
+            JSONObject authConfig = tokenMapper.getAuthConfig(advertiserId);
+            String devAppId = authConfig.getString("app_id");
             String url = "https://api.oceanengine.com/open_api/v3.0/report/custom/get/";
             Map<String, Object> paramsMap = new HashMap<>();
             paramsMap.put("advertiser_id", advertiserId);
@@ -122,7 +136,10 @@ public class BytedanceAccountReportServiceImpl implements IBytedanceAccountRepor
             Map<String, String> headers = new HashMap<>();
             headers.put("Access-Token", accessToken);
             headers.put("Content-Type", "application/json;charset=UTF-8");
-            String s = HttpUtils.httpGet(url, paramsMap, headers);
+            String s = null;
+            if (rateLimitUtil.acquire(devAppId, url)) {
+                s = HttpUtils.httpGet(url, paramsMap, headers);
+            }
             JSONObject result = JSONObject.parseObject(s);
             if (Check.isNull(result)) {
                 log.info("账户id:{}，日期:{},广告时报（账户粒度）请求结果返回为空", advertiserId, startDate);
@@ -180,6 +197,8 @@ public class BytedanceAccountReportServiceImpl implements IBytedanceAccountRepor
     @Override
     public void getBytedanceAccountHour(Long advertiserId, String accessToken, String startDate, String endDate, Integer page, Integer isLog) {
         try {
+            JSONObject authConfig = tokenMapper.getAuthConfig(advertiserId);
+            String devAppId = authConfig.getString("app_id");
             String url = "https://api.oceanengine.com/open_api/v3.0/report/custom/get/";
             Map<String, Object> paramsMap = new HashMap<>();
             paramsMap.put("advertiser_id", advertiserId);
@@ -194,7 +213,10 @@ public class BytedanceAccountReportServiceImpl implements IBytedanceAccountRepor
             paramsMap.put("page_size", 100);
             Map<String, String> headers = new HashMap<>();
             headers.put("Access-Token", accessToken);
-            String s = HttpUtils.httpGet(url, paramsMap, headers);
+            String s = null;
+            if (rateLimitUtil.acquire(devAppId, url)) {
+                s = HttpUtils.httpGet(url, paramsMap, headers);
+            }
             JSONObject result = JSONObject.parseObject(s);
             if (Check.isNull(result)) {
                 log.info("账户id:{}，日期:{},广告时报（账户粒度）请求结果返回为空", advertiserId, startDate);
@@ -271,6 +293,8 @@ public class BytedanceAccountReportServiceImpl implements IBytedanceAccountRepor
     @Override
     public void getBytedanceAccountDaily(Long advertiserId, String accessToken, String startDate, String endDate, Integer page, Integer isLog) {
         try {
+            JSONObject authConfig = tokenMapper.getAuthConfig(advertiserId);
+            String devAppId = authConfig.getString("app_id");
             String url = "https://api.oceanengine.com/open_api/v3.0/report/custom/get/";
             Map<String, Object> paramsMap = new HashMap<>();
             paramsMap.put("advertiser_id", advertiserId);
@@ -285,7 +309,10 @@ public class BytedanceAccountReportServiceImpl implements IBytedanceAccountRepor
             paramsMap.put("page_size", 100);
             Map<String, String> headers = new HashMap<>();
             headers.put("Access-Token", accessToken);
-            String s = HttpUtils.httpGet(url, paramsMap, headers);
+            String s = null;
+            if (rateLimitUtil.acquire(devAppId, url)) {
+                s = HttpUtils.httpGet(url, paramsMap, headers);
+            }
             JSONObject result = JSONObject.parseObject(s);
             if (Check.isNull(result)) {
                 log.info("账户id:{}，日期:{},账户日报（账户粒度）请求结果返回为空", advertiserId, startDate);

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/impl/BytedanceCdpProjectServiceImpl.java

@@ -7,6 +7,7 @@ import cn.com.ctop.job.bytedance.data.mapper.BytedancePromoterReportMapper;
 import cn.com.ctop.job.bytedance.data.mapper.BytedanceRefreshTokenMapper;
 import cn.com.ctop.job.bytedance.data.service.IBytedanceCdpProjectService;
 import cn.com.ctop.job.bytedance.data.utils.*;
+import cn.com.ctop.job.bytedance.utils.RateLimitUtil;
 import cn.com.ctop.job.bytedance.utils.doris.sink.DorisMultiTableStreamLoadManager;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
@@ -44,6 +45,9 @@ public class BytedanceCdpProjectServiceImpl implements IBytedanceCdpProjectServi
     @Autowired
     private BytedancePromoterReportMapper promoterReportMapper;
 
+    @Autowired
+    private RateLimitUtil rateLimitUtil;
+
     @Override
     public List<Long> queryAllAdverIdList(String type) {
         return bytedanceCdpProjectMapper.queryAllAdverIdList(type);
@@ -95,11 +99,13 @@ public class BytedanceCdpProjectServiceImpl implements IBytedanceCdpProjectServi
             boolean hasNextPage = false;
             Integer nextPage = null;
             Long nextCursor = null;
-            String accessToken = tokenMapper.onlyGetToken(advertiserId);
-            if (Check.isNull(accessToken)) {
+            JSONObject authConfig = tokenMapper.getAuthConfig(advertiserId);
+            if (Check.isNull(authConfig)) {
                 log.error("账户（{}）查询token失败", advertiserId);
                 return;
             }
+            String accessToken = authConfig.getString("access_token");
+            String devAppId = authConfig.getString("app_id");
             Map<String, Object> params = new HashMap<>();
             params.put("advertiser_id", advertiserId);
             JSONObject filtering = new JSONObject();
@@ -119,7 +125,10 @@ public class BytedanceCdpProjectServiceImpl implements IBytedanceCdpProjectServi
                 params.put("page_size", 20);//最大20
             }
             String url = "https://api.oceanengine.com/open_api/v3.0/promotion/list/";
-            String results = HttpUtils.httpGet(url, params, HttpUtils.postMHeaders(accessToken));
+            String results = null;
+            if (rateLimitUtil.acquire(devAppId, url)) {
+                results = HttpUtils.httpGet(url, params, HttpUtils.postMHeaders(accessToken));
+            }
             JSONObject obj = JSONObject.parseObject(results);
             if (!Check.isNull(obj)) {
                 Integer code = obj.getInteger("code");
@@ -161,6 +170,9 @@ public class BytedanceCdpProjectServiceImpl implements IBytedanceCdpProjectServi
                     }
                     return;
                 }
+            } else {
+                insertErrorLog(isLog, advertiserId, "-", "-", "promotion_list", results);
+                return;
             }
             if (hasNextPage) {
                 getPromotionListVo(advertiserId, ids, date, nextPage, nextCursor, 1, errorId, isLog);
@@ -200,11 +212,13 @@ public class BytedanceCdpProjectServiceImpl implements IBytedanceCdpProjectServi
     private void getProjectListVo(Long advertiserId, JSONArray ids, String type, String date, Integer page, int count, Long errorId, Integer isLog) {
         try {
             boolean nextPage = false;
-            String accessToken = tokenMapper.onlyGetToken(advertiserId);
-            if (Check.isNull(accessToken)) {
+            JSONObject authConfig = tokenMapper.getAuthConfig(advertiserId);
+            if (Check.isNull(authConfig)) {
                 log.error("账户（{}）查询token失败", advertiserId);
                 return;
             }
+            String accessToken = authConfig.getString("access_token");
+            String devAppId = authConfig.getString("app_id");
             Map<String, Object> params = new HashMap<>();
             params.put("advertiser_id", advertiserId);
             JSONObject filtering = new JSONObject();
@@ -223,7 +237,10 @@ public class BytedanceCdpProjectServiceImpl implements IBytedanceCdpProjectServi
             params.put("page", page);
             params.put("page_size", 100);//最大100
             String url = "https://api.oceanengine.com/open_api/v3.0/project/list/";
-            String results = HttpUtils.httpGet(url, params, HttpUtils.postMHeaders(accessToken));
+            String results = null;
+            if (rateLimitUtil.acquire(devAppId, url)) {
+                results = HttpUtils.httpGet(url, params, HttpUtils.postMHeaders(accessToken));
+            }
             JSONObject obj = JSONObject.parseObject(results);
             if (!Check.isNull(obj)) {
                 Integer code = obj.getInteger("code");
@@ -277,11 +294,13 @@ public class BytedanceCdpProjectServiceImpl implements IBytedanceCdpProjectServi
     private void getByteAppInfoVo(Long advertiserId, String searchType, Integer page, int count) {
         try {
             boolean nextPage = false;
-            String accessToken = tokenMapper.onlyGetToken(advertiserId);
-            if (Check.isNull(accessToken)) {
+            JSONObject authConfig = tokenMapper.getAuthConfig(advertiserId);
+            if (Check.isNull(authConfig)) {
                 log.error("账户（{}）查询token失败", advertiserId);
                 return;
             }
+            String accessToken = authConfig.getString("access_token");
+            String devAppId = authConfig.getString("app_id");
             JSONObject params = new JSONObject();
             params.put("advertiser_id", advertiserId);
             params.put("page", page);
@@ -294,7 +313,10 @@ public class BytedanceCdpProjectServiceImpl implements IBytedanceCdpProjectServi
             params.put("filtering", filtering);
             //获取微信小程序列表
             String url = "https://api.oceanengine.com/open_api/v3.0/tools/micro_app/list/";
-            JSONObject obj = Okhttp3Utils.bytedanceGetRequest(accessToken, url, params);
+            JSONObject obj = null;
+            if(rateLimitUtil.acquire(devAppId, url)) {
+                obj = Okhttp3Utils.bytedanceGetRequest(accessToken, url, params);
+            }
             if (!Check.isNull(obj)) {
                 Integer code = obj.getInteger("code");
                 if (code == 0) {
@@ -317,6 +339,10 @@ public class BytedanceCdpProjectServiceImpl implements IBytedanceCdpProjectServi
                         bytedanceCdpProjectMapper.insertErrorAdvertiserId(advertiserId, "tt");
                     }
                 }
+            }else {
+                log.error("账户（{}）获取抖小列表失败，入参：{}\n结果：{}", advertiserId, JSONObject.toJSONString(params), obj);
+                bytedanceCdpProjectMapper.insertErrorAdvertiserId(advertiserId, "tt");
+                return;
             }
             if (nextPage) {
                 getByteAppInfoVo(advertiserId, searchType, page + 1, 1);
@@ -330,18 +356,20 @@ public class BytedanceCdpProjectServiceImpl implements IBytedanceCdpProjectServi
 
     @Override
     public void getByteAppLinkInfo(Long advertiserId) {
-        String accessToken = tokenMapper.onlyGetToken(advertiserId);
-        if (Check.isNull(accessToken)) {
+        JSONObject authConfig = tokenMapper.getAuthConfig(advertiserId);
+        if (Check.isNull(authConfig)) {
             log.error("账户（{}）查询token失败", advertiserId);
             return;
         }
+        String accessToken = authConfig.getString("access_token");
+        String devAppId = authConfig.getString("app_id");
         List<String> ids = bytedanceCdpProjectMapper.queryAppInstanceIdList(advertiserId);
         for (String id : ids) {
-            this.getByteAppLinkInfoVo(accessToken, advertiserId, id, 1, 1);
+            this.getByteAppLinkInfoVo(accessToken, devAppId, advertiserId, id, 1, 1);
         }
     }
 
-    private void getByteAppLinkInfoVo(String accessToken, Long advertiserId, String instanceId, Integer page, int count) {
+    private void getByteAppLinkInfoVo(String accessToken, String devAppId, Long advertiserId, String instanceId, Integer page, int count) {
         try {
             boolean nextPage = false;
             JSONObject params = new JSONObject();
@@ -353,7 +381,10 @@ public class BytedanceCdpProjectServiceImpl implements IBytedanceCdpProjectServi
             params.put("filtering", filtering);
             //获取微信小程序列表
             String url = "https://api.oceanengine.com/open_api/v3.0/tools/asset_link/list/";
-            JSONObject obj = Okhttp3Utils.bytedanceGetRequest(accessToken, url, params);
+            JSONObject obj = null;
+            if(rateLimitUtil.acquire(devAppId, url)) {
+                obj = Okhttp3Utils.bytedanceGetRequest(accessToken, url, params);
+            }
             if (!Check.isNull(obj)) {
                 Integer code = obj.getInteger("code");
                 if (code == 0) {
@@ -374,15 +405,19 @@ public class BytedanceCdpProjectServiceImpl implements IBytedanceCdpProjectServi
                     //40110请求超频 51010 内部服务超时
                     if ((40110 == code || 51010 == code) && count <= 3) {
                         Thread.sleep(700);
-                        getByteAppLinkInfoVo(accessToken, advertiserId, instanceId, page, count + 1);
+                        getByteAppLinkInfoVo(accessToken, devAppId, advertiserId, instanceId, page, count + 1);
                     } else {
                         bytedanceCdpProjectMapper.insertTTErrorAdvertiserId(advertiserId);
                         log.error("账户（{}）获取字节小程序/小游戏详情内容失败，入参：{}\n结果：{}", advertiserId, JSONObject.toJSONString(params), obj);
                     }
                 }
+            } else {
+                bytedanceCdpProjectMapper.insertTTErrorAdvertiserId(advertiserId);
+                log.error("账户（{}）获取字节小程序/小游戏详情内容失败，入参：{}\n结果：{}", advertiserId, JSONObject.toJSONString(params), obj);
+                return;
             }
             if (nextPage) {
-                getByteAppLinkInfoVo(accessToken, advertiserId, instanceId, page + 1, count);
+                getByteAppLinkInfoVo(accessToken, devAppId, advertiserId, instanceId, page + 1, count);
             }
         } catch (Exception e) {
             bytedanceCdpProjectMapper.insertTTErrorAdvertiserId(advertiserId);
@@ -400,11 +435,13 @@ public class BytedanceCdpProjectServiceImpl implements IBytedanceCdpProjectServi
     private void getWxAppInfoVo(Long advertiserId, String searchType, Integer page, int count) {
         try {
             boolean nextPage = false;
-            String accessToken = tokenMapper.onlyGetToken(advertiserId);
-            if (Check.isNull(accessToken)) {
+            JSONObject authConfig = tokenMapper.getAuthConfig(advertiserId);
+            if (Check.isNull(authConfig)) {
                 log.error("账户（{}）查询token失败", advertiserId);
                 return;
             }
+            String accessToken = authConfig.getString("access_token");
+            String devAppId = authConfig.getString("app_id");
             JSONObject params = new JSONObject();
             params.put("advertiser_id", advertiserId);
             params.put("page", page);
@@ -418,7 +455,10 @@ public class BytedanceCdpProjectServiceImpl implements IBytedanceCdpProjectServi
             params.put("filtering", filtering);
             //获取微信小程序列表
             String url = "https://api.oceanengine.com/open_api/v3.0/tools/wechat_applet/list/";
-            JSONObject obj = Okhttp3Utils.bytedanceGetRequest(accessToken, url, params);
+            JSONObject obj = null;
+            if(rateLimitUtil.acquire(devAppId, url)) {
+                obj = Okhttp3Utils.bytedanceGetRequest(accessToken, url, params);
+            }
             if (!Check.isNull(obj)) {
                 Integer code = obj.getInteger("code");
                 if (code == 0) {
@@ -455,6 +495,10 @@ public class BytedanceCdpProjectServiceImpl implements IBytedanceCdpProjectServi
                         bytedanceCdpProjectMapper.insertErrorAdvertiserId(advertiserId, "wx");
                     }
                 }
+            } else {
+                log.error("账户（{}）获取微小列表失败，入参：{}\n结果：{}", advertiserId, JSONObject.toJSONString(params), obj);
+                bytedanceCdpProjectMapper.insertErrorAdvertiserId(advertiserId, "wx");
+                return;
             }
             if (nextPage) {
                 getWxAppInfoVo(advertiserId, searchType, page + 1, 1);

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/impl/BytedanceMaterialReportServiceImpl.java

@@ -19,10 +19,8 @@ import org.springframework.data.redis.core.RedisTemplate;
 import org.springframework.stereotype.Service;
 
 import java.time.Duration;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
+import java.util.stream.Collectors;
 
 @Service
 @Slf4j
@@ -39,9 +37,14 @@ public class BytedanceMaterialReportServiceImpl implements IBytedanceMaterialRep
     private RateLimitUtil rateLimitUtil;
 
     @Override
-    public void getBytedanceMaterialDailyReport(Long advertiserId, String accessToken, String startDate, String endDate, Integer page, Long errorId, Integer isLog, Integer isSend, int retry) {
+    public void getBytedanceMaterialDailyReport(Long advertiserId, String startDate, String endDate, Integer page, Long errorId, Integer isLog, Integer isSend, int retry) {
         try {
-            accessToken = tokenMapper.onlyGetToken(advertiserId);
+            JSONObject authConfig = tokenMapper.getAuthConfig(advertiserId);
+            if (Check.isNull(authConfig)) {
+                log.info("账户id:{}，获取素材日报token为空", advertiserId);
+            }
+            String accessToken = authConfig.getString("access_token");
+            String devAppId = authConfig.getString("app_id");
             String url = "https://api.oceanengine.com/open_api/v3.0/report/custom/get/";
             Map<String, Object> paramsMap = new HashMap<>();
             paramsMap.put("advertiser_id", advertiserId);
@@ -57,7 +60,10 @@ public class BytedanceMaterialReportServiceImpl implements IBytedanceMaterialRep
             Map<String, String> headers = new HashMap<>();
             headers.put("Access-Token", accessToken);
             headers.put("Content-Type", "application/json");
-            String s = HttpUtils.httpGet(url, paramsMap, headers);
+            String s = null;
+            if (rateLimitUtil.acquire(devAppId, url)) {
+                s = HttpUtils.httpGet(url, paramsMap, headers);
+            }
             JSONObject result = JSONObject.parseObject(s);
             Set<Long> materialIds = new HashSet<>();
             if (Check.isNull(result)) {
@@ -106,7 +112,7 @@ public class BytedanceMaterialReportServiceImpl implements IBytedanceMaterialRep
                         Thread.sleep(800);
                     } catch (Exception ignored) {
                     }
-                    getBytedanceMaterialDailyReport(advertiserId, accessToken, startDate, endDate, page, errorId, isLog, isSend, retry + 1);
+                    getBytedanceMaterialDailyReport(advertiserId, startDate, endDate, page, errorId, isLog, isSend, retry + 1);
                     return;
                 }
                 log.info("账户id:{}，日期:{}素材日报请求结果失败，返回数据:{}", advertiserId, startDate, result);
@@ -128,7 +134,7 @@ public class BytedanceMaterialReportServiceImpl implements IBytedanceMaterialRep
             }
 
             if (page < totalPage) {
-                getBytedanceMaterialDailyReport(advertiserId, accessToken, startDate, endDate, page + 1, errorId, isLog, isSend, 1);
+                getBytedanceMaterialDailyReport(advertiserId, startDate, endDate, page + 1, errorId, isLog, isSend, 1);
             } else {
                 log.info("账户ID：{}, 开始日期:{}, 结束日期:{}, totalPage:{}, 素材日报数据请求成功", advertiserId, startDate, endDate, totalPage);
                 if (errorId != null) {
@@ -162,7 +168,15 @@ public class BytedanceMaterialReportServiceImpl implements IBytedanceMaterialRep
                 filtering.put("end_time", endTime);
             }
             if (!Check.isNull(materialIds)) {
-                filtering.put("material_ids", materialIds);
+                List<Long> collect = materialIds.stream().filter(id -> !redisTemplate_db2.hasKey("mvid:1:" + id)).collect(Collectors.toList());
+                if (!Check.isNull(collect)) {
+                    filtering.put("material_ids", collect);
+                } else {
+                    if (errorId != null) {
+                        promoterReportMapper.deleteInfoByErrorId(errorId);
+                    }
+                    return;
+                }
             }
             paramsMap.put("filtering", filtering);
             paramsMap.put("page", page);
@@ -171,7 +185,7 @@ public class BytedanceMaterialReportServiceImpl implements IBytedanceMaterialRep
             headers.put("Access-Token", accessToken);
             headers.put("Content-Type", "application/json");
             String s = null;
-            if (rateLimitUtil.acquire(devAppId, url, 1, 100, RateIntervalUnit.MILLISECONDS)) {
+            if (rateLimitUtil.acquire(devAppId, url)) {
                 s = HttpUtils.httpGet(url, paramsMap, headers);
             }
             JSONObject result = JSONObject.parseObject(s);

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/impl/BytedanceProductReportServiceImpl.java

@@ -4,16 +4,19 @@ import cn.com.ctop.job.bytedance.data.entity.BytedanceErrorLog;
 import cn.com.ctop.job.bytedance.data.entity.BytedanceProductReportDaily;
 import cn.com.ctop.job.bytedance.data.mapper.BytedanceProductReportMapper;
 import cn.com.ctop.job.bytedance.data.mapper.BytedancePromoterReportMapper;
+import cn.com.ctop.job.bytedance.data.mapper.BytedanceRefreshTokenMapper;
 import cn.com.ctop.job.bytedance.data.service.IBytedanceProductReportService;
 import cn.com.ctop.job.bytedance.data.utils.AlertUtils;
 import cn.com.ctop.job.bytedance.data.utils.Check;
 import cn.com.ctop.job.bytedance.data.utils.FieldsConfig;
 import cn.com.ctop.job.bytedance.data.utils.HttpUtils;
+import cn.com.ctop.job.bytedance.utils.RateLimitUtil;
 import cn.com.ctop.job.bytedance.utils.doris.sink.DorisMultiTableStreamLoadManager;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.data.redis.core.RedisTemplate;
 import org.springframework.stereotype.Service;
 
 import java.util.ArrayList;
@@ -32,6 +35,11 @@ public class BytedanceProductReportServiceImpl implements IBytedanceProductRepor
 
     @Autowired
     private BytedanceProductReportMapper productReportMapper;
+    @Autowired
+    private BytedanceRefreshTokenMapper tokenMapper;
+
+    @Autowired
+    private RateLimitUtil rateLimitUtil;
 
     @Override
     public void deleteBytedanceProductDailyReport(Long advertiserId, String startDate, String endDate) {
@@ -47,6 +55,8 @@ public class BytedanceProductReportServiceImpl implements IBytedanceProductRepor
     public void getBytedanceProductDailyReport(Long advertiserId, String accessToken, String startDate, String endDate, Integer page,
                                                Long errorId, Integer isLog, Integer isSend) {
         try {
+            JSONObject authConfig = tokenMapper.getAuthConfig(advertiserId);
+            String devAppId = authConfig.getString("app_id");
             String url = "https://api.oceanengine.com/open_api/v3.0/report/custom/get/";
             Map<String, Object> paramsMap = new HashMap<>();
             paramsMap.put("advertiser_id", advertiserId);
@@ -62,7 +72,10 @@ public class BytedanceProductReportServiceImpl implements IBytedanceProductRepor
             Map<String, String> headers = new HashMap<>();
             headers.put("Access-Token", accessToken);
             headers.put("Content-Type", "application/json");
-            String s = HttpUtils.httpGet(url, paramsMap, headers);
+            String s = null;
+            if (rateLimitUtil.acquire(devAppId, url)) {
+                s = HttpUtils.httpGet(url, paramsMap, headers);
+            }
             JSONObject result = JSONObject.parseObject(s);
             if (Check.isNull(result)) {
                 log.info("账户id:{}，产品数据日报请求结果返回为空", advertiserId);

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/impl/BytedancePromoterReportServiceImpl.java

@@ -4,11 +4,13 @@ import cn.com.ctop.job.bytedance.data.entity.BytedanceErrorLog;
 import cn.com.ctop.job.bytedance.data.entity.BytedancePromoterReportDaily;
 import cn.com.ctop.job.bytedance.data.entity.BytedancePromoterReportHour;
 import cn.com.ctop.job.bytedance.data.mapper.BytedancePromoterReportMapper;
+import cn.com.ctop.job.bytedance.data.mapper.BytedanceRefreshTokenMapper;
 import cn.com.ctop.job.bytedance.data.service.IBytedancePromoterReportService;
 import cn.com.ctop.job.bytedance.data.utils.Check;
 import cn.com.ctop.job.bytedance.data.utils.FieldsConfig;
 import cn.com.ctop.job.bytedance.data.utils.HttpUtils;
 import cn.com.ctop.job.bytedance.data.utils.AlertUtils;
+import cn.com.ctop.job.bytedance.utils.RateLimitUtil;
 import cn.com.ctop.job.bytedance.utils.doris.sink.DorisMultiTableStreamLoadManager;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
@@ -32,6 +34,12 @@ public class BytedancePromoterReportServiceImpl implements IBytedancePromoterRep
     @Autowired
     private BytedancePromoterReportMapper promoterReportMapper;
 
+    @Autowired
+    private BytedanceRefreshTokenMapper tokenMapper;
+
+    @Autowired
+    private RateLimitUtil rateLimitUtil;
+
     @Override
     public List<JSONObject> getAccountList() {
         return promoterReportMapper.getAccountList();
@@ -40,6 +48,8 @@ public class BytedancePromoterReportServiceImpl implements IBytedancePromoterRep
     @Override
     public void getBytedanceRetryPromoterHourReport(Long errorId, Long advertiserId, String accessToken, String startDate, String endDate, Integer page, Integer isSend) {
         try {
+            JSONObject authConfig = tokenMapper.getAuthConfig(advertiserId);
+            String devAppId = authConfig.getString("app_id");
             String url = "https://api.oceanengine.com/open_api/v3.0/report/custom/get/";
             Map<String, Object> paramsMap = new HashMap<>();
             paramsMap.put("advertiser_id", advertiserId);
@@ -55,7 +65,10 @@ public class BytedancePromoterReportServiceImpl implements IBytedancePromoterRep
             Map<String, String> headers = new HashMap<>();
             headers.put("Access-Token", accessToken);
             headers.put("Content-Type", "application/json;charset=UTF-8");
-            String s = HttpUtils.httpGet(url, paramsMap, headers);
+            String s = null;
+            if (rateLimitUtil.acquire(devAppId, url)) {
+                s = HttpUtils.httpGet(url, paramsMap, headers);
+            }
             JSONObject result = JSONObject.parseObject(s);
             if (Check.isNull(result)) {
                 log.info("账户id:{}，日期:{},广告时报（最细粒度）请求结果返回为空", advertiserId, startDate);
@@ -124,6 +137,8 @@ public class BytedancePromoterReportServiceImpl implements IBytedancePromoterRep
     @Override
     public void getBytedancePromoterRetryDailyReport(Long errorId, Long advertiserId, String accessToken, String startDate, String endDate, Integer page, Integer isSend) {
         try {
+            JSONObject authConfig = tokenMapper.getAuthConfig(advertiserId);
+            String devAppId = authConfig.getString("app_id");
             String url = "https://api.oceanengine.com/open_api/v3.0/report/custom/get/";
             Map<String, Object> paramsMap = new HashMap<>();
             paramsMap.put("advertiser_id", advertiserId);
@@ -139,7 +154,10 @@ public class BytedancePromoterReportServiceImpl implements IBytedancePromoterRep
             Map<String, String> headers = new HashMap<>();
             headers.put("Access-Token", accessToken);
             headers.put("Content-Type", "application/json");
-            String s = HttpUtils.httpGet(url, paramsMap, headers);
+            String s = null;
+            if (rateLimitUtil.acquire(devAppId, url)) {
+                s = HttpUtils.httpGet(url, paramsMap, headers);
+            }
             JSONObject result = JSONObject.parseObject(s);
             if (Check.isNull(result)) {
                 log.info("账户id:{}，广告日报（最细粒度）请求结果返回为空", advertiserId);
@@ -207,6 +225,8 @@ public class BytedancePromoterReportServiceImpl implements IBytedancePromoterRep
     @Override
     public void getBytedancePromoterHour(Long advertiserId, String accessToken, String startDate, String endDate, Integer page, Integer isLog) {
         try {
+            JSONObject authConfig = tokenMapper.getAuthConfig(advertiserId);
+            String devAppId = authConfig.getString("app_id");
             String url = "https://api.oceanengine.com/open_api/v3.0/report/custom/get/";
             Map<String, Object> paramsMap = new HashMap<>();
             paramsMap.put("advertiser_id", advertiserId);
@@ -222,7 +242,10 @@ public class BytedancePromoterReportServiceImpl implements IBytedancePromoterRep
             Map<String, String> headers = new HashMap<>();
             headers.put("Access-Token", accessToken);
             headers.put("Content-Type", "application/json;charset=UTF-8");
-            String s = HttpUtils.httpGet(url, paramsMap, headers);
+            String s = null;
+            if (rateLimitUtil.acquire(devAppId, url)) {
+                s = HttpUtils.httpGet(url, paramsMap, headers);
+            }
             JSONObject result = JSONObject.parseObject(s);
             if (Check.isNull(result)) {
                 log.info("账户id:{}，日期:{},广告时报（最细粒度）请求结果返回为空", advertiserId, startDate);
@@ -304,6 +327,8 @@ public class BytedancePromoterReportServiceImpl implements IBytedancePromoterRep
     @Override
     public void getBytedancePromoterDaily(Long advertiserId, String accessToken, String startDate, String endDate, Integer page, Integer isLog) {
         try {
+            JSONObject authConfig = tokenMapper.getAuthConfig(advertiserId);
+            String devAppId = authConfig.getString("app_id");
             String url = "https://api.oceanengine.com/open_api/v3.0/report/custom/get/";
             Map<String, Object> paramsMap = new HashMap<>();
             paramsMap.put("advertiser_id", advertiserId);
@@ -319,7 +344,10 @@ public class BytedancePromoterReportServiceImpl implements IBytedancePromoterRep
             Map<String, String> headers = new HashMap<>();
             headers.put("Access-Token", accessToken);
             headers.put("Content-Type", "application/json");
-            String s = HttpUtils.httpGet(url, paramsMap, headers);
+            String s = null;
+            if (rateLimitUtil.acquire(devAppId, url)) {
+                s = HttpUtils.httpGet(url, paramsMap, headers);
+            }
             JSONObject result = JSONObject.parseObject(s);
             if (Check.isNull(result)) {
                 log.info("账户id:{}，广告日报（最细粒度）请求结果返回为空", advertiserId);

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/impl/ToolsSiteServiceImpl.java

@@ -10,6 +10,7 @@ import cn.com.ctop.job.bytedance.data.service.IToolsSiteService;
 import cn.com.ctop.job.bytedance.data.utils.Check;
 import cn.com.ctop.job.bytedance.data.utils.HttpUtils;
 import cn.com.ctop.job.bytedance.data.utils.Okhttp3Utils;
+import cn.com.ctop.job.bytedance.utils.RateLimitUtil;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import lombok.extern.slf4j.Slf4j;
@@ -30,6 +31,9 @@ public class ToolsSiteServiceImpl implements IToolsSiteService {
     @Autowired
     private BytedanceRefreshTokenMapper tokenMapper;
 
+    @Autowired
+    private RateLimitUtil rateLimitUtil;
+
     @Override
     public List<Long> getAvatarAdvertiserIds() {
         return toolsSiteMapper.getAvatarAdvertiserIds();
@@ -37,11 +41,19 @@ public class ToolsSiteServiceImpl implements IToolsSiteService {
 
     @Override
     public void updateAvatarStatus(Long advertiserId) {
-        String accessToken = tokenMapper.onlyGetToken(advertiserId);
+        JSONObject authConfig = tokenMapper.getAuthConfig(advertiserId);
+        if (Check.isNull(authConfig)) {
+            log.info("账户id:{}，获取token为空", advertiserId);
+        }
+        String accessToken = authConfig.getString("access_token");
+        String devAppId = authConfig.getString("app_id");
         JSONObject params = new JSONObject();
         params.put("advertiser_id", advertiserId);
         String url = "https://ad.oceanengine.com/open_api/2/advertiser/avatar/get/";
-        JSONObject obj = Okhttp3Utils.bytedanceGetRequest(accessToken, url, params);
+        JSONObject obj = null;
+        if (rateLimitUtil.acquire(devAppId,url)){
+            obj = Okhttp3Utils.bytedanceGetRequest(accessToken, url, params);
+        }
         if (!Check.isNull(obj)) {
             Integer code = obj.getInteger("code");
             if (code == 0) {
@@ -53,6 +65,8 @@ public class ToolsSiteServiceImpl implements IToolsSiteService {
             } else {
                 log.error("更新广告主账户头像失败,返回结果：{}", obj);
             }
+        } else {
+            log.error("更新广告主账户头像失败.");
         }
     }
 
@@ -63,14 +77,21 @@ public class ToolsSiteServiceImpl implements IToolsSiteService {
 
     @Override
     public void updateNameStatus(Long advertiserId) {
-        String accessToken = tokenMapper.onlyGetToken(advertiserId);
+        JSONObject authConfig = tokenMapper.getAuthConfig(advertiserId);
+        if (Check.isNull(authConfig)) {
+            log.info("账户id:{}，获取token为空", advertiserId);
+        }
+        String accessToken = authConfig.getString("access_token");
+        String devAppId = authConfig.getString("app_id");
         Map<String, Object> params = new HashMap<>();
         params.put("account_ids", new JSONArray(Collections.singletonList(advertiserId)));
         String url = "https://api.oceanengine.com/open_api/2/agent/advertiser_info/query/";
-        String results = HttpUtils.httpGet(url, params, HttpUtils.postMHeaders(accessToken));
+        String results = null;
+        if (rateLimitUtil.acquire(devAppId,url)){
+            results = HttpUtils.httpGet(url, params, HttpUtils.postMHeaders(accessToken));
+        }
         JSONObject obj = JSONObject.parseObject(results);
-        Integer code = obj.getInteger("code");
-        if (code == 0) {
+        if (!Check.isNull(obj) && 0 == obj.getInteger("code")) {
             JSONArray array = obj.getJSONObject("data").getJSONArray("account_detail_list");
             JSONObject object = array.getJSONObject(0);
             if (!Check.isNull(object)) {
@@ -105,7 +126,12 @@ public class ToolsSiteServiceImpl implements IToolsSiteService {
     @Override
     public void updateWxAppStatus(Long advertiserId, String searchType, Integer page) {
         boolean nextPage = false;
-        String accessToken = tokenMapper.onlyGetToken(advertiserId);
+        JSONObject authConfig = tokenMapper.getAuthConfig(advertiserId);
+        if (Check.isNull(authConfig)) {
+            log.info("账户id:{}，获取token为空", advertiserId);
+        }
+        String accessToken = authConfig.getString("access_token");
+        String devAppId = authConfig.getString("app_id");
         JSONObject params = new JSONObject();
         params.put("advertiser_id", advertiserId);
         params.put("page", page);
@@ -125,9 +151,11 @@ public class ToolsSiteServiceImpl implements IToolsSiteService {
         params.put("filtering", filtering);
         //获取微信小程序列表
         String url = "https://api.oceanengine.com/open_api/v3.0/tools/wechat_applet/list/";
-        JSONObject obj = Okhttp3Utils.bytedanceGetRequest(accessToken, url, params);
-        Integer code = obj.getInteger("code");
-        if (code == 0) {
+        JSONObject obj = null;
+        if(rateLimitUtil.acquire(devAppId,url)){
+            obj = Okhttp3Utils.bytedanceGetRequest(accessToken, url, params);
+        }
+        if (!Check.isNull(obj) && obj.getInteger("code") == 0) {
             JSONObject data = obj.getJSONObject("data");
             Integer totalPage = data.getJSONObject("page_info").getInteger("total_page");
             if (totalPage != 0 && totalPage != data.getJSONObject("page_info").getInteger("page")) {
@@ -174,7 +202,12 @@ public class ToolsSiteServiceImpl implements IToolsSiteService {
 
     private void getTTList(AdvertiserApplet targetApplet, Long advertiserId, String searchType, Integer page) {
         boolean nextPage = false;
-        String accessToken = tokenMapper.onlyGetToken(advertiserId);
+        JSONObject authConfig = tokenMapper.getAuthConfig(advertiserId);
+        if (Check.isNull(authConfig)) {
+            log.info("账户id:{}，获取token为空", advertiserId);
+        }
+        String accessToken = authConfig.getString("access_token");
+        String devAppId = authConfig.getString("app_id");
         JSONObject params = new JSONObject();
         params.put("advertiser_id", advertiserId);
         params.put("page", page);
@@ -186,9 +219,11 @@ public class ToolsSiteServiceImpl implements IToolsSiteService {
         params.put("filtering", filtering);
         //获取微信小程序列表
         String url = "https://api.oceanengine.com/open_api/v3.0/tools/micro_app/list/";
-        JSONObject obj = Okhttp3Utils.bytedanceGetRequest(accessToken, url, params);
-        Integer code = obj.getInteger("code");
-        if (code == 0) {
+        JSONObject obj = null;
+        if(rateLimitUtil.acquire(devAppId,url)){
+            obj = Okhttp3Utils.bytedanceGetRequest(accessToken, url, params);
+        }
+        if (!Check.isNull(obj) && 0 == obj.getInteger("code")) {
             JSONObject data = obj.getJSONObject("data");
             Integer totalPage = data.getJSONObject("page_info").getInteger("total_page");
             if (totalPage != 0 && totalPage != data.getJSONObject("page_info").getInteger("page")) {
@@ -231,7 +266,12 @@ public class ToolsSiteServiceImpl implements IToolsSiteService {
 
     @Override
     public void updateSiteStatus(Long advertiserId, Integer page) {
-        String accessToken = tokenMapper.onlyGetToken(advertiserId);
+        JSONObject authConfig = tokenMapper.getAuthConfig(advertiserId);
+        if (Check.isNull(authConfig)) {
+            log.info("账户id:{}，获取token为空", advertiserId);
+        }
+        String accessToken = authConfig.getString("access_token");
+        String devAppId = authConfig.getString("app_id");
         boolean nextPage = false;
         Map<String, Object> params = new HashMap<>();
         params.put("advertiser_id", advertiserId);
@@ -240,9 +280,11 @@ public class ToolsSiteServiceImpl implements IToolsSiteService {
         params.put("page", page);
         params.put("page_size", 100);//默认值为10，范围：20～300
         String url = "https://ad.oceanengine.com/open_api/2/tools/site/get/";
-        JSONObject obj = Okhttp3Utils.bytedanceGetRequest(accessToken, url, params);
-        Integer code = obj.getInteger("code");
-        if (code == 0) {
+        JSONObject obj = null;
+        if(rateLimitUtil.acquire(devAppId,url)){
+            obj = Okhttp3Utils.bytedanceGetRequest(accessToken, url, params);
+        }
+        if (!Check.isNull(obj) && 0 == obj.getInteger("code")) {
             JSONObject data = obj.getJSONObject("data");
             Integer totalPage = data.getJSONObject("page_info").getInteger("total_page");
             if (totalPage != 0 && totalPage != data.getJSONObject("page_info").getInteger("page")) {

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/utils/Check.java

@@ -1,6 +1,7 @@
 package cn.com.ctop.job.bytedance.data.utils;
 
 import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 
@@ -29,6 +30,10 @@ public class Check {
         return jsonArray == null || jsonArray.isEmpty();
     }
 
+    public static boolean isNull(JSONObject json) {
+        return json == null || json.isEmpty();
+    }
+
 
     public static boolean isNull(String str) {
         return str == null || str.isEmpty();

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/utils/Okhttp3Utils.java

@@ -25,8 +25,7 @@ import java.util.Map;
  */
 @Slf4j
 public class Okhttp3Utils {
-    @Autowired
-    private static Gson gson = new Gson();
+    private static final Gson gson = new Gson();
 
     public static JSONObject bytedanceGetRequest(String token, String url, Map<String, Object> paramsMap) {
         try {

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedancePromoterReportJob.java

@@ -144,7 +144,7 @@ public class BytedancePromoterReportJob {
         dailyExecutorService.submit(new Runnable() {
             @Override
             public void run() {
-                bytedanceMaterialReportService.getBytedanceMaterialDailyReport(advertiserId, accessToken, date, date, 1, null, 1, 0, 1);
+                bytedanceMaterialReportService.getBytedanceMaterialDailyReport(advertiserId, date, date, 1, null, 1, 0, 1);
             }
         });
     }

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedanceReportRetryJob.java

@@ -163,7 +163,7 @@ public class BytedanceReportRetryJob {
                         @Override
                         public void run() {
                             if (!Check.isNull(accessToken)) {
-                                bytedanceMaterialReportService.getBytedanceMaterialDailyReport(advertiserId, accessToken, startDate, endDate, 1, id, null, isSend, 1);
+                                bytedanceMaterialReportService.getBytedanceMaterialDailyReport(advertiserId, startDate, endDate, 1, id, null, isSend, 1);
                             }
                         }
                     });

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/utils/RateLimitUtil.java

@@ -1,6 +1,7 @@
 package cn.com.ctop.job.bytedance.utils;
 
 import cn.com.ctop.job.bytedance.data.utils.Check;
+import com.google.common.collect.ImmutableMap;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.redisson.api.RRateLimiter;
@@ -22,21 +23,37 @@ public class RateLimitUtil {
     @Autowired
     public RedissonClient redissonClient;
     private static final Map<String, Boolean> initMap = new ConcurrentHashMap<>();
+    private static final Map<String, Integer> pathQpsMap = new ImmutableMap.Builder<String, Integer>()
+            .put("/open_api/v3.0/report/custom/get/", 70)
+            .put("/open_api/2/file/video/get/", 10)
+            .put("/open_api/v3.0/promotion/list/", 30)
+            .put("/open_api/v3.0/project/list/", 30)
+            .put("/open_api/2/customer_center/advertiser/list/", 50)
+            .put("/open_api/v3.0/tools/micro_app/list/", 10)
+            .put("/open_api/v3.0/tools/asset_link/list/", 10)
+            .put("/open_api/v3.0/tools/wechat_applet/list/", 10)
+            .put("/open_api/2/advertiser/fund/daily_stat/", 50)
+            .put("/open_api/2/advertiser/avatar/get/", 10)
+            .put("/open_api/2/agent/advertiser_info/query/", 10)
+            .put("/open_api/2/tools/site/get/", 10)
+            .build();
 
     /**
      * 自定义限流规则
      */
-    public boolean acquire(String devAppId, String url, long limit, long interval, RateIntervalUnit unit) {
+    public boolean acquire(String devAppId, String url) {
         URI uri = URI.create(url);
-        String limitKey = uri.getPath().replaceAll("/$", "").replace("/", ":");
+        String path = uri.getPath();
+        String limitKey = path.replaceAll("/$", "").replace("/", ":");
         String key = "rate:limit" + limitKey + (Check.isNull(devAppId) ? "" : ":" + devAppId);
         if (!initMap.containsKey(key)) {
+            Integer qps = pathQpsMap.getOrDefault(path, 10);
             RRateLimiter limiter = redissonClient.getRateLimiter(key);
             limiter.setRate(
                     RateType.OVERALL,
-                    limit,
-                    interval,
-                    unit
+                    qps / 2,
+                    500,
+                    RateIntervalUnit.MILLISECONDS
             );
             initMap.put(key, true);
         }