yumeng před 2 měsíci
rodič
revize
93eb8117ae

+ 10 - 0
job-bytedance/pom.xml

@@ -178,6 +178,16 @@
             <artifactId>spring-boot-starter-data-redis</artifactId>
         </dependency>
         <dependency>
+            <groupId>redis.clients</groupId>
+            <artifactId>jedis</artifactId>
+            <version>3.1.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.redisson</groupId>
+            <artifactId>redisson</artifactId>
+            <version>3.36.0</version>
+        </dependency>
+        <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-pool2</artifactId>
             <version>2.6.2</version>

+ 16 - 4
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/impl/BytedanceReportNewServiceImpl.java

@@ -7,6 +7,7 @@ import cn.com.ctop.job.bytedance.data.mapper.AccountReportNewMapper;
 import cn.com.ctop.job.bytedance.data.service.IBytedanceReportNewService;
 import cn.com.ctop.job.bytedance.data.utils.Check;
 import cn.com.ctop.job.bytedance.data.utils.HttpUtils;
+import cn.com.ctop.job.bytedance.data.utils.RateLimitUtil;
 import cn.com.ctop.job.bytedance.utils.FieldsConfig;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
@@ -21,6 +22,9 @@ import java.util.*;
 public class BytedanceReportNewServiceImpl implements IBytedanceReportNewService {
     @Autowired
     private AccountReportNewMapper accountReportNewMapper;
+    @Autowired
+    private RateLimitUtil rateLimitUtil;
+
     private String urlPath = "http://192.168.0.85:8765/webhook/bytedance_account_report";
     private static JSONArray metrics = new JSONArray();
 
@@ -42,7 +46,12 @@ public class BytedanceReportNewServiceImpl implements IBytedanceReportNewService
             paramsMap.put("page_size", 100);
             Map<String, String> headers = new HashMap<>();
             headers.put("Access-Token", accessToken);
-            String s = HttpUtils.httpGet(url, paramsMap, headers);
+            String s = null;
+            if (rateLimitUtil.acquire("1635316529903624", url)) {
+                s = HttpUtils.httpGet(url, paramsMap, headers);
+            }
+
+
             JSONObject result = JSONObject.parseObject(s);
             List<BytedanceAdvertiserReportHourly> hours = new ArrayList<>();
             Integer code = result.getInteger("code");
@@ -117,8 +126,8 @@ public class BytedanceReportNewServiceImpl implements IBytedanceReportNewService
                 log.info("账户ID:{},开始日期:{},结束日期:{},广告日报(账户粒度)数据请求成功", advertiserId, startDate, endDate);
                 JSONObject param = new JSONObject();
                 param.put("advertiser_id", advertiserId);
-                param.put("start_date", startDate.replace("-",""));
-                param.put("end_date", endDate.replace("-",""));
+                param.put("start_date", startDate.replace("-", ""));
+                param.put("end_date", endDate.replace("-", ""));
                 param.put("date_type", "hourly");
                 HttpUtils.httpPostRequest(urlPath, param, null);
             }
@@ -145,7 +154,10 @@ public class BytedanceReportNewServiceImpl implements IBytedanceReportNewService
             paramsMap.put("page_size", 100);
             Map<String, String> headers = new HashMap<>();
             headers.put("Access-Token", oauthToken);
-            String s = HttpUtils.httpGet(url, paramsMap, headers);
+            String s = null;
+            if (rateLimitUtil.acquire("1635316529903624", url)) {
+                s = HttpUtils.httpGet(url, paramsMap, headers);
+            }
             JSONObject result = JSONObject.parseObject(s);
             List<BytedanceAdvertiserReportDaily> dailies = new ArrayList<>();
             Integer code = result.getInteger("code");

+ 62 - 0
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/utils/RateLimitUtil.java

@@ -0,0 +1,62 @@
+package cn.com.ctop.job.bytedance.data.utils;
+
+import com.google.common.collect.ImmutableMap;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+import org.redisson.api.RRateLimiter;
+import org.redisson.api.RateIntervalUnit;
+import org.redisson.api.RateType;
+import org.redisson.api.RedissonClient;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.net.URI;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+
+@Slf4j
+@Component
+@RequiredArgsConstructor
+public class RateLimitUtil {
+    @Autowired
+    public RedissonClient redissonClient;
+    private static final Map<String, Boolean> initMap = new ConcurrentHashMap<>();
+    private static final Map<String, Integer> pathQpsMap = new ImmutableMap.Builder<String, Integer>()
+            .put("/open_api/v3.0/report/custom/get/", 24)
+            .put("/open_api/2/file/video/get/", 10)
+            .put("/open_api/v3.0/promotion/list/", 30)
+            .put("/open_api/v3.0/project/list/", 30)
+            .put("/open_api/2/customer_center/advertiser/list/", 50)
+            .put("/open_api/v3.0/tools/micro_app/list/", 10)
+            .put("/open_api/v3.0/tools/asset_link/list/", 10)
+            .put("/open_api/v3.0/tools/wechat_applet/list/", 10)
+            .put("/open_api/2/advertiser/fund/daily_stat/", 50)
+            .put("/open_api/2/advertiser/avatar/get/", 10)
+            .put("/open_api/2/agent/advertiser_info/query/", 10)
+            .put("/open_api/2/tools/site/get/", 10)
+            .build();
+
+    /**
+     * 自定义限流规则
+     */
+    public boolean acquire(String devAppId, String url) {
+        URI uri = URI.create(url);
+        String path = uri.getPath();
+        String limitKey = path.replaceAll("/$", "").replace("/", ":");
+        String key = "rate:limit" + limitKey + (Check.isNull(devAppId) ? "" : ":" + devAppId);
+        if (!initMap.containsKey(key)) {
+            Integer qps = pathQpsMap.getOrDefault(path, 10);
+            RRateLimiter limiter = redissonClient.getRateLimiter(key);
+            limiter.setRate(
+                    RateType.OVERALL,
+                    qps / 2,
+                    500,
+                    RateIntervalUnit.MILLISECONDS
+            );
+            initMap.put(key, true);
+        }
+        return redissonClient.getRateLimiter(key).tryAcquire(1, 1, TimeUnit.SECONDS);
+    }
+}