点众

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/controller/DianZhongController.java

@@ -13,12 +13,16 @@ import org.springframework.web.bind.annotation.RestController;
 
 import javax.annotation.Resource;
 import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 
 @RestController
 @RequestMapping("/bytedance/dianzhong")
 @Api(tags = "点众拉取")
 @Slf4j
 public class DianZhongController {
+    private static final ExecutorService executorService = Executors.newFixedThreadPool(5);
+
     @Autowired
     private IDianZhongService dianZhongService;
 
@@ -45,7 +49,13 @@ public class DianZhongController {
 
     @GetMapping("/getUrlData")
     public void urlData(String statDate) {
-        dianZhongService.getUrlData(statDate);
+        executorService.submit(new Runnable() {
+            @Override
+            public void run() {
+                dianZhongService.getUrlData(statDate);
+            }
+        });
+
     }
 
 }

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/mapper/xml/DianZhongMapper.xml

@@ -26,6 +26,9 @@
         select task_id,url,type,client_id
         from media_api.dianzhong_data_download_url
         where downloaded = 0 and url is not null and url != '-1'
+        <if test="statDate != null">
+            and stat_date = #{statDate}
+        </if>
         order by stat_date, type desc , create_time
     </select>
 

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/impl/DianZhongServiceImpl.java

@@ -136,7 +136,7 @@ public class DianZhongServiceImpl implements IDianZhongService {
                 String url = task.getString("url");
                 String type = task.getString("type");
                 String clientId = task.getString("client_id");
-
+                log.info("下载点众数据");
                 downloadFileRestTemplate.execute(url, org.springframework.http.HttpMethod.GET, null,
                         (ResponseExtractor<Void>) response -> {
                             try (BufferedReader reader = new BufferedReader(

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/utils/doris/sink/SimpleHttpDorisSinkClient.java

@@ -68,6 +68,7 @@ public class SimpleHttpDorisSinkClient implements DorisSinkClient {
             put.setHeader("Authorization", "Basic " + auth);
         }
 
+        put.setHeader("max_filter_ratio", "1");
         put.setHeader("format", format); // json
         put.setHeader("strip_outer_array", "true"); // json array
         put.setHeader("fuzzy_parse", "true"); // json array