拉取小程序，all

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/mapper/BytedanceCdpProjectMapper.java

@@ -2,7 +2,6 @@ package cn.com.ctop.job.bytedance.data.mapper;
 
 import cn.com.ctop.job.bytedance.data.entity.BytedanceCdpProject;
 import com.alibaba.fastjson.JSONArray;
-import com.alibaba.fastjson.JSONObject;
 import org.apache.ibatis.annotations.Param;
 
 import java.util.List;
@@ -17,7 +16,7 @@ public interface BytedanceCdpProjectMapper {
 
     void replaceBytedanceCdpProjectList(@Param("projectList") List<BytedanceCdpProject> projectList);
 
-    void insertErrorAdvertiserId(@Param("advertiserId") Long advertiserId);
+    void insertErrorAdvertiserId(@Param("advertiserId") Long advertiserId, @Param("type") String type);
 
     List<Long> getAllAdverIdList();
 

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/mapper/xml/BytedanceCdpProjectListMapper.xml

@@ -106,9 +106,10 @@
 
     <insert id="insertErrorAdvertiserId">
         replace
-        into media_api.error_advertiser (advertiser_id)
+        into media_api.error_advertiser (advertiser_id,`type`)
         values(
-        #{advertiserId}
+        #{advertiserId},
+        #{type}
         )
     </insert>
 

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/IBytedanceCdpProjectService.java

@@ -9,8 +9,7 @@ import java.util.List;
  * @date 2025-04-23
  */
 public interface IBytedanceCdpProjectService {
-
-    void getProjectList(Long advertiserId, String type, String date, Integer page);
+    void getProjectList(Long advertiserId, String date);
 
     List<Long> getAllAdverIdList();
 

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/impl/BytedanceCdpProjectServiceImpl.java

@@ -49,9 +49,12 @@ public class BytedanceCdpProjectServiceImpl implements IBytedanceCdpProjectServi
     }
 
     @Override
-    public void getProjectList(Long advertiserId, String type, String date, Integer page) {
-        if (!Check.isNull(type))
-            log.info("--------执行：{}", advertiserId);
+    public void getProjectList(Long advertiserId, String date) {
+        this.getProjectListVo(advertiserId, null, date, 1, 1);
+        this.getProjectListVo(advertiserId, "del", date, 1, 1);
+    }
+
+    private void getProjectListVo(Long advertiserId, String type, String date, Integer page, int count) {
         try {
             boolean nextPage = false;
             String accessToken = tokenMapper.onlyGetToken(advertiserId);
@@ -72,10 +75,9 @@ public class BytedanceCdpProjectServiceImpl implements IBytedanceCdpProjectServi
             String url = "https://api.oceanengine.com/open_api/v3.0/project/list/";
             String results = HttpUtils.httpGet(url, params, HttpUtils.postMHeaders(accessToken));
             JSONObject obj = JSONObject.parseObject(results);
-//            System.out.println("入参：" + params);
-//            System.out.println("结果：" + obj);
             Integer code = obj.getInteger("code");
             if (code == 0) {
+                log.info("---账户（{}）获取项目成功", advertiserId);
                 JSONObject data = obj.getJSONObject("data");
                 Integer totalPage = data.getJSONObject("page_info").getInteger("total_page");
                 if (totalPage != 0 && totalPage > data.getJSONObject("page_info").getInteger("page")) {
@@ -90,14 +92,15 @@ public class BytedanceCdpProjectServiceImpl implements IBytedanceCdpProjectServi
                 if (!Check.isNull(projectList))
                     bytedanceCdpProjectMapper.replaceBytedanceCdpProjectList(projectList);
             } else {
+                //40100请求超频
+                if (40100 == code && count <= 3)
+                    getProjectListVo(advertiserId, type, date, page, count + 1);
                 log.error("获取项目列表失败(url:{}),入参：{}\n结果：{}", url, JSONObject.toJSONString(params), obj);
-                bytedanceCdpProjectMapper.insertErrorAdvertiserId(advertiserId);
             }
             if (nextPage) {
-                getProjectList(advertiserId, type, date, page + 1);
+                getProjectListVo(advertiserId, type, date, page + 1, 1);
             }
         } catch (Exception e) {
-            bytedanceCdpProjectMapper.insertErrorAdvertiserId(advertiserId);
             e.printStackTrace();
             log.error(e.getMessage());
         }
@@ -110,45 +113,52 @@ public class BytedanceCdpProjectServiceImpl implements IBytedanceCdpProjectServi
     }
 
     private void getByteAppInfoVo(Long advertiserId, String searchType, Integer page, int count) {
-        log.info("------执行{}", advertiserId);
-        boolean nextPage = false;
-        String accessToken = tokenMapper.onlyGetToken(advertiserId);
-        if (Check.isNull(accessToken)) {
-            log.error("账户（{}）查询token失败", advertiserId);
-            return;
-        }
-        JSONObject params = new JSONObject();
-        params.put("advertiser_id", advertiserId);
-        params.put("page", page);
-        //页面大小，默认值为10，最大值100
-        params.put("page_size", 100);
-        JSONObject filtering = new JSONObject();
-        //CREATE_ONLY本户创建，SHARE_ONLY 只查共享应用
-        filtering.put("search_type", searchType);
+        try {
+            boolean nextPage = false;
+            String accessToken = tokenMapper.onlyGetToken(advertiserId);
+            if (Check.isNull(accessToken)) {
+                log.error("账户（{}）查询token失败", advertiserId);
+                return;
+            }
+            JSONObject params = new JSONObject();
+            params.put("advertiser_id", advertiserId);
+            params.put("page", page);
+            //页面大小，默认值为10，最大值100
+            params.put("page_size", 100);
+            JSONObject filtering = new JSONObject();
+            //CREATE_ONLY本户创建，SHARE_ONLY 只查共享应用
+            filtering.put("search_type", searchType);
 //        filtering.put("audit_status", "AUDIT_ACCEPTED");
-        params.put("filtering", filtering);
-        //获取微信小程序列表
-        String url = "https://api.oceanengine.com/open_api/v3.0/tools/micro_app/list/";
-        JSONObject obj = Okhttp3Utils.bytedanceGetRequest(accessToken, url, params);
-        Integer code = obj.getInteger("code");
-        if (code == 0) {
-            JSONObject data = obj.getJSONObject("data");
-            Integer totalPage = data.getJSONObject("page_info").getInteger("total_page");
-            if (totalPage != 0 && totalPage > data.getJSONObject("page_info").getInteger("page")) {
-                nextPage = true;
+            params.put("filtering", filtering);
+            //获取微信小程序列表
+            String url = "https://api.oceanengine.com/open_api/v3.0/tools/micro_app/list/";
+            JSONObject obj = Okhttp3Utils.bytedanceGetRequest(accessToken, url, params);
+            Integer code = obj.getInteger("code");
+            if (code == 0) {
+                log.info("---账户（{}）获取抖小信息成功", advertiserId);
+                JSONObject data = obj.getJSONObject("data");
+                Integer totalPage = data.getJSONObject("page_info").getInteger("total_page");
+                if (totalPage != 0 && totalPage > data.getJSONObject("page_info").getInteger("page")) {
+                    nextPage = true;
+                }
+                JSONArray list = data.getJSONArray("list");
+                if (!Check.isNull(list)) {
+                    bytedanceCdpProjectMapper.replaceBytedanceByteAppInfo(list, advertiserId);
+                }
+            } else {
+                //40110请求超频
+                if (40110 == code && count <= 3)
+                    getByteAppInfoVo(advertiserId, searchType, page, count + 1);
+                log.error("账户（{}）获取抖小列表失败，入参：{}\n结果：{}", advertiserId, JSONObject.toJSONString(params), obj);
+                bytedanceCdpProjectMapper.insertErrorAdvertiserId(advertiserId, "tt");
             }
-            JSONArray list = data.getJSONArray("list");
-            if (!Check.isNull(list)) {
-                bytedanceCdpProjectMapper.replaceBytedanceByteAppInfo(list, advertiserId);
+            if (nextPage) {
+                getByteAppInfoVo(advertiserId, searchType, page + 1, 1);
             }
-        } else {
-            //40110请求超频
-            if (40110 == code && count <= 3)
-                getByteAppInfoVo(advertiserId, searchType, page, count + 1);
-            log.error("账户{}获取字节小程序列表失败，入参：{}\n结果：{}", advertiserId, JSONObject.toJSONString(params), obj);
-        }
-        if (nextPage) {
-            getByteAppInfoVo(advertiserId, searchType, page + 1, 1);
+        } catch (Exception e) {
+            bytedanceCdpProjectMapper.insertErrorAdvertiserId(advertiserId, "tt");
+            e.printStackTrace();
+            log.error(e.getMessage());
         }
     }
 
@@ -159,45 +169,53 @@ public class BytedanceCdpProjectServiceImpl implements IBytedanceCdpProjectServi
     }
 
     private void getWxAppInfoVo(Long advertiserId, String searchType, Integer page, int count) {
-        boolean nextPage = false;
-        String accessToken = tokenMapper.onlyGetToken(advertiserId);
-        if (Check.isNull(accessToken)) {
-            log.error("账户（{}）查询token失败", advertiserId);
-            return;
-        }
-        JSONObject params = new JSONObject();
-        params.put("advertiser_id", advertiserId);
-        params.put("page", page);
-        //页面大小，默认值为10，最大值100
-        params.put("page_size", 100);
-        JSONObject filtering = new JSONObject();
-        //CREATE_ONLY本户创建，SHARE_ONLY 只查共享应用
-        filtering.put("search_type", searchType);
-        //状态 审核通过：AUDIT_ACCEPTED
+        try {
+            boolean nextPage = false;
+            String accessToken = tokenMapper.onlyGetToken(advertiserId);
+            if (Check.isNull(accessToken)) {
+                log.error("账户（{}）查询token失败", advertiserId);
+                return;
+            }
+            JSONObject params = new JSONObject();
+            params.put("advertiser_id", advertiserId);
+            params.put("page", page);
+            //页面大小，默认值为10，最大值100
+            params.put("page_size", 100);
+            JSONObject filtering = new JSONObject();
+            //CREATE_ONLY本户创建，SHARE_ONLY 只查共享应用
+            filtering.put("search_type", searchType);
+            //状态 审核通过：AUDIT_ACCEPTED
 //        filtering.put("audit_status", "AUDIT_ACCEPTED");
-        params.put("filtering", filtering);
-        //获取微信小程序列表
-        String url = "https://api.oceanengine.com/open_api/v3.0/tools/wechat_applet/list/";
-        JSONObject obj = Okhttp3Utils.bytedanceGetRequest(accessToken, url, params);
-        Integer code = obj.getInteger("code");
-        if (code == 0) {
-            JSONObject data = obj.getJSONObject("data");
-            Integer totalPage = data.getJSONObject("page_info").getInteger("total_page");
-            if (totalPage != 0 && totalPage > data.getJSONObject("page_info").getInteger("page")) {
-                nextPage = true;
+            params.put("filtering", filtering);
+            //获取微信小程序列表
+            String url = "https://api.oceanengine.com/open_api/v3.0/tools/wechat_applet/list/";
+            JSONObject obj = Okhttp3Utils.bytedanceGetRequest(accessToken, url, params);
+            Integer code = obj.getInteger("code");
+            if (code == 0) {
+                log.info("---账户（{}）获取微小信息成功", advertiserId);
+                JSONObject data = obj.getJSONObject("data");
+                Integer totalPage = data.getJSONObject("page_info").getInteger("total_page");
+                if (totalPage != 0 && totalPage > data.getJSONObject("page_info").getInteger("page")) {
+                    nextPage = true;
+                }
+                JSONArray list = data.getJSONArray("list");
+                if (!Check.isNull(list)) {
+                    bytedanceCdpProjectMapper.replaceBytedanceWxAppInfo(list, advertiserId);
+                }
+            } else {
+                //40110请求超频
+                if (40110 == code && count <= 3)
+                    getWxAppInfoVo(advertiserId, searchType, page, count + 1);
+                log.error("账户（{}）获取微小列表失败，入参：{}\n结果：{}", advertiserId, JSONObject.toJSONString(params), obj);
+                bytedanceCdpProjectMapper.insertErrorAdvertiserId(advertiserId, "wx");
             }
-            JSONArray list = data.getJSONArray("list");
-            if (!Check.isNull(list)) {
-                bytedanceCdpProjectMapper.replaceBytedanceWxAppInfo(list, advertiserId);
+            if (nextPage) {
+                getWxAppInfoVo(advertiserId, searchType, page + 1, 1);
             }
-        } else {
-            //40110请求超频
-            if (40110 == code && count <= 3)
-                getByteAppInfoVo(advertiserId, searchType, page, count + 1);
-            log.error("账户{}获取微信小程序列表失败，入参：{}\n结果：{}", advertiserId, JSONObject.toJSONString(params), obj);
-        }
-        if (nextPage) {
-            getWxAppInfoVo(advertiserId, searchType, page + 1, 1);
+        } catch (Exception e) {
+            bytedanceCdpProjectMapper.insertErrorAdvertiserId(advertiserId, "wx");
+            e.printStackTrace();
+            log.error(e.getMessage());
         }
     }
 

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedanceCdpProjectJob.java

@@ -4,7 +4,6 @@ package cn.com.ctop.job.bytedance.handler;
 import cn.com.ctop.job.bytedance.data.service.IBytedanceCdpProjectService;
 import cn.com.ctop.job.bytedance.data.utils.Check;
 import cn.com.ctop.job.bytedance.data.utils.DateUtils;
-import com.google.common.collect.Lists;
 import com.xxl.job.core.handler.annotation.XxlJob;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -25,45 +24,112 @@ public class BytedanceCdpProjectJob {
     @Autowired
     private IBytedanceCdpProjectService cdpProjectService;
 
-    private static ExecutorService cdpProjectExecutorService = Executors.newFixedThreadPool(5);
-    private static ExecutorService cdpNProjectExecutorService = Executors.newFixedThreadPool(3);
-    private static ExecutorService cdpYProjectExecutorService = Executors.newFixedThreadPool(3);
-
-    @XxlJob("getProjectListJob")
-    public void getProjectListJob() throws Exception {
-        List<Long> adIds = cdpProjectService.getAllAdverIdList();
-        log.info("---------账户总数:{}", adIds.size());
-        List<List<Long>> smallLists = Lists.partition(adIds, 2000); // 使用Guava的partition方法拆分列表
-        if (!Check.isNull(smallLists)) {
-            smallLists.forEach(list -> cdpProjectExecutorService.submit(() -> {
-                for (Long id : list) {
-                    cdpProjectService.getProjectList(id, null, null, 1);
-                    cdpProjectService.getProjectList(id, "del", null, 1);
-                }
+    private static ExecutorService allService = Executors.newFixedThreadPool(5);
+    private static ExecutorService allService2 = Executors.newFixedThreadPool(5);
+
+    private static ExecutorService projectExecutorService = Executors.newFixedThreadPool(3);
+    private static ExecutorService ttAppsExecutorService = Executors.newFixedThreadPool(5);
+    private static ExecutorService wxAppsExecutorService = Executors.newFixedThreadPool(5);
+
+
+    @XxlJob("getAll")
+    public void getAll() {
+        log.info("[getAll] 任务启动...");
+        //BYTE_APP 字节小程序， WECHAT_APP：微信小程序
+        List<Long> ttList = cdpProjectService.queryMediaAdverIds(null, "BYTE_APP");
+        if (!Check.isNull(ttList)) {
+            ttList.forEach(id -> allService.submit(() -> {
+                cdpProjectService.getByteAppInfo(id);
+            }));
+        }
+        List<Long> wxlist = cdpProjectService.queryMediaAdverIds(null, "WECHAT_APP");
+        if (!Check.isNull(wxlist)) {
+            wxlist.forEach(id -> allService2.submit(() -> {
+                cdpProjectService.getWxAppInfo(id);
             }));
         }
+        log.info("[getAll] 任务结束");
     }
 
 
-    @XxlJob("getNearlyTwoDaysProjects")
-    public void getProjectListToday() throws Exception {
-        //拉取今天数据
-        String today = DateUtils.getNowDate("yyyy-MM-dd");
-        List<Long> nowAdIds = cdpProjectService.queryAdvertiserIds("bytedance_advertiser_report_hour", DateUtils.getYYMMDDTime(today));
-        if (!Check.isNull(nowAdIds)) {
-            nowAdIds.forEach(id -> cdpNProjectExecutorService.submit(() -> {
-                cdpProjectService.getProjectList(id, null, today, 1);
-                cdpProjectService.getProjectList(id, "del", today, 1);
-            }));
+    @XxlJob("nowProjects")
+    public void nowProjects() {
+        log.info("[nowProjects] 任务启动...");
+        try {
+            //拉取今天数据
+            String today = DateUtils.getNowDate("yyyy-MM-dd");
+            List<Long> nowAdIds = cdpProjectService.queryAdvertiserIds("bytedance_advertiser_report_hour", DateUtils.getYYMMDDTime(today));
+            if (!Check.isNull(nowAdIds)) {
+                nowAdIds.forEach(id -> projectExecutorService.submit(() -> cdpProjectService.getProjectList(id, today)));
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+            log.error("[nowProjects]" + e.getMessage());
+        } finally {
+            log.info("[nowProjects] 任务结束");
         }
-        //拉取昨天数据
-        String yesterday = DateUtils.getNowAddDate("yyyy-MM-dd", -1);
-        List<Long> adIds = cdpProjectService.queryAdvertiserIds("bytedance_advertiser_report_daily", DateUtils.getYYMMDDTime(yesterday));
-        if (!Check.isNull(adIds)) {
-            adIds.forEach(id -> cdpNProjectExecutorService.submit(() -> {
-                cdpProjectService.getProjectList(id, null, yesterday, 1);
-                cdpProjectService.getProjectList(id, "del", yesterday, 1);
-            }));
+    }
+
+    @XxlJob("yesProjects")
+    public void yesProjects() {
+        log.info("[yesProjects] 任务启动...");
+        try {
+            //拉取昨天数据
+            String yesterday = DateUtils.getNowAddDate("yyyy-MM-dd", -1);
+            List<Long> adIds = cdpProjectService.queryAdvertiserIds("bytedance_advertiser_report_daily", DateUtils.getYYMMDDTime(yesterday));
+            if (!Check.isNull(adIds)) {
+                adIds.forEach(id -> projectExecutorService.submit(() -> cdpProjectService.getProjectList(id, yesterday)));
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+            log.error("[yesProjects]" + e.getMessage());
+        } finally {
+            log.info("[yesProjects] 任务结束");
+        }
+    }
+
+    @XxlJob("nowApps")
+    public void nowApps() {
+        log.info("[nowApps] 任务启动...");
+        try {
+            String today = DateUtils.getNowDate("yyyy-MM-dd");
+            //BYTE_APP 字节小程序， WECHAT_APP：微信小程序
+            List<Long> ttList = cdpProjectService.queryMediaAdverIds(today, "BYTE_APP");
+            if (!Check.isNull(ttList)) {
+                ttList.forEach(id -> ttAppsExecutorService.submit(() -> cdpProjectService.getByteAppInfo(id)));
+            }
+            List<Long> wxlist = cdpProjectService.queryMediaAdverIds(today, "WECHAT_APP");
+            if (!Check.isNull(wxlist)) {
+                wxlist.forEach(id -> wxAppsExecutorService.submit(() -> cdpProjectService.getWxAppInfo(id)));
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+            log.error("[nowApps]" + e.getMessage());
+        } finally {
+            log.info("[nowApps] 任务结束");
+        }
+    }
+
+    @XxlJob("yesApps")
+    public void yesApps() {
+        log.info("[yesApps] 任务启动...");
+        try {
+            //昨天
+            String yesterday = DateUtils.getNowAddDate("yyyy-MM-dd", -1);
+            //BYTE_APP 字节小程序， WECHAT_APP：微信小程序
+            List<Long> ttList = cdpProjectService.queryMediaAdverIds(yesterday, "BYTE_APP");
+            if (!Check.isNull(ttList)) {
+                ttList.forEach(id -> ttAppsExecutorService.submit(() -> cdpProjectService.getByteAppInfo(id)));
+            }
+            List<Long> wxlist = cdpProjectService.queryMediaAdverIds(yesterday, "WECHAT_APP");
+            if (!Check.isNull(wxlist)) {
+                wxlist.forEach(id -> wxAppsExecutorService.submit(() -> cdpProjectService.getWxAppInfo(id)));
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+            log.error("[yesApps]" + e.getMessage());
+        } finally {
+            log.info("[yesApps] 任务结束");
         }
     }