拉取项目 初版补全失败的id

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/entity/BytedanceByteAppInfo.java

@@ -1,5 +1,6 @@
 package cn.com.ctop.job.bytedance.data.entity;
 
+import com.alibaba.fastjson.JSONObject;
 import com.fasterxml.jackson.annotation.JsonFormat;
 import lombok.Data;
 
@@ -46,4 +47,13 @@ public class BytedanceByteAppInfo {
     @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
     private Date updateTime;
 
+    public BytedanceByteAppInfo() {
+    }
+
+    public BytedanceByteAppInfo(JSONObject entity, Long advertiserId) {
+        this.advertiserId = advertiserId;
+        this.byteAppName = entity.getString("name");
+        this.byteAppId = entity.getString("app_id");
+        this.instanceId = entity.getLong("instance_id");
+    }
 }

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/mapper/BytedanceCdpProjectMapper.java

@@ -1,6 +1,8 @@
 package cn.com.ctop.job.bytedance.data.mapper;
 
 import cn.com.ctop.job.bytedance.data.entity.BytedanceCdpProject;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
 import org.apache.ibatis.annotations.Param;
 
 import java.util.List;
@@ -18,4 +20,12 @@ public interface BytedanceCdpProjectMapper {
     void insertErrorAdvertiserId(@Param("advertiserId") Long advertiserId);
 
     List<Long> getAllAdverIdList();
+
+    List<Long> queryAdvertiserIds(@Param("db") String db, @Param("date") String date);
+
+    void replaceBytedanceByteAppInfo(@Param("list") JSONArray list, @Param("advertiserId") Long advertiserId);
+
+    void replaceBytedanceWxAppInfo(@Param("list") JSONArray list, @Param("advertiserId") Long advertiserId);
+
+    List<Long> queryMediaAdverIds(@Param("date") String date, @Param("mediaType") String mediaType);
 }

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/mapper/xml/BytedanceCdpProjectListMapper.xml

@@ -66,6 +66,44 @@
         </foreach>
     </insert>
 
+
+    <insert id="replaceBytedanceByteAppInfo">
+        replace into media_api.bytedance_byte_app_info (
+        advertiser_id,
+        byte_app_name,
+        byte_app_id,
+        instance_id
+        )
+        values
+        <foreach collection="list" item="item" separator=",">
+            (
+            #{advertiserId},
+            #{item.name},
+            #{item.app_id},
+            #{item.instance_id}
+            )
+        </foreach>
+    </insert>
+
+
+    <insert id="replaceBytedanceWxAppInfo">
+        replace into media_api.bytedance_wx_app_info (
+        advertiser_id,
+        app_name,
+        user_nme,
+        instance_id
+        )
+        values
+        <foreach collection="list" item="item" separator=",">
+            (
+            #{advertiserId},
+            #{item.name},
+            #{item.user_name},
+            #{item.instance_id}
+            )
+        </foreach>
+    </insert>
+
     <insert id="insertErrorAdvertiserId">
         replace
         into media_api.error_advertiser (advertiser_id)
@@ -74,11 +112,29 @@
         )
     </insert>
 
-
     <select id="getAllAdverIdList" resultType="Long">
         SELECT advertiser_id
-        FROM deepthink.bytedance_advertiser_list
-        order by  create_time
+        FROM media_api.error_advertiser
+        order by create_time
     </select>
 
+    <select id="queryAdvertiserIds" resultType="Long">
+        SELECT DISTINCT advertiser_id
+        FROM media_api.${db}
+        where stat_time_day = #{date}
+        order by create_time
+    </select>
+
+
+    <select id="queryMediaAdverIds" resultType="Long">
+        SELECT DISTINCT advertiser_id
+        FROM media_api.bytedance_cdp_project_list
+        where micro_promotion_type = #{mediaType}
+        <if test="date != null">
+            AND project_modify_time &gt;= CONCAT(#{date}, ' 00:00:00')
+            AND project_modify_time &lt;= CONCAT(#{date}, ' 23:59:59')
+        </if>
+    </select>
+
+
 </mapper>

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/IBytedanceCdpProjectService.java

@@ -10,7 +10,15 @@ import java.util.List;
  */
 public interface IBytedanceCdpProjectService {
 
-    void getProjectList(Long advertiserId, String type, Integer page);
+    void getProjectList(Long advertiserId, String type, String date, Integer page);
 
     List<Long> getAllAdverIdList();
+
+    List<Long> queryAdvertiserIds(String db, String date);
+
+    void getByteAppInfo(Long advertiserId);
+
+    void getWxAppInfo(Long advertiserId);
+
+    List<Long> queryMediaAdverIds(String date, String mediaType);
 }

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/impl/BytedanceCdpProjectServiceImpl.java

@@ -6,6 +6,7 @@ import cn.com.ctop.job.bytedance.data.mapper.BytedanceRefreshTokenMapper;
 import cn.com.ctop.job.bytedance.data.service.IBytedanceCdpProjectService;
 import cn.com.ctop.job.bytedance.data.utils.Check;
 import cn.com.ctop.job.bytedance.data.utils.HttpUtils;
+import cn.com.ctop.job.bytedance.data.utils.Okhttp3Utils;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import lombok.extern.slf4j.Slf4j;
@@ -43,12 +44,21 @@ public class BytedanceCdpProjectServiceImpl implements IBytedanceCdpProjectServi
     }
 
     @Override
-    public void getProjectList(Long advertiserId, String type, Integer page) {
+    public List<Long> queryAdvertiserIds(String db, String date) {
+        return bytedanceCdpProjectMapper.queryAdvertiserIds(db, date);
+    }
+
+    @Override
+    public void getProjectList(Long advertiserId, String type, String date, Integer page) {
         if (!Check.isNull(type))
             log.info("--------执行：{}", advertiserId);
         try {
             boolean nextPage = false;
             String accessToken = tokenMapper.onlyGetToken(advertiserId);
+            if (Check.isNull(accessToken)) {
+                log.error("账户（{}）查询token失败", advertiserId);
+                return;
+            }
             Map<String, Object> params = new HashMap<>();
             params.put("advertiser_id", advertiserId);
             if (!Check.isNull(type)) {
@@ -84,7 +94,7 @@ public class BytedanceCdpProjectServiceImpl implements IBytedanceCdpProjectServi
                 bytedanceCdpProjectMapper.insertErrorAdvertiserId(advertiserId);
             }
             if (nextPage) {
-                getProjectList(advertiserId, type, page + 1);
+                getProjectList(advertiserId, type, date, page + 1);
             }
         } catch (Exception e) {
             bytedanceCdpProjectMapper.insertErrorAdvertiserId(advertiserId);
@@ -93,4 +103,107 @@ public class BytedanceCdpProjectServiceImpl implements IBytedanceCdpProjectServi
         }
     }
 
+    @Override
+    public void getByteAppInfo(Long advertiserId) {
+        this.getByteAppInfoVo(advertiserId, "CREATE_ONLY", 1, 1);
+        this.getByteAppInfoVo(advertiserId, "SHARE_ONLY", 1, 1);
+    }
+
+    private void getByteAppInfoVo(Long advertiserId, String searchType, Integer page, int count) {
+        log.info("------执行{}", advertiserId);
+        boolean nextPage = false;
+        String accessToken = tokenMapper.onlyGetToken(advertiserId);
+        if (Check.isNull(accessToken)) {
+            log.error("账户（{}）查询token失败", advertiserId);
+            return;
+        }
+        JSONObject params = new JSONObject();
+        params.put("advertiser_id", advertiserId);
+        params.put("page", page);
+        //页面大小，默认值为10，最大值100
+        params.put("page_size", 100);
+        JSONObject filtering = new JSONObject();
+        //CREATE_ONLY本户创建，SHARE_ONLY 只查共享应用
+        filtering.put("search_type", searchType);
+//        filtering.put("audit_status", "AUDIT_ACCEPTED");
+        params.put("filtering", filtering);
+        //获取微信小程序列表
+        String url = "https://api.oceanengine.com/open_api/v3.0/tools/micro_app/list/";
+        JSONObject obj = Okhttp3Utils.bytedanceGetRequest(accessToken, url, params);
+        Integer code = obj.getInteger("code");
+        if (code == 0) {
+            JSONObject data = obj.getJSONObject("data");
+            Integer totalPage = data.getJSONObject("page_info").getInteger("total_page");
+            if (totalPage != 0 && totalPage > data.getJSONObject("page_info").getInteger("page")) {
+                nextPage = true;
+            }
+            JSONArray list = data.getJSONArray("list");
+            if (!Check.isNull(list)) {
+                bytedanceCdpProjectMapper.replaceBytedanceByteAppInfo(list, advertiserId);
+            }
+        } else {
+            //40110请求超频
+            if (40110 == code && count <= 3)
+                getByteAppInfoVo(advertiserId, searchType, page, count + 1);
+            log.error("账户{}获取字节小程序列表失败，入参：{}\n结果：{}", advertiserId, JSONObject.toJSONString(params), obj);
+        }
+        if (nextPage) {
+            getByteAppInfoVo(advertiserId, searchType, page + 1, 1);
+        }
+    }
+
+    @Override
+    public void getWxAppInfo(Long advertiserId) {
+        this.getWxAppInfoVo(advertiserId, "CREATE_ONLY", 1, 1);
+        this.getWxAppInfoVo(advertiserId, "SHARE_ONLY", 1, 1);
+    }
+
+    private void getWxAppInfoVo(Long advertiserId, String searchType, Integer page, int count) {
+        boolean nextPage = false;
+        String accessToken = tokenMapper.onlyGetToken(advertiserId);
+        if (Check.isNull(accessToken)) {
+            log.error("账户（{}）查询token失败", advertiserId);
+            return;
+        }
+        JSONObject params = new JSONObject();
+        params.put("advertiser_id", advertiserId);
+        params.put("page", page);
+        //页面大小，默认值为10，最大值100
+        params.put("page_size", 100);
+        JSONObject filtering = new JSONObject();
+        //CREATE_ONLY本户创建，SHARE_ONLY 只查共享应用
+        filtering.put("search_type", searchType);
+        //状态 审核通过：AUDIT_ACCEPTED
+//        filtering.put("audit_status", "AUDIT_ACCEPTED");
+        params.put("filtering", filtering);
+        //获取微信小程序列表
+        String url = "https://api.oceanengine.com/open_api/v3.0/tools/wechat_applet/list/";
+        JSONObject obj = Okhttp3Utils.bytedanceGetRequest(accessToken, url, params);
+        Integer code = obj.getInteger("code");
+        if (code == 0) {
+            JSONObject data = obj.getJSONObject("data");
+            Integer totalPage = data.getJSONObject("page_info").getInteger("total_page");
+            if (totalPage != 0 && totalPage > data.getJSONObject("page_info").getInteger("page")) {
+                nextPage = true;
+            }
+            JSONArray list = data.getJSONArray("list");
+            if (!Check.isNull(list)) {
+                bytedanceCdpProjectMapper.replaceBytedanceWxAppInfo(list, advertiserId);
+            }
+        } else {
+            //40110请求超频
+            if (40110 == code && count <= 3)
+                getByteAppInfoVo(advertiserId, searchType, page, count + 1);
+            log.error("账户{}获取微信小程序列表失败，入参：{}\n结果：{}", advertiserId, JSONObject.toJSONString(params), obj);
+        }
+        if (nextPage) {
+            getWxAppInfoVo(advertiserId, searchType, page + 1, 1);
+        }
+    }
+
+    @Override
+    public List<Long> queryMediaAdverIds(String date, String mediaType) {
+        return bytedanceCdpProjectMapper.queryMediaAdverIds(date, mediaType);
+    }
+
 }

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/utils/DateUtils.java

@@ -64,8 +64,8 @@ public class DateUtils {
         return sdf.format(date);
     }
 
-    public static Integer getIntegerDate(String date) {
-        return Integer.valueOf(date.replace("-", ""));
+    public static String getYYMMDDTime(String date) {
+        return date.replace("-", "");
     }
 
 

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedanceCdpProjectJob.java

@@ -3,6 +3,7 @@ package cn.com.ctop.job.bytedance.handler;
 
 import cn.com.ctop.job.bytedance.data.service.IBytedanceCdpProjectService;
 import cn.com.ctop.job.bytedance.data.utils.Check;
+import cn.com.ctop.job.bytedance.data.utils.DateUtils;
 import com.google.common.collect.Lists;
 import com.xxl.job.core.handler.annotation.XxlJob;
 import lombok.extern.slf4j.Slf4j;
@@ -25,21 +26,45 @@ public class BytedanceCdpProjectJob {
     private IBytedanceCdpProjectService cdpProjectService;
 
     private static ExecutorService cdpProjectExecutorService = Executors.newFixedThreadPool(5);
-    private static ExecutorService nameExecutorService = Executors.newFixedThreadPool(3);
+    private static ExecutorService cdpNProjectExecutorService = Executors.newFixedThreadPool(3);
+    private static ExecutorService cdpYProjectExecutorService = Executors.newFixedThreadPool(3);
 
     @XxlJob("getProjectListJob")
     public void getProjectListJob() throws Exception {
         List<Long> adIds = cdpProjectService.getAllAdverIdList();
         log.info("---------账户总数:{}", adIds.size());
-        List<List<Long>> smallLists = Lists.partition(adIds, 10000); // 使用Guava的partition方法拆分列表
+        List<List<Long>> smallLists = Lists.partition(adIds, 2000); // 使用Guava的partition方法拆分列表
         if (!Check.isNull(smallLists)) {
             smallLists.forEach(list -> cdpProjectExecutorService.submit(() -> {
                 for (Long id : list) {
-                    cdpProjectService.getProjectList(id, null, 1);
-                    cdpProjectService.getProjectList(id, "del", 1);
+                    cdpProjectService.getProjectList(id, null, null, 1);
+                    cdpProjectService.getProjectList(id, "del", null, 1);
                 }
             }));
         }
     }
 
+
+    @XxlJob("getNearlyTwoDaysProjects")
+    public void getProjectListToday() throws Exception {
+        //拉取今天数据
+        String today = DateUtils.getNowDate("yyyy-MM-dd");
+        List<Long> nowAdIds = cdpProjectService.queryAdvertiserIds("bytedance_advertiser_report_hour", DateUtils.getYYMMDDTime(today));
+        if (!Check.isNull(nowAdIds)) {
+            nowAdIds.forEach(id -> cdpNProjectExecutorService.submit(() -> {
+                cdpProjectService.getProjectList(id, null, today, 1);
+                cdpProjectService.getProjectList(id, "del", today, 1);
+            }));
+        }
+        //拉取昨天数据
+        String yesterday = DateUtils.getNowAddDate("yyyy-MM-dd", -1);
+        List<Long> adIds = cdpProjectService.queryAdvertiserIds("bytedance_advertiser_report_daily", DateUtils.getYYMMDDTime(yesterday));
+        if (!Check.isNull(adIds)) {
+            adIds.forEach(id -> cdpNProjectExecutorService.submit(() -> {
+                cdpProjectService.getProjectList(id, null, yesterday, 1);
+                cdpProjectService.getProjectList(id, "del", yesterday, 1);
+            }));
+        }
+    }
+
 }