拉取小程序，广点通创意

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/mapper/BytedanceCdpProjectMapper.java

@@ -20,6 +20,8 @@ public interface BytedanceCdpProjectMapper {
 
     List<Long> getAllAdverIdList(@Param("type") String type);
 
+    boolean delErrApps(@Param("type") String type);
+
     List<Long> queryAdvertiserIds(@Param("db") String db, @Param("date") String date);
 
     void replaceBytedanceByteAppInfo(@Param("list") JSONArray list, @Param("advertiserId") Long advertiserId);
@@ -27,4 +29,9 @@ public interface BytedanceCdpProjectMapper {
     void replaceBytedanceWxAppInfo(@Param("list") JSONArray list, @Param("advertiserId") Long advertiserId);
 
     List<Long> queryMediaAdverIds(@Param("date") String date, @Param("mediaType") String mediaType);
+
+    void replaceAdqDynamicCreateives(@Param("list") JSONArray list, @Param("advertiserId") Long advertiserId);
+
+    List<Long> queryADQAdverIds(String date);
+
 }

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/mapper/BytedanceRefreshTokenMapper.java

@@ -15,4 +15,6 @@ public interface BytedanceRefreshTokenMapper {
     void updateBaiduYunToken(@Param("obj") JSONObject obj);
 
     String onlyGetToken(@Param("advertiserId") Long advertiserId);
+
+    String onlyGetADQToken(@Param("advertiserId") Long advertiserId);
 }

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/mapper/xml/BytedanceCdpProjectListMapper.xml

@@ -120,6 +120,13 @@
         order by create_time
     </select>
 
+    <delete id="delErrApps">
+        DELETE
+        FROM media_api.error_advertiser
+        WHERE type =#{type}
+    </delete>
+
+
     <select id="queryAdvertiserIds" resultType="Long">
         SELECT DISTINCT advertiser_id
         FROM media_api.${db}
@@ -127,7 +134,6 @@
         order by create_time
     </select>
 
-
     <select id="queryMediaAdverIds" resultType="Long">
         SELECT DISTINCT advertiser_id
         FROM media_api.bytedance_cdp_project_list
@@ -138,5 +144,28 @@
         </if>
     </select>
 
+    <insert id="replaceAdqDynamicCreateives">
+        replace into media_api.adq_dynamic_createive_list(
+        advertiser_id,
+        adgroup_id,
+        dynamic_creative_id,
+        mini_program_id
+        )
+        values
+        <foreach collection="list" item="item" separator=",">
+            (
+            #{advertiserId},
+            #{item.adgroup_id},
+            #{item.dynamic_creative_id},
+            #{item.mini_program_id}
+            )
+        </foreach>
+    </insert>
+
+    <select id="queryADQAdverIds" resultType="Long">
+        SELECT DISTINCT advertiser_id
+        FROM deepthink.adq_advertiser_list
+    </select>
+
 
 </mapper>

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/mapper/xml/BytedanceRefreshTokenMapper.xml

@@ -35,9 +35,20 @@
     <select id="onlyGetToken" resultType="java.lang.String">
         SELECT access_token
         FROM deepthink.bytedance_auth_list
-        WHERE admin_advertiser_id = (SELECT admin_advertiser_id
-                                     FROM deepthink.bytedance_advertiser_list
-                                     WHERE advertiser_id = #{advertiserId}
+        WHERE admin_advertiser_id =
+              (SELECT admin_advertiser_id
+               FROM deepthink.bytedance_advertiser_list
+               WHERE advertiser_id = #{advertiserId}
+            LIMIT 1)
+    </select>
+
+    <select id="onlyGetADQToken" resultType="java.lang.String">
+        SELECT access_token
+        FROM deepthink.adq_auth_list
+        WHERE admin_advertiser_id =
+              (SELECT admin_advertiser_id
+               FROM deepthink.adq_advertiser_list
+               WHERE advertiser_id = #{advertiserId}
             LIMIT 1)
     </select>
 

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/IBytedanceCdpProjectService.java

@@ -13,6 +13,8 @@ public interface IBytedanceCdpProjectService {
 
     List<Long> getAllAdverIdList(String type);
 
+    boolean delErrApps(String type);
+
     List<Long> queryAdvertiserIds(String db, String date);
 
     void getByteAppInfo(Long advertiserId);
@@ -20,4 +22,9 @@ public interface IBytedanceCdpProjectService {
     void getWxAppInfo(Long advertiserId);
 
     List<Long> queryMediaAdverIds(String date, String mediaType);
+
+    List<Long> queryADQAdverIds(String date);
+
+    void getAdqCreatives(Long advertiserId, String date);
+
 }

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/impl/BytedanceCdpProjectServiceImpl.java

@@ -5,14 +5,17 @@ import cn.com.ctop.job.bytedance.data.mapper.BytedanceCdpProjectMapper;
 import cn.com.ctop.job.bytedance.data.mapper.BytedanceRefreshTokenMapper;
 import cn.com.ctop.job.bytedance.data.service.IBytedanceCdpProjectService;
 import cn.com.ctop.job.bytedance.data.utils.Check;
+import cn.com.ctop.job.bytedance.data.utils.DateUtils;
 import cn.com.ctop.job.bytedance.data.utils.HttpUtils;
 import cn.com.ctop.job.bytedance.data.utils.Okhttp3Utils;
+import cn.com.ctop.job.bytedance.data.utils.RandomStringGenerator;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -44,6 +47,11 @@ public class BytedanceCdpProjectServiceImpl implements IBytedanceCdpProjectServi
     }
 
     @Override
+    public boolean delErrApps(String type) {
+        return bytedanceCdpProjectMapper.delErrApps(type);
+    }
+
+    @Override
     public List<Long> queryAdvertiserIds(String db, String date) {
         return bytedanceCdpProjectMapper.queryAdvertiserIds(db, date);
     }
@@ -64,12 +72,14 @@ public class BytedanceCdpProjectServiceImpl implements IBytedanceCdpProjectServi
             }
             Map<String, Object> params = new HashMap<>();
             params.put("advertiser_id", advertiserId);
+            JSONObject filtering = new JSONObject();
             if (!Check.isNull(type)) {
-                JSONObject filtering = new JSONObject();
                 //不传默认返回不限（不含已删除）
                 filtering.put("status_first", "PROJECT_STATUS_DELETE");
-                params.put("filtering", filtering);
             }
+            filtering.put("project_modify_time", date);
+
+            params.put("filtering", filtering);
             params.put("page", page);
             params.put("page_size", 100);//最大100
             String url = "https://api.oceanengine.com/open_api/v3.0/project/list/";
@@ -230,4 +240,156 @@ public class BytedanceCdpProjectServiceImpl implements IBytedanceCdpProjectServi
         return bytedanceCdpProjectMapper.queryMediaAdverIds(date, mediaType);
     }
 
+    @Override
+    public List<Long> queryADQAdverIds(String date) {
+        return bytedanceCdpProjectMapper.queryADQAdverIds(date);
+    }
+
+    @Override
+    public void getAdqCreatives(Long advertiserId, String date) {
+        try {
+            String accessToken = tokenMapper.onlyGetADQToken(advertiserId);
+            if (Check.isNull(accessToken)) {
+                log.error("账户（{}）accessToken is null", advertiserId);
+                return;
+            }
+            this.rebackadqCreativesVo(advertiserId, accessToken, date, 1, true, 1);
+        } catch (Exception e) {
+            e.printStackTrace();
+            log.error("账户（{}）获取动态创意异常", e.getMessage());
+        }
+    }
+
+
+    private void rebackadqCreativesVo(Long accountId, String accessToken, String date, Integer page, Boolean isDeleted, int count) throws Exception {
+        boolean nextPage = false;
+        JSONObject returnJson = new JSONObject();
+        Map<String, Object> param = new HashMap<>();
+        long timestamp = Instant.now().getEpochSecond();
+        param.put("timestamp", String.valueOf(timestamp));
+        param.put("nonce", RandomStringGenerator.generateRandomString(32));
+        param.put("access_token", accessToken);
+        param.put("account_id", accountId);
+        param.put("page", page);
+        param.put("page_size", 100);
+        if (!Check.isNull(date)) {
+            JSONArray filtering = new JSONArray();
+            JSONObject greObj = new JSONObject();
+            greObj.put("field", "created_time");
+            greObj.put("operator", "GREATER_EQUALS");
+            JSONArray startDateVls = new JSONArray();
+            startDateVls.add(String.valueOf(DateUtils.dt2milli(date + " 00:00:00")));
+            greObj.put("values", startDateVls);
+            filtering.add(greObj);
+            JSONObject lesObj = new JSONObject();
+            lesObj.put("field", "created_time");
+            lesObj.put("operator", "LESS_EQUALS");
+            JSONArray endDateVls = new JSONArray();
+            endDateVls.add(String.valueOf(DateUtils.dt2milli(date + " 23:59:59")));
+            lesObj.put("values", endDateVls);
+            filtering.add(lesObj);
+            param.put("filtering", filtering);
+        }
+//        param.put("is_deleted", isDeleted);
+        JSONArray fields = new JSONArray();
+        fields.add("adgroup_id");
+        fields.add("dynamic_creative_id");
+        fields.add("creative_components");
+        param.put("fields", fields);
+        String result = HttpUtils.httpGet("https://api.e.qq.com/v3.0/dynamic_creatives/get", param, new HashMap<>());
+        JSONObject obj = JSONObject.parseObject(result);
+        if (!Check.isNull(obj)) {
+            if (0 == obj.getInteger("code")) {
+                log.info("---账户（{}）获取动态创意成功", accountId);
+                JSONObject data = obj.getJSONObject("data");
+                if (!Check.isNull(data)) {
+                    JSONObject pageInfo = data.getJSONObject("page_info");
+                    Integer totalPage = pageInfo.getInteger("total_page");
+                    if (totalPage != 0 && totalPage > pageInfo.getInteger("page")) {
+                        nextPage = true;
+                    }
+                    JSONArray array = data.getJSONArray("list");
+                    for (int i = 0; i < array.size(); i++) {
+                        JSONObject object = array.getJSONObject(i);
+                        getAppId(object);
+                    }
+                    if (!Check.isNull(array)) {
+                        bytedanceCdpProjectMapper.replaceAdqDynamicCreateives(array, accountId);
+                    }
+                }
+            } else {
+                if (count <= 3) {
+                    Thread.sleep(500);
+                    rebackadqCreativesVo(accountId, accessToken, date, page, isDeleted, count + 1);
+                }
+                log.error("账户（{}）获取动态创意失败，入参：{}\n，结果：{}", accountId, JSONObject.toJSONString(param), obj);
+            }
+        }
+        if (nextPage) {
+            rebackadqCreativesVo(accountId, accessToken, date, page + 1, isDeleted, 1);
+        }
+    }
+
+    private void getAppId(JSONObject object) {
+        JSONObject creativeComponents = object.getJSONObject("creative_components");
+        if (!Check.isNull(creativeComponents)) {
+            JSONArray textLink = creativeComponents.getJSONArray("text_link");
+            if (!Check.isNull(textLink)) {
+                JSONObject value = textLink.getJSONObject(0).getJSONObject("value");
+                if (!Check.isNull(value)) {
+                    JSONObject jumpInfo = value.getJSONObject("jump_info");
+                    if (!Check.isNull(jumpInfo)) {
+                        JSONObject pageSpec = jumpInfo.getJSONObject("page_spec");
+                        if (!Check.isNull(pageSpec)) {
+                            JSONObject wechatMiniProgramSpec = pageSpec.getJSONObject("wechat_mini_program_spec");
+                            if (!Check.isNull(wechatMiniProgramSpec)) {
+                                object.put("mini_program_id", wechatMiniProgramSpec.getString("mini_program_id"));
+                            }
+                        }
+                    }
+                }
+            }
+            if (!Check.isNull(object.getString("mini_program_id"))) {
+                return;
+            }
+            JSONArray jumpInfo = creativeComponents.getJSONArray("jump_info");
+            if (!Check.isNull(jumpInfo)) {
+                JSONObject value = jumpInfo.getJSONObject(0).getJSONObject("value");
+                if (!Check.isNull(value)) {
+                    JSONObject pageSpec = value.getJSONObject("page_spec");
+                    if (!Check.isNull(pageSpec)) {
+                        JSONObject wechatMiniProgramSpec = pageSpec.getJSONObject("wechat_mini_program_spec");
+                        if (!Check.isNull(wechatMiniProgramSpec)) {
+                            object.put("mini_program_id", wechatMiniProgramSpec.getString("mini_program_id"));
+                        }
+                    }
+                }
+            }
+            if (!Check.isNull(object.getString("mini_program_id"))) {
+                return;
+            }
+            JSONArray mainJumpInfo = creativeComponents.getJSONArray("main_jump_info");
+            if (!Check.isNull(mainJumpInfo)) {
+                JSONObject value = mainJumpInfo.getJSONObject(0).getJSONObject("value");
+                if (!Check.isNull(value)) {
+                    JSONObject pageSpec = value.getJSONObject("page_spec");
+                    if (!Check.isNull(pageSpec)) {
+                        JSONObject wechatMiniProgramSpec = pageSpec.getJSONObject("wechat_mini_program_spec");
+                        if (!Check.isNull(wechatMiniProgramSpec)) {
+                            object.put("mini_program_id", wechatMiniProgramSpec.getString("mini_program_id"));
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    public static void main(String[] args) {
+        Long l = DateUtils.dt2milli("2025-03-03" + " 00:00:00");
+        Long l2 = DateUtils.dt2milli("2025-03-03" + " 23:59:59");
+
+        System.out.println(Integer.valueOf(l + ""));
+        System.out.println(l);
+        System.out.println(l2);
+    }
 }

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/utils/DateUtils.java

@@ -5,6 +5,9 @@ import lombok.extern.slf4j.Slf4j;
 import java.text.DateFormat;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.Date;
@@ -12,6 +15,8 @@ import java.util.List;
 
 @Slf4j
 public class DateUtils {
+    public static final DateTimeFormatter FORMATTER_DATETIME_SECOND = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
+
     public static Date getDateByPattern(String day, String pattern) {
         SimpleDateFormat format = new SimpleDateFormat(pattern);
         try {
@@ -58,6 +63,12 @@ public class DateUtils {
         }
     }
 
+    public static Long dt2milli(String dateTime) {
+        LocalDateTime parse = LocalDateTime.parse(dateTime, FORMATTER_DATETIME_SECOND);
+        long epochMilli = LocalDateTime.from(parse).atZone(ZoneId.systemDefault()).toInstant().toEpochMilli();
+        return epochMilli/1000;
+    }
+
     public static String formatTimestamp(String format, Long ts) {
         SimpleDateFormat sdf = new SimpleDateFormat(format);
         Date date = new Date(ts * 1000);

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/utils/RandomStringGenerator.java

@@ -0,0 +1,21 @@
+package cn.com.ctop.job.bytedance.data.utils;
+
+import java.security.SecureRandom;
+
+public class RandomStringGenerator {
+    private static final String CHAR_SET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+    // 定义生成器
+    private static final SecureRandom RANDOM = new SecureRandom();
+
+    public static String generateRandomString(int length) {
+        StringBuilder sb = new StringBuilder(length);
+        for (int i = 0; i < length; i++) {
+            int index = RANDOM.nextInt(CHAR_SET.length());
+            sb.append(CHAR_SET.charAt(index));
+        }
+        return sb.toString();
+    }
+
+
+
+}

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedanceCdpProjectJob.java

@@ -24,31 +24,34 @@ public class BytedanceCdpProjectJob {
     @Autowired
     private IBytedanceCdpProjectService cdpProjectService;
 
-    private static ExecutorService allService = Executors.newFixedThreadPool(2);
-    private static ExecutorService allService2 = Executors.newFixedThreadPool(2);
+    private static ExecutorService allttService = Executors.newFixedThreadPool(2);
+    private static ExecutorService allwxService = Executors.newFixedThreadPool(2);
 
     private static ExecutorService projectExecutorService = Executors.newFixedThreadPool(3);
-    private static ExecutorService ttAppsExecutorService = Executors.newFixedThreadPool(5);
-    private static ExecutorService wxAppsExecutorService = Executors.newFixedThreadPool(5);
+    private static ExecutorService ttAppsExecutorService = Executors.newFixedThreadPool(3);
+    private static ExecutorService wxAppsExecutorService = Executors.newFixedThreadPool(3);
+    private static ExecutorService createiveAppsExecutorService = Executors.newFixedThreadPool(3);
 
 
     @XxlJob("getAll")
     public void getAll() {
         log.info("[getAll] 任务启动...");
-        //BYTE_APP 字节小程序， WECHAT_APP：微信小程序
         List<Long> ttList = cdpProjectService.getAllAdverIdList("tt");
         if (!Check.isNull(ttList)) {
-            ttList.forEach(id -> allService.submit(() -> {
+            if (cdpProjectService.delErrApps("tt"))
+                log.info("[getAll] 移除抖小失败记录");
+            ttList.forEach(id -> allttService.submit(() -> {
                 cdpProjectService.getByteAppInfo(id);
             }));
         }
         List<Long> wxlist = cdpProjectService.getAllAdverIdList("wx");
         if (!Check.isNull(wxlist)) {
-            wxlist.forEach(id -> allService2.submit(() -> {
+            if (cdpProjectService.delErrApps("wx"))
+                log.info("[getAll] 移除微小失败记录");
+            wxlist.forEach(id -> allwxService.submit(() -> {
                 cdpProjectService.getWxAppInfo(id);
             }));
         }
-        log.info("[getAll] 任务结束");
     }
 
 
@@ -65,8 +68,6 @@ public class BytedanceCdpProjectJob {
         } catch (Exception e) {
             e.printStackTrace();
             log.error("[nowProjects]" + e.getMessage());
-        } finally {
-            log.info("[nowProjects] 任务结束");
         }
     }
 
@@ -76,15 +77,17 @@ public class BytedanceCdpProjectJob {
         try {
             //拉取昨天数据
             String yesterday = DateUtils.getNowAddDate("yyyy-MM-dd", -1);
+            String yesterday2 = DateUtils.getNowAddDate("yyyy-MM-dd", -2);
             List<Long> adIds = cdpProjectService.queryAdvertiserIds("bytedance_advertiser_report_daily", DateUtils.getYYMMDDTime(yesterday));
             if (!Check.isNull(adIds)) {
-                adIds.forEach(id -> projectExecutorService.submit(() -> cdpProjectService.getProjectList(id, yesterday)));
+                adIds.forEach(id -> projectExecutorService.submit(() -> {
+                    cdpProjectService.getProjectList(id, yesterday);
+                    cdpProjectService.getProjectList(id, yesterday2);
+                }));
             }
         } catch (Exception e) {
             e.printStackTrace();
             log.error("[yesProjects]" + e.getMessage());
-        } finally {
-            log.info("[yesProjects] 任务结束");
         }
     }
 
@@ -105,8 +108,6 @@ public class BytedanceCdpProjectJob {
         } catch (Exception e) {
             e.printStackTrace();
             log.error("[nowApps]" + e.getMessage());
-        } finally {
-            log.info("[nowApps] 任务结束");
         }
     }
 
@@ -128,8 +129,39 @@ public class BytedanceCdpProjectJob {
         } catch (Exception e) {
             e.printStackTrace();
             log.error("[yesApps]" + e.getMessage());
-        } finally {
-            log.info("[yesApps] 任务结束");
+        }
+    }
+
+
+    @XxlJob("nowDynamicCreateives")
+    public void nowDynamicCreateives() {
+        log.info("[nowDynamicCreateives] 任务启动...");
+        try {
+            //拉取今天数据
+            String today = DateUtils.getNowDate("yyyy-MM-dd");
+            List<Long> ids = cdpProjectService.queryAdvertiserIds("adq_advertiser_report_hour", DateUtils.getYYMMDDTime(today));
+            if (!Check.isNull(ids)) {
+                ids.forEach(id -> createiveAppsExecutorService.submit(() -> cdpProjectService.getAdqCreatives(id, today)));
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+            log.error("[nowDynamicCreateives]" + e.getMessage());
+        }
+    }
+
+    @XxlJob("yesDynamicCreateives")
+    public void yesDynamicCreateives() {
+        log.info("[yesDynamicCreateives] 任务启动...");
+        try {
+            //拉取昨天数据
+            String yesterday = DateUtils.getNowAddDate("yyyy-MM-dd", -1);
+            List<Long> ids = cdpProjectService.queryAdvertiserIds("adq_advertiser_report_daily", DateUtils.getYYMMDDTime(yesterday));
+            if (!Check.isNull(ids)) {
+                ids.forEach(id -> createiveAppsExecutorService.submit(() -> cdpProjectService.getAdqCreatives(id, yesterday)));
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+            log.error("[yesDynamicCreateives]" + e.getMessage());
         }
     }