Merge remote-tracking branch 'origin/master'

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/config/DianZhongConfig.java

@@ -1,26 +1,43 @@
 package cn.com.ctop.job.bytedance.config;
 
-import org.springframework.boot.context.properties.ConfigurationProperties;
+import cn.com.ctop.job.bytedance.data.mapper.DianZhongMapper;
+import com.alibaba.fastjson.JSONObject;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.annotation.Configuration;
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
 @Configuration
-@ConfigurationProperties(prefix = "dianzhong.client")
 public class DianZhongConfig {
+    @Autowired
+    private DianZhongMapper dianZhongMapper;
     private static List<DZConfig> dzConfig;
     private static final Map<String, String> tokenMap = new ConcurrentHashMap<>();
 
+
     public List<DZConfig> getDzConfig() {
+        dzConfig = new ArrayList<>();
+        List<JSONObject> arr = dianZhongMapper.getAccountConfig();
+        for (JSONObject jsonObject : arr) {
+            String clientId = jsonObject.getString("account_id");
+            String token = jsonObject.getString("token");
+            String businessMethod = jsonObject.getString("business_method");
+            DZConfig config = new DZConfig(clientId, token, businessMethod);
+            dzConfig.add(config);
+            tokenMap.put(clientId, token);
+        }
         return dzConfig;
     }
 
     public String getToken(String clientId) {
-        if (tokenMap.isEmpty()) {
-            for (DZConfig config : dzConfig) {
-                tokenMap.put(config.getClientId(), config.getToken());
+        if (!tokenMap.containsKey(clientId)) {
+            String token = dianZhongMapper.getAccountToken(clientId);
+            if (StringUtils.isNotBlank(token)) {
+                tokenMap.put(clientId, token);
             }
         }
         return tokenMap.get(clientId);
@@ -35,6 +52,15 @@ public class DianZhongConfig {
         private String token;
         private String type;
 
+        public DZConfig() {
+        }
+
+        public DZConfig(String clientId, String token, String type) {
+            this.clientId = clientId;
+            this.token = token;
+            this.type = type;
+        }
+
         public String getClientId() {
             return clientId;
         }

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/config/RedisCacheConfig.java

@@ -0,0 +1,121 @@
+package cn.com.ctop.job.bytedance.config;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import com.fasterxml.jackson.annotation.PropertyAccessor;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import com.fasterxml.jackson.datatype.jsr310.deser.LocalDateTimeDeserializer;
+import com.fasterxml.jackson.datatype.jsr310.ser.LocalDateTimeSerializer;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.RequiredArgsConstructor;
+import org.springframework.cache.CacheManager;
+import org.springframework.cache.annotation.EnableCaching;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.data.redis.cache.RedisCacheConfiguration;
+import org.springframework.data.redis.cache.RedisCacheManager;
+import org.springframework.data.redis.cache.RedisCacheWriter;
+import org.springframework.data.redis.connection.RedisConnectionFactory;
+import org.springframework.data.redis.serializer.Jackson2JsonRedisSerializer;
+import org.springframework.data.redis.serializer.RedisSerializationContext;
+import org.springframework.data.redis.serializer.StringRedisSerializer;
+
+import java.text.SimpleDateFormat;
+import java.time.Duration;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.*;
+
+@RequiredArgsConstructor
+@Configuration
+@EnableCaching
+public class RedisCacheConfig {
+
+    private static final long DEFAULT_DURATION = 5 * 60L;
+
+    private final static String DATE_TIME_FORMATTER = "yyyy-MM-dd HH:mm:ss.SSSXXX";
+
+    public static final List<CacheSpec> CACHE_SPEC_LIST = new ArrayList<>();
+
+
+    @Bean
+    public CacheManager cacheManager(RedisConnectionFactory factory) {
+        //对象的序列化
+        ObjectMapper objectMapper = getNewInstance();
+        objectMapper.enableDefaultTyping(ObjectMapper.DefaultTyping.NON_FINAL, JsonTypeInfo.As.PROPERTY);
+        Jackson2JsonRedisSerializer<Object> jackson2JsonRedisSerializer = new Jackson2JsonRedisSerializer<>(Object.class);
+        jackson2JsonRedisSerializer.setObjectMapper(objectMapper);
+
+        // 全局redis缓存过期时间，默认60秒
+        RedisCacheConfiguration defaultConfig = RedisCacheConfiguration.defaultCacheConfig()
+                .computePrefixWith(name -> name + ":")
+                .serializeKeysWith(RedisSerializationContext.SerializationPair.fromSerializer(StringRedisSerializer.UTF_8))
+                .serializeValuesWith(RedisSerializationContext.SerializationPair.fromSerializer(jackson2JsonRedisSerializer))
+                .entryTtl(Duration.ofSeconds(DEFAULT_DURATION))
+                .disableCachingNullValues();
+
+        Map<String, RedisCacheConfiguration> redisCacheConfigMap = new HashMap<>();
+        // 缓存N秒
+        CACHE_SPEC_LIST.add(new CacheSpec("getBytedanceAdvertiserByAd", 12 * 3600L));
+        CACHE_SPEC_LIST.add(new CacheSpec("getDZAppIdByChannelId", 12 * 3600L));
+        CACHE_SPEC_LIST.forEach(cache -> {
+            long duration = DEFAULT_DURATION;
+            if (cache.getExpireTtl() != null && cache.getExpireTtl() > 0) {
+                duration = cache.getExpireTtl();
+            }
+            RedisCacheConfiguration config = RedisCacheConfiguration.defaultCacheConfig()
+                    // 默认name + 两个:,改为一个:
+                    .computePrefixWith(name -> name + ":")
+                    .serializeKeysWith(RedisSerializationContext.SerializationPair.fromSerializer(StringRedisSerializer.UTF_8))
+                    .serializeValuesWith(RedisSerializationContext.SerializationPair.fromSerializer(jackson2JsonRedisSerializer))
+                    .entryTtl(Duration.ofSeconds(duration));
+            redisCacheConfigMap.put(cache.getCacheName(), config);
+        });
+
+        return RedisCacheManager.builder(RedisCacheWriter.nonLockingRedisCacheWriter(factory))
+                .cacheDefaults(defaultConfig)
+                .withInitialCacheConfigurations(redisCacheConfigMap)
+                .build();
+    }
+
+    @Data
+    @AllArgsConstructor
+    public class CacheSpec {
+        private String cacheName;
+        private Long expireTtl;
+
+        /**
+         * default ttl 60s
+         *
+         * @param cacheName
+         */
+        public CacheSpec(String cacheName) {
+            this(cacheName, DEFAULT_DURATION);
+        }
+    }
+
+    private ObjectMapper getNewInstance() {
+        ObjectMapper objectMapper = new ObjectMapper();
+        objectMapper.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.ANY);
+        objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+        objectMapper.configure(SerializationFeature.WRITE_ENUMS_USING_TO_STRING, true);
+        objectMapper.configure(DeserializationFeature.READ_ENUMS_USING_TO_STRING, true);
+        objectMapper.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true);
+        objectMapper.disable(SerializationFeature.FAIL_ON_EMPTY_BEANS); // 忽略映射属性
+        //objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
+        objectMapper.setDateFormat(new SimpleDateFormat(DATE_TIME_FORMATTER));
+        // 序列化 LocalDateTime
+        JavaTimeModule timeModule = new JavaTimeModule();
+        DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy_MM_dd_HH_mm_ss", Locale.CHINA);
+        timeModule.addSerializer(LocalDateTime.class, new LocalDateTimeSerializer(dateTimeFormatter));
+        timeModule.addDeserializer(LocalDateTime.class, new LocalDateTimeDeserializer(dateTimeFormatter));
+        objectMapper.registerModule(timeModule);
+
+        return objectMapper;
+    }
+}

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/controller/DianZhongController.java

@@ -26,6 +26,15 @@ public class DianZhongController {
     private DianZhongConfig dianZhongConfig;
 
 
+    @GetMapping("/testGetConfig")
+    public Object testGetConfig(@RequestParam(required = false) String clientId) {
+        if (clientId == null) {
+            return dianZhongConfig.getDzConfig();
+        } else {
+            return dianZhongConfig.getToken(clientId);
+        }
+    }
+
     @GetMapping("/getDownloadUrl")
     public void downloadUrl(String type,
                             @RequestParam(required = false) String statDate,
@@ -44,7 +53,7 @@ public class DianZhongController {
     }
 
     @GetMapping("/getUrlData")
-    public void urlData(String statDate) {
+    public void urlData(@RequestParam(required = false) String statDate) {
         dianZhongService.getUrlData(statDate);
     }
 

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/controller/FeishuController.java

@@ -38,11 +38,13 @@ public class FeishuController {
     OssUtil ossUtil;
     @Autowired
     CsjResoureJob csjResoureJob;
+    @Autowired
+    IDianzhongConfigService dianzhongConfigService;
 
     @Autowired
     IUserLoginRecordService userLoginRecordService;
-    @GetMapping("/test")
-    public void test() throws Exception {
+    @PostMapping("/test")
+    public void test(@RequestBody String str) throws Exception {
 //        bytedanceProductUpdateService.initNew();
 //        bytedanceProductUpdateService.run();
 //        reportsAdPushService.test();
@@ -53,7 +55,12 @@ public class FeishuController {
 //        cutGroupRankPushService.send();
 //        reportsByBusinessPushService.test();
 
-        csjResoureJob.csjResourceOssJob();
+//        csjResoureJob.csjResourceOssJob();
+//    dianzhongConfigService.pullMovieInfo("2025-08-08", 78012L);
+//        dianzhongConfigService.pullChannelInfo();
+//        dianzhongConfigService.pullMovieInfo(str);
+//        dianzhongConfigService.pullMovie();
+    dianzhongConfigService.syncPromoteLinkInfo();
     }
 
 

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/entity/DianzhongDistributorConfig.java

@@ -0,0 +1,20 @@
+package cn.com.ctop.job.bytedance.data.entity;
+
+import lombok.Data;
+
+@Data
+public class DianzhongDistributorConfig {
+    private String distributorId;//渠道账号ID
+    private String distributorName;//渠道账号名称
+    private String nickName;//渠道昵称
+    private String dianzhongAccountId;
+    private String channelCreateTime;//渠道端创建时间
+    private String jumpType;//1：单端 2：双端
+    private String ofAppid;//公众号(小程序)appid
+    private String ofName;//公众号名称
+    private String remark;//备注
+    private String createBy;//创建者
+    private String createTime;//创建时间
+    private String updateBy;//更新者
+    private String updateTime;//修改时间
+}

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/entity/DianzhongMovieConfig.java

@@ -0,0 +1,24 @@
+package cn.com.ctop.job.bytedance.data.entity;
+
+import lombok.Data;
+
+@Data
+public class DianzhongMovieConfig {
+
+    private String movieId; //剧目ID
+    private String movieName; //剧目名称
+    private String coverImgUrl; //封面图链接
+    private Integer chapterCount;
+    private Integer freeChapterCount;
+    private String type1; //男频/女频
+    private String type2; //二级分类
+    private String type3; //三级分类
+    private String firstShelfTime; //上架时间
+    private String remark; //备注
+    private String createBy; //创建者
+    private String createTime; //创建时间
+    private String updateBy; //更新者
+    private String updateTime; //修改时间
+    private String infoJson;//原始json
+
+}

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/mapper/DianZhongMapper.java

@@ -1,9 +1,12 @@
 package cn.com.ctop.job.bytedance.data.mapper;
 
+import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
+import com.baomidou.dynamic.datasource.annotation.DS;
 import org.apache.ibatis.annotations.Param;
 
 import java.util.List;
+import java.util.Map;
 
 public interface DianZhongMapper {
     void insertDownloadTask(@Param("item") JSONObject jsonObject);
@@ -13,4 +16,13 @@ public interface DianZhongMapper {
     List<JSONObject> getTaskUrl(@Param("statDate") String statDate);
 
     void updateTaskDownloaded(@Param("taskId") String taskId);
+
+    @DS("selectdb")
+    Map<String, String> getBytedanceAdvertiserByAd(@Param("adId") String adId, @Param("statTimeTay") Integer statTimeTay);
+
+    String getDZAppIdByChannelId(@Param("channelId") String channelId);
+
+    List<JSONObject> getAccountConfig();
+
+    String getAccountToken(@Param("clientId") String clientId);
 }

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/mapper/DianzhongConfigMapper.java

@@ -0,0 +1,21 @@
+package cn.com.ctop.job.bytedance.data.mapper;
+
+import cn.com.ctop.job.bytedance.data.entity.DianzhongDistributorConfig;
+import cn.com.ctop.job.bytedance.data.entity.DianzhongMovieConfig;
+
+import java.util.List;
+
+public interface DianzhongConfigMapper {
+
+    int insertConfigSelective(DianzhongMovieConfig config);
+
+    int insertDistributorConfigSelective(DianzhongDistributorConfig config);
+
+    List<String> getAllMovieIds();
+
+    int updateMovieImage(DianzhongMovieConfig config);
+
+    List<DianzhongMovieConfig> toUploadImageList();
+
+
+}

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/mapper/xml/DianZhongMapper.xml

@@ -23,15 +23,43 @@
     </update>
 
     <select id="getTaskUrl" resultType="com.alibaba.fastjson.JSONObject">
-        select task_id,url,type,client_id
+        select task_id,url,type,client_id,stat_date
         from media_api.dianzhong_data_download_url
-        where stat_date= ${statDate} and downloaded = 0 and url is not null and url != '-1'
+        where downloaded = 0 and url is not null and url != '-1'
+        <if test="statDate != null">
+            and stat_date = #{statDate}
+        </if>
+        order by stat_date, type desc , create_time
     </select>
 
     <select id="getDZRetry" resultType="com.alibaba.fastjson.JSONObject">
-        SELECT t1.is_send AS 'isSend', t1.id AS 'id', t1.advertiser_id AS 'advertiserId', t1.start_date AS 'startDate', t1.end_date AS 'endDate', t1.data_dimension AS 'dataDimension', t1.data_type AS 'dataType', t3.access_token AS 'accessToken'
+        SELECT t1.is_send AS 'isSend', t1.id AS 'id', t1.advertiser_id AS 'advertiserId', t1.start_date AS 'startDate', t1.end_date AS 'endDate', t1.data_dimension AS 'dataDimension', t1.data_type AS 'dataType'
         FROM bytedance_error_log t1
         where data_dimension like 'dianzhong%'
     </select>
 
+    <select id="getBytedanceAdvertiserByAd" resultType="java.util.Map">
+        SELECT cast(advertiser_id as char) advertiser_id,cast(cdp_project_id as char) a_id from media_api.bytedance_promoter_report_hour
+        where cdp_promotion_id =  ${adId} and stat_time_day >= #{statTimeTay} limit 1
+    </select>
+    <select id="getDZAppIdByChannelId" resultType="java.lang.String">
+        select m.app_id
+        from deepthink.dianzhong_distributor_config d
+                 join deepthink.mini_app_info m
+                      on d.of_appid = m.tt_app_id
+        where d.distributor_id = #{channelId}
+        limit 1
+    </select>
+    <select id="getAccountConfig" resultType="com.alibaba.fastjson.JSONObject">
+        select account_id,business_method,token
+        from deepthink.dianzhong_account_config
+    </select>
+
+    <select id="getAccountToken" resultType="java.lang.String">
+        select token
+        from deepthink.dianzhong_account_config
+        where account_id = #{clientId}
+        limit 1
+    </select>
+
 </mapper>

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/mapper/xml/DianzhongConfigMapper.xml

@@ -0,0 +1,104 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
+<mapper namespace="cn.com.ctop.job.bytedance.data.mapper.DianzhongConfigMapper">
+    <!--create table dianzhong_movie_config(
+    movie_id varchar(64) not null comment '剧目ID'
+    primary key,
+    movie_name varchar(64) not null comment '剧目名称',
+    introduction text null comment '简介',
+    word_count int null comment '字数',
+    cover_img_url varchar(256) null comment '封面图链接',
+    type_1 varchar(64) null comment '男频/女频',
+    type_2 varchar(64) null comment '二级分类',
+    type_3 varchar(64) null comment '三级分类',
+    first_shelf_time datetime null comment '上架时间',
+    show_title varchar(64) null comment '显示标题',
+    remark varchar(64) null comment '备注',
+    create_by          varchar(64) default ''                null comment '创建者',
+    create_time        datetime    default CURRENT_TIMESTAMP null comment '创建时间',
+    update_by          varchar(64) default ''                null comment '更新者',
+    update_time        datetime    default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '修改时间'
+    ) comment '点众-短剧配置表';-->
+
+
+    <insert id="insertConfigSelective">
+        REPLACE INTO deepthink.dianzhong_movie_config
+        <trim prefix="(" suffix=")" suffixOverrides=",">
+            <if test="movieId != null">movie_id,</if>
+            <if test="movieName != null">movie_name,</if>
+            <if test="coverImgUrl != null">cover_img_url,</if>
+            <if test="chapterCount != null">chapter_count,</if>
+            <if test="freeChapterCount != null">free_chapter_count,</if>
+            <if test="type1 != null">type_1,</if>
+            <if test="type2 != null">type_2,</if>
+            <if test="type3 != null">type_3,</if>
+            <if test="firstShelfTime != null">first_shelf_time,</if>
+            <if test="remark != null">remark,</if>
+            <if test="createBy != null">create_by,</if>
+            <if test="updateBy != null">update_by,</if>
+            <if test="infoJson != null">info_json,</if>
+        </trim>
+        VALUES
+        <trim prefix="(" suffix=")" suffixOverrides=",">
+            <if test="movieId != null">#{movieId},</if>
+            <if test="movieName != null">#{movieName},</if>
+            <if test="coverImgUrl != null">#{coverImgUrl},</if>
+            <if test="chapterCount != null">#{chapterCount},</if>
+            <if test="freeChapterCount != null">#{freeChapterCount},</if>
+            <if test="type1 != null">#{type1},</if>
+            <if test="type2 != null">#{type2},</if>
+            <if test="type3 != null">#{type3},</if>
+            <if test="firstShelfTime != null">#{firstShelfTime},</if>
+            <if test="remark != null">#{remark},</if>
+            <if test="createBy != null">#{createBy},</if>
+            <if test="infoJson != null">#{infoJson},</if>
+        </trim>
+    </insert>
+
+    <insert id="insertDistributorConfigSelective">
+        REPLACE INTO deepthink.dianzhong_distributor_config
+        <trim prefix="(" suffix=")" suffixOverrides=",">
+            <if test="distributorId != null">distributor_id,</if>
+            <if test="distributorName != null">distributor_name,</if>
+            <if test="nickName != null">nick_name,</if>
+            <if test="dianzhongAccountId != null">dianzhong_account_id,</if>
+            <if test="channelCreateTime != null">channel_create_time,</if>
+            <if test="jumpType != null">jump_type,</if>
+            <if test="ofAppid != null">of_appid,</if>
+            <if test="ofName != null">of_name,</if>
+            <if test="remark != null">remark,</if>
+            <if test="createBy != null">create_by,</if>
+            <if test="updateBy != null">update_by,</if>
+        </trim>
+        VALUES
+        <trim prefix="(" suffix=")" suffixOverrides=",">
+            <if test="distributorId != null">#{distributorId},</if>
+            <if test="distributorName != null">#{distributorName},</if>
+            <if test="nickName != null">#{nickName},</if>
+            <if test="dianzhongAccountId != null">#{dianzhongAccountId},</if>
+            <if test="channelCreateTime != null">#{channelCreateTime},</if>
+            <if test="jumpType != null">#{jumpType},</if>
+            <if test="ofAppid != null">#{ofAppid},</if>
+            <if test="ofName != null">#{ofName},</if>
+            <if test="remark != null">#{remark},</if>
+            <if test="createBy != null">#{createBy},</if>
+            <if test="updateBy != null">#{updateBy},</if>
+        </trim>
+    </insert>
+    <update id="updateMovieImage">
+        UPDATE deepthink.dianzhong_movie_config
+        SET cover_img_url = #{coverImgUrl}
+        WHERE movie_id = #{movieId}
+    </update>
+    <select id="getAllMovieIds" resultType="java.lang.String">
+        select movie_id
+        from deepthink.dianzhong_movie_config
+        where cover_img_url is not null
+    </select>
+    <select id="toUploadImageList" resultType="cn.com.ctop.job.bytedance.data.entity.DianzhongMovieConfig">
+        select movie_id as movieId, movie_name as movieName, cover_img_url as coverImgUrl
+        from deepthink.dianzhong_movie_config
+        where cover_img_url is not null and cover_img_url like '%resali.ssread.cn%'
+        limit 500
+    </select>
+</mapper>

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/mapper/xml/ReportsAdPushMapper.xml

@@ -60,7 +60,7 @@
         FROM
             (select sum(cost_amount) as cost_amount, sum(real_cost_amount) as real_cost_amount, max(update_time) as update_time
              from application.media_ad_report_daily
-             where stat_date = ${date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type in (1,3,8,9,10,11))
+             where stat_date = ${date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type not in (2,4,6))
             ) tb1,
             (select sum(pay_amount) as pay_amount,
                     sum(iaa_amount) as iaa_amount,
@@ -68,7 +68,7 @@
                     sum(share_pay_amount) as share_pay_amount,
                     sum(share_iaa_amount) as share_iaa_amount
              from application.user_data_report_daily
-             where stat_date = ${date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type in (1,3,8,9,10,11))
+             where stat_date = ${date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type not in (2,4,6))
             ) tb2
     </select>
     <select id="adHourData" resultType="java.util.Map">
@@ -83,13 +83,13 @@
         from
         (select sum(cost_amount) as cost_amount, sum(real_cost_amount) as real_cost_amount
         from application.media_ad_report_hourly
-        where stat_date = #{date} and `hour`= #{hour} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type in (1,3,8,9,10,11))
+        where stat_date = #{date} and `hour`= #{hour} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type not in (2,4,6))
         ) tb1,
         (select sum(pay_amount) as pay_amount, sum(iaa_amount) as iaa_amount,
         sum(share_iaa_amount_3) as share_iaa_amount_3,
         sum(share_iaa_amount) as share_iaa_amount, sum(share_pay_amount) as share_pay_amount
         from application.user_data_report_hourly
-        where stat_date = #{date} and `hour`= #{hour} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type in (1,3,8,9,10,11))
+        where stat_date = #{date} and `hour`= #{hour} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type not in (2,4,6))
         ) tb2
     </select>
     <select id="tmpTopMovie" resultType="java.util.Map">
@@ -102,8 +102,8 @@
         <include refid="iaaRoi"/>,
         <include refid="iapRoi"/>
         from
-        (select product_id from application.media_product_report_daily where stat_date=${date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type in (1,3,8,9,10,11))
-        union select movie_id from application.user_data_report_daily where stat_date=${date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type in (1,3,8,9,10,11))
+        (select product_id from application.media_product_report_daily where stat_date=${date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type not in (2,4,6))
+        union select movie_id from application.user_data_report_daily where stat_date=${date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type not in (2,4,6))
         ) tb1
         left join
         (select product_id, sum(iaa_cost_amount) as iaa_cost_amount, sum(iaa_real_cost_amount) as iaa_real_cost_amount,
@@ -112,12 +112,12 @@
         sum(iaa_real_cost_amount)+sum(iap_real_cost_amount) as real_cost_amount
         from
         (select product_id, round(ifnull(sum(cost_amount),0),2) as iaa_cost_amount, sum(real_cost_amount) as iaa_real_cost_amount, 0 as iap_cost_amount, 0 as iap_real_cost_amount
-        from application.media_product_report_daily where stat_date=${date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type in (1,3,8,9,10,11) and business_method='IAA')
+        from application.media_product_report_daily where stat_date=${date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type not in (2,4,6) and business_method='IAA')
         group by product_id
         union all
         select product_id, 0 as iaa_cost_amount, 0 as iaa_real_cost_amount,
         round(ifnull(sum(cost_amount),0),2) as cost_amount, sum(real_cost_amount) as real_cost_amount
-        from application.media_product_report_daily where stat_date=${date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type in (1,3,8,9,10,11) and business_method='IAP')
+        from application.media_product_report_daily where stat_date=${date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type not in (2,4,6) and business_method='IAP')
         group by product_id) subt2
         group by product_id
         ) tb2 on tb1.product_id=tb2.product_id
@@ -127,7 +127,7 @@
         ifnull(sum(share_pay_amount), 0) as share_pay_amount,
         ifnull(sum(share_iaa_amount), 0) as share_iaa_amount,
         round(ifnull(sum(share_iaa_amount_3), 0),2) as share_iaa_amount_3
-        from application.user_data_report_daily where stat_date=${date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type in (1,3,8,9,10,11))
+        from application.user_data_report_daily where stat_date=${date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type not in (2,4,6))
         group by movie_id
         ) tb3 on tb1.product_id=tb3.movie_id
         left join
@@ -152,13 +152,13 @@
         from
         (select sum(cost_amount) as cost_amount, sum(real_cost_amount) as real_cost_amount, max(update_time) as update_time
         from application.media_ad_report_daily
-        where stat_date = #{date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type in (1,3,8,9,10,11))
+        where stat_date = #{date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type not in (2,4,6))
         ) tb1,
         (select sum(pay_amount) as pay_amount, sum(iaa_amount) as iaa_amount,
         sum(share_iaa_amount) as share_iaa_amount, sum(share_pay_amount) as share_pay_amount,
         sum(share_iaa_amount_3) as share_iaa_amount_3
         from application.user_data_report_daily
-        where stat_date = #{date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type in (1,3,8,9,10,11))
+        where stat_date = #{date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type not in (2,4,6))
         ) tb2
     </select>
 
@@ -174,8 +174,8 @@
         <include refid="iapRoi"/>,
         round(ifnull(sum(hour_cost_amount), 0), 2) as hour_cost_amount
         from
-        (select product_id from application.media_product_report_daily where stat_date=${date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type in (1,3,8,9,10,11))
-        union select movie_id from application.user_data_report_daily where stat_date=${date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type in (1,3,8,9,10,11))) tb1
+        (select product_id from application.media_product_report_daily where stat_date=${date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type not in (2,4,6))
+        union select movie_id from application.user_data_report_daily where stat_date=${date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type not in (2,4,6))) tb1
         left join
         (
         select product_id, sum(iaa_cost_amount) as iaa_cost_amount, sum(iaa_real_cost_amount) as iaa_real_cost_amount,
@@ -184,11 +184,11 @@
         sum(iaa_real_cost_amount)+sum(iap_real_cost_amount) as real_cost_amount
         from
         (select product_id, round(ifnull(sum(cost_amount),0),2) as iaa_cost_amount, sum(real_cost_amount) as iaa_real_cost_amount, 0 as iap_cost_amount, 0 as iap_real_cost_amount
-        from application.media_product_report_daily where stat_date=${date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type in (1,3,8,9,10,11) and business_method='IAA')
+        from application.media_product_report_daily where stat_date=${date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type not in (2,4,6) and business_method='IAA')
         group by product_id
         union all
         select product_id, 0 as iaa_cost_amount, 0 as iaa_real_cost_amount,round(ifnull(sum(cost_amount),0),2) as cost_amount, sum(real_cost_amount) as real_cost_amount
-        from application.media_product_report_daily where stat_date=${date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type in (1,3,8,9,10,11) and business_method='IAP')
+        from application.media_product_report_daily where stat_date=${date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type not in (2,4,6) and business_method='IAP')
         group by product_id) subt2
         group by product_id
         ) tb2 on tb1.product_id=tb2.product_id
@@ -197,7 +197,7 @@
         sum(share_iaa_amount_3) as share_iaa_amount_3,
         ifnull(sum(share_iaa_amount), 0) as share_iaa_amount,
         ifnull(sum(share_pay_amount), 0) as share_pay_amount
-        from application.user_data_report_daily where stat_date=${date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type in (1,3,8,9,10,11))
+        from application.user_data_report_daily where stat_date=${date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type not in (2,4,6))
         group by movie_id
         ) tb3 on tb1.product_id=tb3.movie_id
         left join
@@ -205,7 +205,7 @@
         ) tb4 on tb1.product_id=tb4.moive_id
         left join
         (select product_id, ifnull(sum(cost_amount), 0) as hour_cost_amount from application.media_product_report_hourly
-        where stat_date=${date} and `hour`=${hour}  and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type in (1,3,8,9,10,11))
+        where stat_date=${date} and `hour`=${hour}  and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type not in (2,4,6))
         group by product_id) tb5 on tb1.product_id=tb5.product_id
         where 1=1 and tb1.product_id != '-9999'
         group by tb4.standard_movie_name
@@ -215,7 +215,7 @@
 
 
     <select id="optimizerCostRank" resultType="java.util.Map">
-        select tb1.user_id, tb4.phonenumber, tb4.nick_name, tb4.dept_id, tb5.dept_name,
+        select tb1.user_id, tb4.phonenumber, tb4.nick_name, tb5.dept_id, tb5.dept_name,
         rank() over (order by sum(cost_amount) desc) as rk,
         <include refid="costAmount"/>,
         <include refid="iaaRoi"/>,
@@ -229,11 +229,11 @@
         sum(iaa_real_cost_amount)+sum(iap_real_cost_amount) as real_cost_amount
         FROM
         (select advertiser_id, round(ifnull(sum(cost_amount),0),2) as iaa_cost_amount, sum(real_cost_amount) as iaa_real_cost_amount, 0 as iap_cost_amount, 0 as iap_real_cost_amount
-        from application.media_ad_report_daily where stat_date=${date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type in (1,3,8,9,10,11) and business_method='IAA')
+        from application.media_ad_report_daily where stat_date=${date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type not in (2,4,6) and business_method='IAA')
         group by advertiser_id
         UNION ALL
         select advertiser_id, 0 as iaa_cost_amount, 0 as iaa_real_cost_amount,round(ifnull(sum(cost_amount),0),2) as iap_cost_amount, sum(real_cost_amount) as iap_real_cost_amount
-        from application.media_ad_report_daily where stat_date=${date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type in (1,3,8,9,10,11) and business_method='IAP')
+        from application.media_ad_report_daily where stat_date=${date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type not in (2,4,6) and business_method='IAP')
         group by advertiser_id)subtb2
         group by advertiser_id
         ) tb2
@@ -242,19 +242,25 @@
         round(ifnull(sum(share_iaa_amount_3), 0),2) as share_iaa_amount_3,
         sum(share_iaa_amount) as share_iaa_amount,
         sum(share_pay_amount) as share_pay_amount
-        from application.user_data_report_daily where stat_date=${date}  and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type in (1,3,8,9,10,11))
+        from application.user_data_report_daily where stat_date=${date}  and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type not in (2,4,6))
         group by advertiser_id) tb3
         on tb1.advertiser_id = tb3.advertiser_id
         left join (select user_id, nick_name, phonenumber, dept_id from tidb.deepthink.sys_user) tb4
         on tb1.user_id=tb4.user_id
-        left join (select dept_id, dept_name from tidb.deepthink.sys_dept) tb5
-        on tb4.dept_id = tb5.dept_id
-        group by tb1.user_id, tb4.phonenumber, tb4.nick_name, tb4.dept_id, tb5.dept_name
+        left join (
+            select dp.dept_id, dp.dept_name, ifnull(subdp.dept_id, dp.dept_id) as sub_dept_id, ifnull(subdp.dept_name, dp.dept_name) as sub_dept_name from (select * from tidb.deepthink.sys_dept where parent_id in (select dept_id from tidb.deepthink.sys_dept where parent_id=300)) dp
+            right join tidb.deepthink.sys_dept subdp on dp.dept_id=subdp.parent_id
+            where dp.dept_id is not null
+            union all
+            select dept_id, dept_name, dept_id as sub_dept_id, dept_name as sub_dept_name from tidb.deepthink.sys_dept where parent_id in (select dept_id from tidb.deepthink.sys_dept where parent_id=300)
+        ) tb5
+        on tb4.dept_id = tb5.sub_dept_id
+        group by tb1.user_id, tb4.phonenumber, tb4.nick_name, tb5.dept_id, tb5.dept_name
         having sum(cost_amount)>1
         order by rk asc
     </select>
     <select id="optimizerDateData" resultType="java.util.Map">
-        select tb1.user_id,tb4.phonenumber,tb4.nick_name, tb4.dept_id,
+        select tb1.user_id,tb4.phonenumber,tb4.nick_name, tb5.dept_id,
         <include refid="costAmount"/>,
         <include refid="iaaAmount"/>,
         <include refid="payAmount"/>,
@@ -265,17 +271,25 @@
         left join
         (select advertiser_id, sum(cost_amount) as cost_amount, sum(real_cost_amount) as real_cost_amount, max(update_time) as update_time
         from application.media_ad_report_daily
-        where stat_date = #{date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type in (1,3,8,9,10,11))
+        where stat_date = #{date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type not in (2,4,6))
         group by advertiser_id) tb2 on tb1.advertiser_id = tb2.advertiser_id
         left join
         (select advertiser_id, sum(pay_amount) as pay_amount, sum(iaa_amount) as iaa_amount,
         sum(share_iaa_amount) as share_iaa_amount, sum(share_pay_amount) as share_pay_amount
         from application.user_data_report_daily
-        where stat_date = #{date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type in (1,3,8,9,10,11))
+        where stat_date = #{date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type not in (2,4,6))
         group by advertiser_id) tb3 on tb1.advertiser_id = tb3.advertiser_id
         left join (select user_id, nick_name, phonenumber, dept_id from tidb.deepthink.sys_user) tb4
         on tb1.user_id=tb4.user_id
-        group by tb1.user_id, tb4.phonenumber, tb4.nick_name, tb4.dept_id
+        left join (
+        select dp.dept_id, dp.dept_name, ifnull(subdp.dept_id, dp.dept_id) as sub_dept_id, ifnull(subdp.dept_name, dp.dept_name) as sub_dept_name from (select * from tidb.deepthink.sys_dept where parent_id in (select dept_id from tidb.deepthink.sys_dept where parent_id=300)) dp
+        right join tidb.deepthink.sys_dept subdp on dp.dept_id=subdp.parent_id
+        where dp.dept_id is not null
+        union all
+        select dept_id, dept_name, dept_id as sub_dept_id, dept_name as sub_dept_name from tidb.deepthink.sys_dept where parent_id in (select dept_id from tidb.deepthink.sys_dept where parent_id=300)
+        ) tb5
+        on tb4.dept_id = tb5.sub_dept_id
+        group by tb1.user_id, tb4.phonenumber, tb4.nick_name, tb5.dept_id
         having sum(cost_amount)>0
     </select>
     <select id="optimizerHourData" resultType="java.util.Map">
@@ -289,13 +303,13 @@
         left join
         (select advertiser_id, sum(cost_amount) as cost_amount, sum(real_cost_amount) as real_cost_amount, max(update_time) as update_time
         from application.media_ad_report_hourly
-        where stat_date = #{date} and `hour`= #{hour} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type in (1,3,8,9,10,11))
+        where stat_date = #{date} and `hour`= #{hour} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type not in (2,4,6))
         group by advertiser_id) tb2 on tb1.advertiser_id = tb2.advertiser_id
         left join
         (select advertiser_id, sum(pay_amount) as pay_amount, sum(iaa_amount) as iaa_amount,
         sum(share_iaa_amount) as share_iaa_amount, sum(share_pay_amount) as share_pay_amount
         from application.user_data_report_hourly
-        where stat_date = #{date} and `hour`= #{hour} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type in (1,3,8,9,10,11))
+        where stat_date = #{date} and `hour`= #{hour} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type not in (2,4,6))
         group by advertiser_id) tb3 on tb1.advertiser_id = tb3.advertiser_id
         left join (select user_id, nick_name, phonenumber from tidb.deepthink.sys_user) tb4
         on tb1.user_id=tb4.user_id
@@ -305,25 +319,31 @@
 
 
     <select id="deptCostRank" resultType="java.util.Map">
-        select tb4.dept_id, tb5.dept_name,
+        select tb5.dept_id, tb5.dept_name,
         rank() over (order by sum(cost_amount) desc) as rk,
         <include refid="costAmount"/>,
         ifnull(sum(cost_amount_hour), 0) as cost_amount_hour
         from
         (select cast(advertiser_id as char) advertiser_id, user_id from tidb.deepthink.advertiser_bind_info) tb1
         left join (select advertiser_id, sum(cost_amount) as cost_amount, sum(real_cost_amount) as real_cost_amount
-        from application.media_ad_report_daily where stat_date=#{date}  and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type in (1,3,8,9,10,11))
+        from application.media_ad_report_daily where stat_date=#{date}  and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type not in (2,4,6))
         group by advertiser_id) tb2
         on tb1.advertiser_id = tb2.advertiser_id
         left join (select advertiser_id, sum(cost_amount) as cost_amount_hour
-        from application.media_ad_report_hourly where stat_date=#{date} and `hour`= ${hour}  and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type in (1,3,8,9,10,11))
+        from application.media_ad_report_hourly where stat_date=#{date} and `hour`= ${hour}  and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type not in (2,4,6))
         group by advertiser_id) tb3
         on tb1.advertiser_id = tb3.advertiser_id
         left join (select user_id, nick_name, phonenumber, dept_id from tidb.deepthink.sys_user) tb4
         on tb1.user_id=tb4.user_id
-        left join (select dept_id, dept_name from tidb.deepthink.sys_dept) tb5
-        on tb4.dept_id = tb5.dept_id
-        group by tb4.dept_id, tb5.dept_name
+        left join (
+        select dp.dept_id, dp.dept_name, ifnull(subdp.dept_id, dp.dept_id) as sub_dept_id, ifnull(subdp.dept_name, dp.dept_name) as sub_dept_name from (select * from tidb.deepthink.sys_dept where parent_id in (select dept_id from tidb.deepthink.sys_dept where parent_id=300)) dp
+        right join tidb.deepthink.sys_dept subdp on dp.dept_id=subdp.parent_id
+        where dp.dept_id is not null
+        union all
+        select dept_id, dept_name, dept_id as sub_dept_id, dept_name as sub_dept_name from tidb.deepthink.sys_dept where parent_id in (select dept_id from tidb.deepthink.sys_dept where parent_id=300)
+        ) tb5
+        on tb4.dept_id = tb5.sub_dept_id
+        group by tb5.dept_id, tb5.dept_name
         having sum(cost_amount)>1
         order by rk asc
         limit 10
@@ -393,7 +413,7 @@
         order by tb2.user_id
     </select>
     <select id="deptRank2" resultType="java.util.Map">
-        select tb4.dept_id, tb5.dept_name,
+        select tb5.dept_id, tb5.dept_name,
         rank() over (order by sum(cost_amount) desc) as rk,
         <include refid="costAmount"/>,
         <include refid="iaaRoi"/>,
@@ -407,11 +427,11 @@
         sum(iaa_real_cost_amount)+sum(iap_real_cost_amount) as real_cost_amount
         FROM
         (select advertiser_id, round(ifnull(sum(cost_amount),0),2) as iaa_cost_amount, sum(real_cost_amount) as iaa_real_cost_amount, 0 as iap_cost_amount, 0 as iap_real_cost_amount
-        from application.media_ad_report_daily where stat_date=${date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type in (1,3,8,9,10,11) and business_method='IAA')
+        from application.media_ad_report_daily where stat_date=${date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type not in (2,4,6) and business_method='IAA')
         group by advertiser_id
         UNION ALL
         select advertiser_id, 0 as iaa_cost_amount, 0 as iaa_real_cost_amount,round(ifnull(sum(cost_amount),0),2) as cost_amount, sum(real_cost_amount) as real_cost_amount
-        from application.media_ad_report_daily where stat_date=${date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type in (1,3,8,9,10,11) and business_method='IAP')
+        from application.media_ad_report_daily where stat_date=${date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type not in (2,4,6) and business_method='IAP')
         group by advertiser_id)subtb2
         group by advertiser_id
         ) tb2
@@ -420,14 +440,20 @@
         round(ifnull(sum(share_iaa_amount_3), 0),2) as share_iaa_amount_3,
         ifnull(sum(share_iaa_amount), 0) as share_iaa_amount,
         sum(share_pay_amount) as share_pay_amount
-        from application.user_data_report_daily where stat_date=${date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type in (1,3,8,9,10,11))
+        from application.user_data_report_daily where stat_date=${date} and ma_appid in (select app_id from tidb.deepthink.mini_app_info where business_type not in (2,4,6))
         group by advertiser_id) tb3
         on tb1.advertiser_id = tb3.advertiser_id
         left join (select user_id, nick_name, phonenumber, dept_id from tidb.deepthink.sys_user) tb4
         on tb1.user_id=tb4.user_id
-        left join (select dept_id, dept_name from tidb.deepthink.sys_dept) tb5
-        on tb4.dept_id = tb5.dept_id
-        group by tb4.dept_id, tb5.dept_name
+        left join (
+            select dp.dept_id, dp.dept_name, ifnull(subdp.dept_id, dp.dept_id) as sub_dept_id, ifnull(subdp.dept_name, dp.dept_name) as sub_dept_name from (select * from tidb.deepthink.sys_dept where parent_id in (select dept_id from tidb.deepthink.sys_dept where parent_id=300)) dp
+            right join tidb.deepthink.sys_dept subdp on dp.dept_id=subdp.parent_id
+            where dp.dept_id is not null
+            union all
+            select dept_id, dept_name, dept_id as sub_dept_id, dept_name as sub_dept_name from tidb.deepthink.sys_dept where parent_id in (select dept_id from tidb.deepthink.sys_dept where parent_id=300)
+        ) tb5
+        on tb4.dept_id = tb5.sub_dept_id
+        group by tb5.dept_id, tb5.dept_name
         having sum(cost_amount)>1
         order by rk
     </select>

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/ICacheService.java

@@ -0,0 +1,9 @@
+package cn.com.ctop.job.bytedance.data.service;
+
+
+import java.util.Map;
+
+public interface ICacheService {
+    Map<String, String> getBytedanceAdvertiserByAd(String adId, Integer date);
+    String getDZAppIdByChannelId(String channelId);
+}

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/IDianzhongConfigService.java

@@ -0,0 +1,13 @@
+package cn.com.ctop.job.bytedance.data.service;
+
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+
+public interface IDianzhongConfigService {
+
+    void pullChannelInfo();
+
+    void pullMovie() throws IOException;
+
+    void syncPromoteLinkInfo();
+}

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/impl/BytedanceProductUpdateServiceImpl.java

@@ -568,14 +568,14 @@ public class BytedanceProductUpdateServiceImpl implements IBytedanceProductUpdat
         String res = HttpUtils.httpGet(getPlayletDetailUrl, params, headers);
         if (res == null || res.isEmpty()) {
             errorBlanket.add("获取字节小程序推广链接失败, 广告账号ID：" + advertiserId + ", 小程序资产ID：" + instanceId);
-            return null;
+            return new JSONArray();
         }else {
             JSONObject jsonObject = JSONObject.parseObject(res);
             int code = jsonObject.getIntValue("code");
             if (code != 0) {
                 errorBlanket.add("获取字节小程序推广链接失败，code: " + code + ", msg: " + jsonObject.getString("message") + ", 广告账号ID：" + advertiserId + ", 小程序资产ID：" + instanceId);
                 System.out.println("获取字节小程序推广链接失败，code: " + code + ", msg: " + jsonObject.getString("message"));
-                return null;
+                return new JSONArray();
             }else {
                 JSONArray list = jsonObject.getJSONObject("data").getJSONArray("list");
                 JSONObject page_info = jsonObject.getJSONObject("data").getJSONObject("page_info");

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/impl/CacheServiceImpl.java

@@ -0,0 +1,28 @@
+package cn.com.ctop.job.bytedance.data.service.impl;
+
+
+import cn.com.ctop.job.bytedance.data.mapper.DianZhongMapper;
+import cn.com.ctop.job.bytedance.data.service.ICacheService;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.cache.annotation.Cacheable;
+import org.springframework.stereotype.Service;
+
+import java.util.Map;
+
+@Service
+@Slf4j
+public class CacheServiceImpl implements ICacheService {
+    @Autowired
+    private DianZhongMapper dianZhongMapper;
+
+    @Cacheable(value = "getBytedanceAdvertiserByAd", key = "#adId", unless = "#result == null")
+    public Map<String, String> getBytedanceAdvertiserByAd(String adId, Integer date) {
+        return dianZhongMapper.getBytedanceAdvertiserByAd(adId, date);
+    }
+
+    @Cacheable(value = "getDZAppIdByChannelId", key = "#channelId", unless = "#result == null")
+    public String getDZAppIdByChannelId(String channelId) {
+        return dianZhongMapper.getDZAppIdByChannelId(channelId);
+    }
+}

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/impl/DianZhongServiceImpl.java

@@ -3,12 +3,14 @@ package cn.com.ctop.job.bytedance.data.service.impl;
 import cn.com.ctop.job.bytedance.data.entity.BytedanceErrorLog;
 import cn.com.ctop.job.bytedance.data.mapper.BytedancePromoterReportMapper;
 import cn.com.ctop.job.bytedance.data.mapper.DianZhongMapper;
+import cn.com.ctop.job.bytedance.data.service.ICacheService;
 import cn.com.ctop.job.bytedance.data.service.IDianZhongService;
 import cn.com.ctop.job.bytedance.data.utils.Check;
 import cn.com.ctop.job.bytedance.data.utils.DateUtils;
 import cn.com.ctop.job.bytedance.data.utils.HttpUtils;
 import cn.com.ctop.job.bytedance.utils.doris.sink.DorisMultiTableStreamLoadManager;
 import cn.hutool.core.date.DateUtil;
+import cn.hutool.core.util.NumberUtil;
 import cn.hutool.crypto.SecureUtil;
 import com.alibaba.fastjson.JSONObject;
 import lombok.extern.slf4j.Slf4j;
@@ -24,10 +26,17 @@ import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 
 @Service
 @Slf4j
 public class DianZhongServiceImpl implements IDianZhongService {
+    private static final ExecutorService executorService = Executors.newFixedThreadPool(5);
+
+    @Autowired
+    private ICacheService cacheService;
+
     @Autowired
     private DorisMultiTableStreamLoadManager dorisLoadManager;
 
@@ -46,10 +55,6 @@ public class DianZhongServiceImpl implements IDianZhongService {
         put(17, 3);//快小
     }};
 
-    private static final Map<String, String> clientDomain2AppId = new HashMap<String, String>() {{
-        put("10003129" + "14", "dzdyIAP");//抖小
-    }};
-
     private static final Map<String, String> media2Id = new HashMap<String, String>() {{
         put("adq", "3");//腾讯广点通
         put("dy", "1");//字节巨量
@@ -128,47 +133,61 @@ public class DianZhongServiceImpl implements IDianZhongService {
     下载点众IAP数据
      */
     public void getUrlData(String statDate) {
-        List<JSONObject> tasks = dianZhongMapper.getTaskUrl(statDate.replace("-", ""));
+        List<JSONObject> tasks = dianZhongMapper.getTaskUrl(statDate == null ? null : statDate.replace("-", ""));
         if (!Check.isNull(tasks)) {
             for (JSONObject task : tasks) {
-                String taskId = task.getString("task_id");
-                String url = task.getString("url");
-                String type = task.getString("type");
-                String clientId = task.getString("client_id");
-
-                downloadFileRestTemplate.execute(url, org.springframework.http.HttpMethod.GET, null,
-                        (ResponseExtractor<Void>) response -> {
-                            try (BufferedReader reader = new BufferedReader(
-                                    new InputStreamReader(response.getBody(), StandardCharsets.UTF_8))) {
-                                String line;
-                                while ((line = reader.readLine()) != null) {
-                                    if ("user".equals(type)) {
-                                        JSONObject dzUser = getDZUser(line, clientId);
-                                        dorisLoadManager.add("hr_playlet", "user_event", dzUser);
-                                        JSONObject dzLogin = getDZLogin(line, clientId);
-                                        dorisLoadManager.add("hr_playlet", "server_event", dzLogin);
-                                    } else if ("order".equals(type)) {
-                                        JSONObject dzOrder = getDZOrder(line, clientId);
-                                        dorisLoadManager.add("hr_playlet", "server_event", dzOrder);
-                                    }
-                                }
-                            }
-                            return null;
-                        });
-                dianZhongMapper.updateTaskDownloaded(taskId);
+                executorService.submit(new Runnable() {
+                    @Override
+                    public void run() {
+                        String taskId = task.getString("task_id");
+                        String url = task.getString("url");
+                        String type = task.getString("type");
+                        String clientId = task.getString("client_id");
+                        String dataDate = task.getString("stat_date");
+                        try {
+                            log.info("开始下载点众数据 taskId:{}, clientId:{}, type:{}", taskId, clientId, type);
+                            downloadFileRestTemplate.execute(url, org.springframework.http.HttpMethod.GET, null,
+                                    (ResponseExtractor<Void>) response -> {
+                                        try (BufferedReader reader = new BufferedReader(
+                                                new InputStreamReader(response.getBody(), StandardCharsets.UTF_8))) {
+                                            String line;
+                                            while ((line = reader.readLine()) != null) {
+                                                if ("user".equals(type)) {
+                                                    getDZUser(line);
+                                                    getDZLogin(line);
+                                                    getDZActivation(line);
+                                                    getDZRegister(line, dataDate);
+                                                } else if ("order".equals(type)) {
+                                                    getDZOrder(line);
+                                                }
+                                            }
+                                        }
+                                        return null;
+                                    });
+                            dianZhongMapper.updateTaskDownloaded(taskId);
+                            log.info("下载点众数据成功 taskId:{}, clientId:{}, type:{}", taskId, clientId, type);
+                        } catch (Exception e) {
+                            log.error("下载点众数据失败 taskId:{}, clientId:{}, type:{}", taskId, clientId, type, e);
+                        }
+                    }
+                });
             }
         }
     }
 
-    private static String getSignKey(String clientId, String token, long timeStamp) {
+    private String getSignKey(String clientId, String token, long timeStamp) {
         return SecureUtil.md5(clientId + token + timeStamp).toLowerCase();
     }
 
-    private static JSONObject getDZUser(String line, String clientId) {
+    private void getDZUser(String line) {
         JSONObject jsonObject = JSONObject.parseObject(line);
+        String channelId = jsonObject.getString("channelId");
+        String appIdByChannelId = cacheService.getDZAppIdByChannelId(channelId);
+        if (StringUtils.isBlank(appIdByChannelId)) {
+            return;
+        }
         JSONObject userJson = new JSONObject();
         int domain = jsonObject.getIntValue("domain");
-        String appId = clientId + domain;
         Long ctime = jsonObject.getLong("ctime");
         String statDate = DateUtils.formatDate(ctime * 1000, "yyyyMMdd");
         String registerTime = DateUtils.formatDate(ctime * 1000, "yyyy-MM-dd HH:mm:ss");
@@ -176,9 +195,9 @@ public class DianZhongServiceImpl implements IDianZhongService {
         String promotionId = jsonObject.getString("promotionId");
         adId = StringUtils.isNotBlank(promotionId) ? promotionId : adId;
         Long dyeTime = jsonObject.getLong("dyeTime");
-        String activationTime = DateUtils.formatDate(dyeTime * 1000, "yyyy-MM-dd HH:mm:ss");
+        String activationTime = dyeTime != null ? DateUtils.formatDate(dyeTime * 1000, "yyyy-MM-dd HH:mm:ss") : null;
         String fromMedia = jsonObject.getString("from");
-        userJson.put("app_id", clientDomain2AppId.getOrDefault(appId, appId));
+        userJson.put("app_id", appIdByChannelId);
         userJson.put("uid", jsonObject.getString("userId"));
         userJson.put("stat_date", statDate);
         userJson.put("register_time", registerTime);
@@ -186,7 +205,7 @@ public class DianZhongServiceImpl implements IDianZhongService {
         userJson.put("open_id", jsonObject.getString("openId"));
         userJson.put("platform", domain2Platform.getOrDefault(domain, domain));
         userJson.put("official_account_open_id", jsonObject.getString("thirdWxId"));
-        userJson.put("channel_id", jsonObject.getString("channelId"));
+        userJson.put("channel_id", channelId);
         userJson.put("link_id", jsonObject.getString("referralId"));
         userJson.put("expiration_time", jsonObject.getString("vipEndDate"));
         userJson.put("pay_amount", jsonObject.getString("payAmount"));
@@ -199,36 +218,54 @@ public class DianZhongServiceImpl implements IDianZhongService {
         userJson.put("ad_id", adId);
         userJson.put("activation_time", activationTime);
         userJson.put("media_id", media2Id.getOrDefault(fromMedia, fromMedia));
-        return userJson;
+
+        if (NumberUtil.isLong(adId) && !"0".equals(adId)) {
+            String anotherDay = DateUtils.getAnotherDay("yyyy-MM-dd", registerTime.substring(0, 10), -15);
+            Map<String, String> ad2Advertiser = cacheService.getBytedanceAdvertiserByAd(adId, DateUtils.getDateInteger(anotherDay));
+            if (!Check.isNullMap(ad2Advertiser)) {
+                String advertiserId = ad2Advertiser.get("advertiser_id");
+                userJson.put("advertiser_id", advertiserId);
+            }
+        }
+        dorisLoadManager.add("hr_playlet", "user_event", userJson);
     }
 
-    private static JSONObject getDZLogin(String line, String clientId) {
+    private void getDZLogin(String line) {
         JSONObject jsonObject = JSONObject.parseObject(line);
+        String channelId = jsonObject.getString("channelId");
+        String appIdByChannelId = cacheService.getDZAppIdByChannelId(channelId);
+        if (StringUtils.isBlank(appIdByChannelId)) {
+            return;
+        }
         JSONObject loginJson = new JSONObject();
         int domain = jsonObject.getIntValue("domain");
-        String appId = clientId + domain;
         Long loginTime = jsonObject.getLong("loginTime");
         String statDate = DateUtils.formatDate(loginTime * 1000, "yyyyMMdd");
         String adId = jsonObject.getString("adid");
         String promotionId = jsonObject.getString("promotionId");
         adId = StringUtils.isNotBlank(promotionId) ? promotionId : adId;
         Long dyeTime = jsonObject.getLong("dyeTime");
-        String activationTime = DateUtils.formatDate(dyeTime * 1000, "yyyy-MM-dd HH:mm:ss");
+        String activationTime = dyeTime != null ? DateUtils.formatDate(dyeTime * 1000, "yyyy-MM-dd HH:mm:ss") : null;
+        String activationDate = dyeTime != null ? DateUtils.formatDate(dyeTime * 1000, "yyyyMMdd") : null;
+        Long ctime = jsonObject.getLong("ctime");
+        String registerTime = DateUtils.formatDate(ctime * 1000, "yyyy-MM-dd HH:mm:ss");
+        String registerDate = DateUtils.formatDate(ctime * 1000, "yyyyMMdd");
         String fromMedia = jsonObject.getString("from");
         String userId = jsonObject.getString("userId");
+        loginJson.put("app_id", appIdByChannelId);
         loginJson.put("stat_date", statDate);
         loginJson.put("event", "LOGIN");
         loginJson.put("event_pk", SecureUtil.md5(String.format("dianzhong_login_%s_%s", userId, loginTime)));
         loginJson.put("event_tag", "dianzhong_login");
-        loginJson.put("app_id", clientDomain2AppId.getOrDefault(appId, appId));
-
+        loginJson.put("activation_date", activationDate);
         loginJson.put("uid", userId);
         loginJson.put("open_id", jsonObject.getString("openId"));
         loginJson.put("platform", domain2Platform.getOrDefault(domain, domain));
         loginJson.put("official_account_open_id", jsonObject.getString("thirdWxId"));
-        loginJson.put("channel_id", jsonObject.getString("channelId"));
+        loginJson.put("channel_id", channelId);
         loginJson.put("link_id", jsonObject.getString("referralId"));
-        loginJson.put("register_date", jsonObject.getString("registerDate"));
+        loginJson.put("register_date", registerDate);
+        loginJson.put("register_time", registerTime);
         loginJson.put("expiration_time", jsonObject.getString("vipEndDate"));
         loginJson.put("last_login_time", jsonObject.getString("loginTime"));
         loginJson.put("created_at", loginTime);
@@ -239,28 +276,166 @@ public class DianZhongServiceImpl implements IDianZhongService {
         loginJson.put("ad_id", adId);
         loginJson.put("activation_time", activationTime);
         loginJson.put("media_id", media2Id.getOrDefault(fromMedia, fromMedia));
-        return loginJson;
+
+        if (NumberUtil.isLong(adId) && !"0".equals(adId)) {
+            String anotherDay = DateUtils.getAnotherDay("yyyy-MM-dd", DateUtils.formatDate(loginTime * 1000, "yyyy-MM-dd"), -15);
+            Map<String, String> ad2Advertiser = cacheService.getBytedanceAdvertiserByAd(adId, DateUtils.getDateInteger(anotherDay));
+            if (!Check.isNullMap(ad2Advertiser)) {
+                String advertiserId = ad2Advertiser.get("advertiser_id");
+                loginJson.put("advertiser_id", advertiserId);
+            }
+        }
+        dorisLoadManager.add("hr_playlet", "server_event", loginJson);
     }
 
-    private static JSONObject getDZOrder(String line, String clientId) {
+    private void getDZActivation(String line) {
+        JSONObject jsonObject = JSONObject.parseObject(line);
+        String channelId = jsonObject.getString("channelId");
+        String appIdByChannelId = cacheService.getDZAppIdByChannelId(channelId);
+        if (StringUtils.isBlank(appIdByChannelId)) {
+            return;
+        }
+        JSONObject loginJson = new JSONObject();
+        int domain = jsonObject.getIntValue("domain");
+        Long loginTime = jsonObject.getLong("loginTime");
+        String adId = jsonObject.getString("adid");
+        String promotionId = jsonObject.getString("promotionId");
+        adId = StringUtils.isNotBlank(promotionId) ? promotionId : adId;
+        Long dyeTime = jsonObject.getLong("dyeTime");
+        if (dyeTime == null) {
+            return;
+        }
+        String statDate = DateUtils.formatDate(dyeTime * 1000, "yyyyMMdd");
+        String activationTime = DateUtils.formatDate(dyeTime * 1000, "yyyy-MM-dd HH:mm:ss");
+        Long ctime = jsonObject.getLong("ctime");
+        String registerTime = DateUtils.formatDate(ctime * 1000, "yyyy-MM-dd HH:mm:ss");
+        String registerDate = DateUtils.formatDate(ctime * 1000, "yyyyMMdd");
+        String fromMedia = jsonObject.getString("from");
+        String userId = jsonObject.getString("userId");
+        loginJson.put("app_id", appIdByChannelId);
+        loginJson.put("stat_date", statDate);
+        loginJson.put("event", "ACTIVATION");
+        loginJson.put("event_pk", SecureUtil.md5(String.format("dianzhong_activation_%s_%s", userId, dyeTime)));
+        loginJson.put("event_tag", "dianzhong_activation");
+        loginJson.put("activation_date", statDate);
+        loginJson.put("uid", userId);
+        loginJson.put("open_id", jsonObject.getString("openId"));
+        loginJson.put("platform", domain2Platform.getOrDefault(domain, domain));
+        loginJson.put("official_account_open_id", jsonObject.getString("thirdWxId"));
+        loginJson.put("channel_id", channelId);
+        loginJson.put("link_id", jsonObject.getString("referralId"));
+        loginJson.put("register_date", registerDate);
+        loginJson.put("register_time", registerTime);
+        loginJson.put("expiration_time", jsonObject.getString("vipEndDate"));
+        loginJson.put("last_login_time", jsonObject.getString("loginTime"));
+        loginJson.put("created_at", dyeTime);
+        loginJson.put("ip", jsonObject.getString("ip"));
+        loginJson.put("mobile_system", jsonObject.getIntValue("os") == 0 ? "Android" : "ios");
+        loginJson.put("person_id", jsonObject.getString("kdrId"));
+        loginJson.put("a_id", jsonObject.getString("projectId"));
+        loginJson.put("ad_id", adId);
+        loginJson.put("activation_time", activationTime);
+        loginJson.put("media_id", media2Id.getOrDefault(fromMedia, fromMedia));
+
+        if (NumberUtil.isLong(adId) && !"0".equals(adId)) {
+            String anotherDay = DateUtils.getAnotherDay("yyyy-MM-dd", DateUtils.formatDate(loginTime * 1000, "yyyy-MM-dd"), -15);
+            Map<String, String> ad2Advertiser = cacheService.getBytedanceAdvertiserByAd(adId, DateUtils.getDateInteger(anotherDay));
+            if (!Check.isNullMap(ad2Advertiser)) {
+                String advertiserId = ad2Advertiser.get("advertiser_id");
+                loginJson.put("advertiser_id", advertiserId);
+            }
+        }
+        dorisLoadManager.add("hr_playlet", "server_event", loginJson);
+    }
+
+    private void getDZRegister(String line, String date) {
+        JSONObject jsonObject = JSONObject.parseObject(line);
+        String channelId = jsonObject.getString("channelId");
+        String appIdByChannelId = cacheService.getDZAppIdByChannelId(channelId);
+        if (StringUtils.isBlank(appIdByChannelId)) {
+            return;
+        }
+        JSONObject loginJson = new JSONObject();
+        int domain = jsonObject.getIntValue("domain");
+        Long ctime = jsonObject.getLong("ctime");
+        String statDate = DateUtils.formatDate(ctime * 1000, "yyyyMMdd");
+        Long loginTime = jsonObject.getLong("loginTime");
+        String adId = jsonObject.getString("adid");
+        String promotionId = jsonObject.getString("promotionId");
+        adId = StringUtils.isNotBlank(promotionId) ? promotionId : adId;
+        Long dyeTime = jsonObject.getLong("dyeTime");
+        String activationTime = dyeTime != null ? DateUtils.formatDate(dyeTime * 1000, "yyyy-MM-dd HH:mm:ss") : null;
+        String activationDate = dyeTime != null ? DateUtils.formatDate(dyeTime * 1000, "yyyyMMdd") : null;
+        String registerTime = DateUtils.formatDate(ctime * 1000, "yyyy-MM-dd HH:mm:ss");
+        String registerDate = DateUtils.formatDate(ctime * 1000, "yyyyMMdd");
+        String fromMedia = jsonObject.getString("from");
+        String userId = jsonObject.getString("userId");
+        loginJson.put("app_id", appIdByChannelId);
+        loginJson.put("stat_date", statDate);
+        loginJson.put("event", "REGISTER");
+        loginJson.put("event_pk", SecureUtil.md5(String.format("dianzhong_register_%s_%s", userId, ctime)));
+        loginJson.put("event_tag", "dianzhong_register");
+        loginJson.put("activation_date", activationDate);
+        loginJson.put("uid", userId);
+        loginJson.put("open_id", jsonObject.getString("openId"));
+        loginJson.put("platform", domain2Platform.getOrDefault(domain, domain));
+        loginJson.put("official_account_open_id", jsonObject.getString("thirdWxId"));
+        loginJson.put("channel_id", channelId);
+        loginJson.put("link_id", jsonObject.getString("referralId"));
+        loginJson.put("register_date", registerDate);
+        loginJson.put("register_time", registerTime);
+        loginJson.put("expiration_time", jsonObject.getString("vipEndDate"));
+        loginJson.put("last_login_time", jsonObject.getString("loginTime"));
+        loginJson.put("created_at", ctime);
+        loginJson.put("ip", jsonObject.getString("ip"));
+        loginJson.put("mobile_system", jsonObject.getIntValue("os") == 0 ? "Android" : "ios");
+        loginJson.put("person_id", jsonObject.getString("kdrId"));
+        loginJson.put("a_id", jsonObject.getString("projectId"));
+        loginJson.put("ad_id", adId);
+        loginJson.put("activation_time", activationTime);
+        loginJson.put("media_id", media2Id.getOrDefault(fromMedia, fromMedia));
+
+        if (NumberUtil.isLong(adId) && !"0".equals(adId)) {
+            String anotherDay = DateUtils.getAnotherDay("yyyy-MM-dd", DateUtils.formatDate(loginTime * 1000, "yyyy-MM-dd"), -15);
+            Map<String, String> ad2Advertiser = cacheService.getBytedanceAdvertiserByAd(adId, DateUtils.getDateInteger(anotherDay));
+            if (!Check.isNullMap(ad2Advertiser)) {
+                String advertiserId = ad2Advertiser.get("advertiser_id");
+                loginJson.put("advertiser_id", advertiserId);
+            }
+        }
+        if (date.replace("-", "").equals(statDate)) {
+            dorisLoadManager.add("hr_playlet", "server_event", loginJson);
+        }
+    }
+
+    private void getDZOrder(String line) {
         JSONObject jsonObject = JSONObject.parseObject(line);
         JSONObject orderJson = new JSONObject();
+        String channelId = jsonObject.getString("channelId");
+        String appIdByChannelId = cacheService.getDZAppIdByChannelId(channelId);
+        if (StringUtils.isBlank(appIdByChannelId)) {
+            return;
+        }
+
         String orderId = jsonObject.getString("id");
         int domain = jsonObject.getIntValue("domain");
-        String appId = clientId + domain;
         Long ctime = jsonObject.getLong("ctime");
         String statDate = DateUtils.formatDate(ctime * 1000, "yyyyMMdd");
         String adId = jsonObject.getString("adid");
         String promotionId = jsonObject.getString("promotionId");
         adId = StringUtils.isNotBlank(promotionId) ? promotionId : adId;
         Long dyeTime = jsonObject.getLong("dyeTime");
-        String activationTime = DateUtils.formatDate(dyeTime * 1000, "yyyy-MM-dd HH:mm:ss");
+        String activationTime = dyeTime != null ? DateUtils.formatDate(dyeTime * 1000, "yyyy-MM-dd HH:mm:ss") : null;
+        String activationDate = dyeTime != null ? DateUtils.formatDate(dyeTime * 1000, "yyyyMMdd") : null;
+        Long registerT = jsonObject.getLong("registerDate");
+        String registerTime = DateUtils.formatDate(registerT * 1000, "yyyy-MM-dd HH:mm:ss");
+        String registerDate = DateUtils.formatDate(registerT * 1000, "yyyyMMdd");
         String fromMedia = jsonObject.getString("from");
+        orderJson.put("app_id", appIdByChannelId);
         orderJson.put("stat_date", statDate);
         orderJson.put("event", "PAY");
         orderJson.put("event_pk", SecureUtil.md5("dianzhong_order_id:" + orderId));
         orderJson.put("event_tag", "dianzhong_order");
-        orderJson.put("app_id", clientDomain2AppId.getOrDefault(appId, appId));
         orderJson.put("order_no", orderId);
         orderJson.put("platform_order_no", jsonObject.getString("outTradeNo"));
         orderJson.put("total_fee", jsonObject.getString("discount"));
@@ -269,12 +444,13 @@ public class DianZhongServiceImpl implements IDianZhongService {
         orderJson.put("created_at", ctime);
         orderJson.put("pay_time", jsonObject.getLong("finishTime"));
         orderJson.put("uid", jsonObject.getString("userId"));
-        orderJson.put("channel_id", jsonObject.getString("channelId"));
+        orderJson.put("channel_id", channelId);
         orderJson.put("platform", domain2Platform.getOrDefault(domain, domain));
         orderJson.put("video_id", jsonObject.getString("sourceInfo"));
         orderJson.put("video_detail_id", jsonObject.getString("chapterId"));
         orderJson.put("video_name", jsonObject.getString("sourceDesc"));
-        orderJson.put("register_date", jsonObject.getString("registerDate"));
+        orderJson.put("register_date", registerDate);
+        orderJson.put("register_time", registerTime);
         orderJson.put("open_id", jsonObject.getString("openId"));
         orderJson.put("mobile_system", jsonObject.getIntValue("os") == 0 ? "Android" : "ios");
         orderJson.put("link_id", jsonObject.getString("referralId"));
@@ -284,10 +460,20 @@ public class DianZhongServiceImpl implements IDianZhongService {
         orderJson.put("scene", jsonObject.getString("scene"));
         orderJson.put("official_account_open_id", jsonObject.getString("thirdWxId"));
         orderJson.put("activation_time", activationTime);
+        orderJson.put("activation_date", activationDate);
         orderJson.put("pay_type", jsonObject.getString("orderSubType"));
         orderJson.put("media_id", media2Id.getOrDefault(fromMedia, fromMedia));
         orderJson.put("money_benefit", jsonObject.getDouble("moneyBenefit"));
         orderJson.put("params", jsonObject.toJSONString());
-        return orderJson;
+
+        if (NumberUtil.isLong(adId) && !"0".equals(adId)) {
+            String anotherDay = DateUtils.getAnotherDay("yyyy-MM-dd", DateUtils.formatDate(ctime * 1000, "yyyy-MM-dd"), -15);
+            Map<String, String> ad2Advertiser = cacheService.getBytedanceAdvertiserByAd(adId, DateUtils.getDateInteger(anotherDay));
+            if (!Check.isNullMap(ad2Advertiser)) {
+                String advertiserId = ad2Advertiser.get("advertiser_id");
+                orderJson.put("advertiser_id", advertiserId);
+            }
+        }
+        dorisLoadManager.add("hr_playlet", "server_event", orderJson);
     }
 }

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/impl/DianzhongConfigServiceImpl.java

@@ -0,0 +1,144 @@
+package cn.com.ctop.job.bytedance.data.service.impl;
+
+import cn.com.ctop.job.bytedance.config.DianZhongConfig;
+import cn.com.ctop.job.bytedance.data.entity.DianzhongDistributorConfig;
+import cn.com.ctop.job.bytedance.data.entity.DianzhongMovieConfig;
+import cn.com.ctop.job.bytedance.data.mapper.DianzhongConfigMapper;
+import cn.com.ctop.job.bytedance.data.service.IDianzhongConfigService;
+import cn.com.ctop.job.bytedance.data.utils.HttpUtils;
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.http.Header;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.message.BasicNameValuePair;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import javax.annotation.Resource;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.util.*;
+
+@Service
+@Slf4j
+public class DianzhongConfigServiceImpl implements IDianzhongConfigService {
+    @Autowired
+    DianZhongConfig dianZhongConfig;
+    @Resource
+    DianzhongConfigMapper dianzhongConfigMapper;
+
+
+
+    @Override
+    public void pullChannelInfo() {
+        for (DianZhongConfig.DZConfig dzAuth : dianZhongConfig.getDzConfig()){
+            String url = "https://routine.wqxsw.com/flames/channel/query/channelInfoList";
+            JSONObject requestJson = initRequestJson2(dzAuth);
+            System.out.println("请求点众渠道用户列表接口，参数：" + requestJson.toJSONString());
+            String res = HttpUtils.httpPostRequest(url, requestJson, null);
+            System.out.println("点众回调渠道用户列表接口返回结果：" + res);
+            if (StringUtils.isEmpty(res)){
+                log.error("点众渠道用户列表接口返回结果为空");
+                return;
+            }
+            JSONObject resObj = JSONObject.parseObject(res);
+            int code = resObj.getIntValue("retCode");
+            if (code != 0){
+                log.error("点众渠道用户列表接口返回错误，code: {}, msg: {}", code, resObj.getString("retMsg"));
+                return;
+            }
+            JSONObject data = resObj.getJSONObject("data");
+            JSONArray channelInfoList = data.getJSONArray("channelInfoList");
+            if (channelInfoList == null || channelInfoList.isEmpty()) {
+                log.info("点众渠道用户列表接口没有返回数据");
+                return;
+            }
+            for (int i = 0; i < channelInfoList.size(); i++) {
+                JSONObject obj = channelInfoList.getJSONObject(i);
+                DianzhongDistributorConfig config = new DianzhongDistributorConfig();
+                config.setDistributorId(obj.getString("channelId"));
+                config.setDistributorName(obj.getString("userName"));
+                config.setNickName(obj.getString("nickName"));
+                config.setDianzhongAccountId(dzAuth.getClientId());
+                config.setChannelCreateTime( obj.getString("createTime"));
+                config.setJumpType( obj.getString("jumpType"));
+                config.setOfAppid( obj.getString("ofAppid"));
+                config.setCreateBy("system");
+                config.setUpdateBy("system");
+
+                dianzhongConfigMapper.insertDistributorConfigSelective(config);
+            }
+
+        }
+    }
+
+    @Override
+    public void pullMovie() throws IOException {
+        try {
+            String url = "https://api.deepthink.artlinkworld.cn/dianzhong/syncMovie";
+            String s = HttpUtils.httpPostRequest(url, null, null);
+        }catch (Exception ignored){
+
+        }
+    }
+
+    @Override
+    public void syncPromoteLinkInfo() {
+        try {
+            String url = "https://api.deepthink.artlinkworld.cn/dianzhong/syncAll";
+            String s = HttpUtils.httpPostRequest(url, null, null);
+        }catch (Exception ignored){
+
+        }
+    }
+
+
+
+
+
+
+    //获取调用点众接口的必要参数
+    private JSONObject initRequestJson(DianZhongConfig.DZConfig authConfig){
+        JSONObject object = new JSONObject();
+        String clientId = authConfig.getClientId();
+        String token = authConfig.getToken();
+        object.put("clientId", clientId);
+        long timestamp = System.currentTimeMillis() / 1000;
+        object.put("timestamp", String.valueOf(timestamp));
+        System.out.println(clientId + token + timestamp);
+        String signKey = DigestUtils.md5Hex(
+                clientId + token + timestamp
+        ).toLowerCase();
+        object.put("signKey", signKey);
+        return object;
+    }
+
+    private JSONObject initRequestJson2(DianZhongConfig.DZConfig authConfig){
+        JSONObject object = new JSONObject();
+        String clientId = authConfig.getClientId();
+        String token = authConfig.getToken();
+        object.put("clientId", clientId);
+        long timestamp = System.currentTimeMillis();
+        object.put("timestamp", String.valueOf(timestamp));
+        System.out.println(clientId + token + timestamp);
+        String signKey = DigestUtils.md5Hex(
+                clientId + token + timestamp
+        ).toLowerCase();
+        object.put("signKey", signKey);
+        return object;
+    }
+
+
+
+
+}

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/impl/OptimizerRankPushServiceImpl.java

@@ -125,8 +125,8 @@ public class OptimizerRankPushServiceImpl implements IOptimizerRankPushService {
             }
             template_variable.put("object_list_5", object_list_5);
             System.out.println("发送目标: nickName: "+nickName+", userId: "+ userId + ", feishuUserId: "+mobileMap.get(phoneNumber)+", rank: "+rank+", costAmount: "+costAmount);
-            sendToFeishu(token, template_variable, mobileMap.get(phoneNumber));
-//            sendToFeishu(token, template_variable, "44fbaed6");
+//            sendToFeishu(token, template_variable, mobileMap.get(phoneNumber));
+            sendToFeishu(token, template_variable, "44fbaed6");
             //clear
         }
     }

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/DianzhongConfigJob.java

@@ -0,0 +1,39 @@
+package cn.com.ctop.job.bytedance.handler;
+
+import cn.com.ctop.job.bytedance.data.service.IDianzhongConfigService;
+import cn.com.ctop.job.bytedance.utils.FeishuRobotUtil;
+import com.xxl.job.core.handler.annotation.XxlJob;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.io.IOException;
+
+@Slf4j
+@Component
+public class DianzhongConfigJob {
+    @Autowired
+    IDianzhongConfigService dianzhongConfigService;
+
+    public static final String hook_url = "https://open.feishu.cn/open-apis/bot/v2/hook/a0fcfb57-c053-47bd-b1e7-b826f44841e4";
+
+
+    @XxlJob("pullDianzhongChannel")
+    public void pullDianzhongChannel(){
+        dianzhongConfigService.pullChannelInfo();
+    }
+
+    @XxlJob("pullDianzhongMovie")
+    public void pullDianzhongMovie() throws IOException {
+        try {
+            dianzhongConfigService.pullMovie();
+        }catch (Exception e){
+            FeishuRobotUtil.sendTextToWebhook("点众短剧同步失败", hook_url);
+        }
+    }
+
+    @XxlJob("pullDianzhongPromoteList")
+    public void pullDianzhongPromoteList(){
+        dianzhongConfigService.syncPromoteLinkInfo();
+    }
+}

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/utils/doris/sink/SimpleHttpDorisSinkClient.java

@@ -68,6 +68,7 @@ public class SimpleHttpDorisSinkClient implements DorisSinkClient {
             put.setHeader("Authorization", "Basic " + auth);
         }
 
+        put.setHeader("max_filter_ratio", "1");
         put.setHeader("format", format); // json
         put.setHeader("strip_outer_array", "true"); // json array
         put.setHeader("fuzzy_parse", "true"); // json array

job-bytedance/src/main/resources/application.yml

@@ -103,7 +103,7 @@ spring:
   # redis 配置
   redis:
     # 地址
-    host: r-2ze77hczbrb3boqn16pd.redis.rds.aliyuncs.com
+    host: r-2ze77hczbrb3boqn16.redis.rds.aliyuncs.com
     # 端口，默认为6379
     port: 6379
     # 数据库索引
@@ -125,7 +125,7 @@ spring:
 
   redis2:
     database: 0
-    host: r-2zelacjrljt205nsb1pd.redis.rds.aliyuncs.com
+    host: r-2zelacjrljt205nsb1.redis.rds.aliyuncs.com
     password: Huanrui@2025!@
     pool:
       max-active: 10
@@ -189,15 +189,3 @@ playlet:
     bucketName: short-theatre-hr
     csjDomain: https://oss.artlinkworld.cn/
     csjFileDir: csj/
-
-dianzhong:
-  client:
-    dzConfig[0]:
-      clientId: 10003129
-      token: ckogrgCE4PEDxT1oL5
-      type: iap
-    dzConfig[1]:
-      clientId: 10004501
-      token: HgmmEQM4ddcLQ8mn5i
-      type: iaa
-