Merge branch 'master' of codeup.aliyun.com:60c06368dbbed34fe0f0c263/job/hr_job-bytedance into pingkang

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/mapper/CPSReportMapper.java

@@ -8,12 +8,10 @@ import java.util.List;
 
 public interface CPSReportMapper {
 
-    boolean cleanCPSPayDailyReports(@Param("nowDate") Integer nowDate);
-
-    boolean cleanCPSIaaDailyReports(@Param("nowDate") Integer nowDate);
+    boolean cleanCPSPayDailyReports(@Param("nowDate") String nowDate);
 
     @DS("selectdb")
-    List<JSONObject> queryCPSIaaDaily(@Param("nowDate") Integer nowDate);
+    List<JSONObject> queryCPSIaaDaily(@Param("nowDate") Integer nowDate, @Param("dateStr") String dateStr);
 
     boolean cleanCPSIaaDailyReportsDS(@Param("list") List<JSONObject> list);
 

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/mapper/xml/CPSReportMapper.xml

@@ -90,10 +90,7 @@
                        LEFT JOIN thirtpart.ly_official_accounts t2 ON t0.official_account_id = t2.id
               WHERE official_account_id IS NOT NULL
               GROUP BY t1.link_id
-
-
-              union all
-
+              UNION ALL
               SELECT t3.date,
                      t0.official_account_id,
                      t0.responsibility_person_id,
@@ -113,7 +110,8 @@
               FROM (SELECT DATE_FORMAT(activation_time, '%Y%m%d') AS 'date', link_id,
                            count(DISTINCT uid) AS 'new_users'
                     FROM thirtpart.ly_user_activation_info
-                    WHERE DATE_FORMAT(activation_time, '%Y%m%d') = #{nowDate}
+                    WHERE created_at >= unix_timestamp(#{nowDate})
+                      AND created_at < unix_timestamp(DATE_ADD(#{nowDate}, INTERVAL 1 day))
                     GROUP BY link_id) t3
                        LEFT JOIN thirtpart.ly_link t0 ON t0.id = t3.link_id
                        LEFT JOIN (SELECT consume_date,
@@ -165,107 +163,6 @@
         GROUP BY t.id
     </insert>
 
-    <insert id="cleanCPSIaaDailyReports">
-        replace
-        into thirtpart.cps_link_data_report
-        (
-                stat_date,
-                official_account_id,
-                responsibility_person_id,
-                ma_appid,
-                link_id,
-                new_users,
-                new_pay_users,
-                new_pay_counts,
-                new_order_counts,
-                new_pay_amount,
-                new_xn_pay_amount,
-                pay_amount,
-                xn_pay_amount,
-                pay_users,pay_counts,
-                order_counts,
-                new_iaa_users,
-                new_iaa_counts,
-                new_iaa_amount,
-                iaa_users,
-                iaa_counts,
-                iaa_amount,
-				active_users
-                )
-        SELECT IFNULL(t.date, t0.date),
-               tt.official_account_id,
-               tt.responsibility_person_id,
-               tt.app_id,
-               tt.id,
-               t1.new_users,
-               t3.new_pay_users,
-               t3.new_pay_counts,
-               t3.new_order_counts,
-               t3.new_pay_amount,
-               t3.new_xn_pay_amount,
-               t3.pay_amount,
-               t3.xn_pay_amount,
-               t3.pay_users,
-               t3.pay_counts,
-               t3.order_counts,
-               t0.new_iaa_users,
-               t0.new_iaa_counts,
-               t0.new_iaa_amount,
-               t0.iaa_users,
-               t0.iaa_counts,
-               t0.iaa_amount,
-               t.active_users
-        FROM (SELECT t0.id,
-                     t0.responsibility_person_id,
-                     t0.official_account_id,
-                     t2.app_id
-              FROM thirtpart.ly_link t0
-                       LEFT JOIN thirtpart.ly_official_accounts t2 ON t0.official_account_id = t2.id) tt
-                 LEFT JOIN
-             (SELECT t.date, t.link_id, COUNT(DISTINCT t.uid) as 'active_users'
-              FROM thirtpart.ly_user_login_record t
-              where date = #{nowDate}
-                AND link_id
-                  >0
-              GROUP BY link_id) t ON tt.id = t.link_id
-                 LEFT JOIN
-             (SELECT t.stat_date as 'date', t.link_id,
-                     count(distinct case
-                                        when SUBSTR(dying_time, 1, 10) = SUBSTR(event_time, 1, 10)
-                                            then uid end) 'new_iaa_users', sum(if(SUBSTR(dying_time, 1, 10) = SUBSTR(event_time, 1, 10), 1, 0)) 'new_iaa_counts', IFNULL(ROUND(
-                             sum(if(SUBSTR(dying_time, 1, 10) = SUBSTR(event_time, 1, 10), cost / 100000, 0)),
-                             2),0.00) 'new_iaa_amount', count(distinct uid) 'iaa_users', sum(1) 'iaa_counts', IFNULL(ROUND(sum(cost / 100000), 2),0.00) 'iaa_amount'
-              FROM media_api.bytedance_webhook_ma_ecpm t
-                       LEFT JOIN thirtpart.ly_link t0 ON t.link_id = t0.id
-              where t.stat_date = #{nowDate}
-              GROUP BY t.link_id
-              ORDER BY t.link_id) t0 ON tt.id = t0.link_id
-                 LEFT JOIN (SELECT t0.id,
-                                   DATE_FORMAT(t.activation_time, '%Y%m%d') as 'date', t0.official_account_id,
-                                   t0.responsibility_person_id,
-                                   count(DISTINCT t.uid) AS 'new_users'
-                            FROM thirtpart.ly_user_activation_info t
-                                     LEFT JOIN thirtpart.ly_link t0 ON t.link_id = t0.id
-                            WHERE DATE_FORMAT(t.activation_time, '%Y%m%d') = #{nowDate}
-                            GROUP BY t0.id) t1 ON tt.id = t1.id
-                 LEFT JOIN
-             (SELECT link_id,
-                     new_pay_users,
-                     new_pay_counts,
-                     new_order_counts,
-                     new_pay_amount,
-                     new_xn_pay_amount,
-                     pay_amount,
-                     xn_pay_amount,
-                     pay_users,
-                     pay_counts,
-                     order_counts
-              FROM thirtpart.cps_link_data_report
-              WHERE stat_date = #{nowDate}) t3 ON t3.link_id = tt.id
-        WHERE t0.iaa_amount is not null
-        GROUP BY tt.id
-    </insert>
-
     <select id="queryCPSIaaDaily" resultType="com.alibaba.fastjson.JSONObject">
         SELECT IFNULL(t.date, t0.date) AS 'date', tt.official_account_id,
                tt.responsibility_person_id,
@@ -294,7 +191,7 @@
                        LEFT JOIN tidb.thirtpart.ly_official_accounts t2 ON t0.official_account_id = t2.id) tt
                  LEFT JOIN (SELECT t.date, t.link_id, COUNT(DISTINCT t.uid) AS 'active_users'
                             FROM tidb.thirtpart.ly_user_login_record t
-                            WHERE date = #{nowDate} AND link_id > 0
+                            WHERE date = #{dateStr} AND link_id > 0
                             GROUP BY link_id, date) t ON tt.id = t.link_id
                  LEFT JOIN (SELECT t.stat_date AS 'date', t.link_id,
                                    count(DISTINCT CASE
@@ -313,7 +210,8 @@
                                    count(DISTINCT t.uid) AS 'new_users'
                             FROM tidb.thirtpart.ly_user_activation_info t
                                      LEFT JOIN tidb.thirtpart.ly_link t0 ON t.link_id = t0.id
-                            WHERE DATE_FORMAT(t.activation_time, '%Y%m%d') = #{nowDate}
+                            WHERE t.created_at >= unix_timestamp(#{nowDate})
+                              AND t.created_at &lt; unix_timestamp(DATE_ADD(#{nowDate}, INTERVAL 1 day))
                             GROUP BY 1, 2, 3, 4) t1 ON tt.id = t1.id
                  LEFT JOIN (SELECT link_id,
                                    new_pay_users,

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/ICPSReportService.java

@@ -2,5 +2,5 @@ package cn.com.ctop.job.bytedance.data.service;
 
 public interface ICPSReportService {
 
-    void cleanCPSDailyReports(Integer nowDate);
+    void cleanCPSDailyReports(String nowDate);
 }

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/impl/CPSReportServiceImpl.java

@@ -13,10 +13,10 @@ public class CPSReportServiceImpl implements ICPSReportService {
     private CPSReportMapper cpsReportMapper;
 
     @Override
-    public void cleanCPSDailyReports(Integer nowDate) {
+    public void cleanCPSDailyReports(String nowDate) {
         try {
             if (cpsReportMapper.cleanCPSPayDailyReports(nowDate))
-                if (cpsReportMapper.cleanCPSIaaDailyReportsDS(cpsReportMapper.queryCPSIaaDaily(nowDate)))
+                if (cpsReportMapper.cleanCPSIaaDailyReportsDS(cpsReportMapper.queryCPSIaaDaily(Integer.valueOf(nowDate), nowDate)))
                     log.info("CPS数据清洗完成:{}", nowDate);
         } catch (Exception e) {
             e.printStackTrace();

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/CPSReportJob.java

@@ -26,7 +26,7 @@ public class CPSReportJob {
         log.info("[dayCleanCPSDailyReports] 任务启动...");
         try {
             String nowDate = DateUtils.getNowDate("yyyyMMdd");
-            cpsReportService.cleanCPSDailyReports(Integer.valueOf(nowDate));
+            cpsReportService.cleanCPSDailyReports(nowDate);
             log.info("[dayCleanCPSDailyReports] 任务完成");
         } catch (Exception e) {
             e.printStackTrace();
@@ -43,7 +43,7 @@ public class CPSReportJob {
         try {
             String nowDate = DateUtils.getNowDate("yyyyMMdd");
             String yesterday = DateUtils.getNowAddDate("yyyyMMdd", -1);
-            cpsReportService.cleanCPSDailyReports(Integer.valueOf(yesterday));
+            cpsReportService.cleanCPSDailyReports(yesterday);
             log.info("[yesCleanCPSDailyReports] 任务完成");
         } catch (Exception e) {
             e.printStackTrace();

job-bytedance/src/main/resources/application.yml

@@ -85,7 +85,7 @@ spring:
         master:
           url: jdbc:mysql://172.16.0.117:3390/media_api?characterEncoding=UTF-8&useUnicode=true&useSSL=false
           #url: jdbc:mysql://8.147.135.80:3390/media_api?characterEncoding=UTF-8&useUnicode=true&allowMultiQueries=true&useSSL=false
-          username: data
+          username: yanfa
           password: hr@2025..
           driver-class-name: com.mysql.jdbc.Driver
         selectdb: