优化sql

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/mapper/CPSReportMapper.java

@@ -10,10 +10,8 @@ public interface CPSReportMapper {
 
     boolean cleanCPSPayDailyReports(@Param("nowDate") Integer nowDate);
 
-    boolean cleanCPSIaaDailyReports(@Param("nowDate") Integer nowDate);
-
     @DS("selectdb")
-    List<JSONObject> queryCPSIaaDaily(@Param("nowDate") Integer nowDate);
+    List<JSONObject> queryCPSIaaDaily(@Param("nowDate") Integer nowDate, @Param("dateStr") String dateStr);
 
     boolean cleanCPSIaaDailyReportsDS(@Param("list") List<JSONObject> list);
 

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/mapper/xml/CPSReportMapper.xml

@@ -90,10 +90,7 @@
                        LEFT JOIN thirtpart.ly_official_accounts t2 ON t0.official_account_id = t2.id
               WHERE official_account_id IS NOT NULL
               GROUP BY t1.link_id
-
-
-              union all
-
+              UNION ALL
               SELECT t3.date,
                      t0.official_account_id,
                      t0.responsibility_person_id,
@@ -165,107 +162,6 @@
         GROUP BY t.id
     </insert>
 
-    <insert id="cleanCPSIaaDailyReports">
-        replace
-        into thirtpart.cps_link_data_report
-        (
-                stat_date,
-                official_account_id,
-                responsibility_person_id,
-                ma_appid,
-                link_id,
-                new_users,
-                new_pay_users,
-                new_pay_counts,
-                new_order_counts,
-                new_pay_amount,
-                new_xn_pay_amount,
-                pay_amount,
-                xn_pay_amount,
-                pay_users,pay_counts,
-                order_counts,
-                new_iaa_users,
-                new_iaa_counts,
-                new_iaa_amount,
-                iaa_users,
-                iaa_counts,
-                iaa_amount,
-				active_users
-                )
-        SELECT IFNULL(t.date, t0.date),
-               tt.official_account_id,
-               tt.responsibility_person_id,
-               tt.app_id,
-               tt.id,
-               t1.new_users,
-               t3.new_pay_users,
-               t3.new_pay_counts,
-               t3.new_order_counts,
-               t3.new_pay_amount,
-               t3.new_xn_pay_amount,
-               t3.pay_amount,
-               t3.xn_pay_amount,
-               t3.pay_users,
-               t3.pay_counts,
-               t3.order_counts,
-               t0.new_iaa_users,
-               t0.new_iaa_counts,
-               t0.new_iaa_amount,
-               t0.iaa_users,
-               t0.iaa_counts,
-               t0.iaa_amount,
-               t.active_users
-        FROM (SELECT t0.id,
-                     t0.responsibility_person_id,
-                     t0.official_account_id,
-                     t2.app_id
-              FROM thirtpart.ly_link t0
-                       LEFT JOIN thirtpart.ly_official_accounts t2 ON t0.official_account_id = t2.id) tt
-                 LEFT JOIN
-             (SELECT t.date, t.link_id, COUNT(DISTINCT t.uid) as 'active_users'
-              FROM thirtpart.ly_user_login_record t
-              where date = #{nowDate}
-                AND link_id
-                  >0
-              GROUP BY link_id) t ON tt.id = t.link_id
-                 LEFT JOIN
-             (SELECT t.stat_date as 'date', t.link_id,
-                     count(distinct case
-                                        when SUBSTR(dying_time, 1, 10) = SUBSTR(event_time, 1, 10)
-                                            then uid end) 'new_iaa_users', sum(if(SUBSTR(dying_time, 1, 10) = SUBSTR(event_time, 1, 10), 1, 0)) 'new_iaa_counts', IFNULL(ROUND(
-                             sum(if(SUBSTR(dying_time, 1, 10) = SUBSTR(event_time, 1, 10), cost / 100000, 0)),
-                             2),0.00) 'new_iaa_amount', count(distinct uid) 'iaa_users', sum(1) 'iaa_counts', IFNULL(ROUND(sum(cost / 100000), 2),0.00) 'iaa_amount'
-              FROM media_api.bytedance_webhook_ma_ecpm t
-                       LEFT JOIN thirtpart.ly_link t0 ON t.link_id = t0.id
-              where t.stat_date = #{nowDate}
-              GROUP BY t.link_id
-              ORDER BY t.link_id) t0 ON tt.id = t0.link_id
-                 LEFT JOIN (SELECT t0.id,
-                                   DATE_FORMAT(t.activation_time, '%Y%m%d') as 'date', t0.official_account_id,
-                                   t0.responsibility_person_id,
-                                   count(DISTINCT t.uid) AS 'new_users'
-                            FROM thirtpart.ly_user_activation_info t
-                                     LEFT JOIN thirtpart.ly_link t0 ON t.link_id = t0.id
-                            WHERE DATE_FORMAT(t.activation_time, '%Y%m%d') = #{nowDate}
-                            GROUP BY t0.id) t1 ON tt.id = t1.id
-                 LEFT JOIN
-             (SELECT link_id,
-                     new_pay_users,
-                     new_pay_counts,
-                     new_order_counts,
-                     new_pay_amount,
-                     new_xn_pay_amount,
-                     pay_amount,
-                     xn_pay_amount,
-                     pay_users,
-                     pay_counts,
-                     order_counts
-              FROM thirtpart.cps_link_data_report
-              WHERE stat_date = #{nowDate}) t3 ON t3.link_id = tt.id
-        WHERE t0.iaa_amount is not null
-        GROUP BY tt.id
-    </insert>
-
     <select id="queryCPSIaaDaily" resultType="com.alibaba.fastjson.JSONObject">
         SELECT IFNULL(t.date, t0.date) AS 'date', tt.official_account_id,
                tt.responsibility_person_id,
@@ -294,7 +190,7 @@
                        LEFT JOIN tidb.thirtpart.ly_official_accounts t2 ON t0.official_account_id = t2.id) tt
                  LEFT JOIN (SELECT t.date, t.link_id, COUNT(DISTINCT t.uid) AS 'active_users'
                             FROM tidb.thirtpart.ly_user_login_record t
-                            WHERE date = #{nowDate} AND link_id > 0
+                            WHERE date = #{dateStr} AND link_id > 0
                             GROUP BY link_id, date) t ON tt.id = t.link_id
                  LEFT JOIN (SELECT t.stat_date AS 'date', t.link_id,
                                    count(DISTINCT CASE
@@ -313,7 +209,8 @@
                                    count(DISTINCT t.uid) AS 'new_users'
                             FROM tidb.thirtpart.ly_user_activation_info t
                                      LEFT JOIN tidb.thirtpart.ly_link t0 ON t.link_id = t0.id
-                            WHERE DATE_FORMAT(t.activation_time, '%Y%m%d') = #{nowDate}
+                            WHERE t.created_at >= unix_timestamp(#{nowDate})
+                              AND t.created_at &lt; unix_timestamp(DATE_ADD(#{nowDate}, INTERVAL 1 day))
                             GROUP BY 1, 2, 3, 4) t1 ON tt.id = t1.id
                  LEFT JOIN (SELECT link_id,
                                    new_pay_users,

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/impl/CPSReportServiceImpl.java

@@ -16,7 +16,7 @@ public class CPSReportServiceImpl implements ICPSReportService {
     public void cleanCPSDailyReports(Integer nowDate) {
         try {
             if (cpsReportMapper.cleanCPSPayDailyReports(nowDate))
-                if (cpsReportMapper.cleanCPSIaaDailyReportsDS(cpsReportMapper.queryCPSIaaDaily(nowDate)))
+                if (cpsReportMapper.cleanCPSIaaDailyReportsDS(cpsReportMapper.queryCPSIaaDaily(nowDate, String.valueOf(nowDate))))
                     log.info("CPS数据清洗完成:{}", nowDate);
         } catch (Exception e) {
             e.printStackTrace();