Просмотр исходного кода

添加增量数据聚合任务

songyh 3 лет назад
Родитель
Сommit
59147ac0c3

+ 2 - 2
module-job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedanceCheckAccountReportJob.java

@@ -34,9 +34,9 @@ public class BytedanceCheckAccountReportJob {
         if (param==null || param.trim().length()==0) {
         if (param==null || param.trim().length()==0) {
             param = date;
             param = date;
         }
         }
-        checkReportTaskInfoService.initTask(date);
+        checkReportTaskInfoService.initTask(param);
         //查询初始化的任务
         //查询初始化的任务
-        List<BytedanceCheckReportTaskInfo>taskInfos = checkReportTaskInfoService.getListByParams(null,date,0);
+        List<BytedanceCheckReportTaskInfo>taskInfos = checkReportTaskInfoService.getListByParams(null,param,0);
         if(null!=taskInfos&&!taskInfos.isEmpty()){
         if(null!=taskInfos&&!taskInfos.isEmpty()){
             countDownLatch = new CountDownLatch(taskInfos.size());
             countDownLatch = new CountDownLatch(taskInfos.size());
             taskInfos.forEach(taskInfo ->executorService.submit(()->{
             taskInfos.forEach(taskInfo ->executorService.submit(()->{

+ 12 - 0
module-job-kuaishou/src/main/java/cn/com/ctop/job/kuaishou/handler/KuaishouDailyAudienceReportJob.java

@@ -89,4 +89,16 @@ public class KuaishouDailyAudienceReportJob {
         HttpUtils2.httpGet(url, null, null);
         HttpUtils2.httpGet(url, null, null);
     }
     }
 
 
+    @XxlJob("kuaishouDailyTaoTeAudienceReportAggregationJob")
+    public void kuaishouDailyTaoTeAudienceReportAggregationJob() throws Exception {
+        //获取日期
+        String date = DateUtils.addDay(DateUtils.formatDate(),-1);
+        String param = XxlJobHelper.getJobParam();
+        if (param==null || param.trim().length()==0) {
+            param = date;
+        }
+        kuaishouAudienceReportDailyService.aggregationData(param);
+    }
+
+
 }
 }

+ 38 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/enums/AudienceReportTableEnum.java

@@ -0,0 +1,38 @@
+package cn.com.ctop.kuaishou.modules.report.enums;
+
+public enum AudienceReportTableEnum {
+    AGE("ctop_kuaishou_audience_report_daily_age", "age_segment","ctop_kuaishou_audience_report_daily_age_material","age_segment"),
+    CLIENT("ctop_kuaishou_audience_report_daily_client", "client","ctop_kuaishou_audience_report_daily_client_material","client"),
+    CITY("ctop_kuaishou_audience_report_daily_city", "city","ctop_kuaishou_audience_report_daily_city_material","city"),
+    GENDER("ctop_kuaishou_audience_report_daily_gender", "gender","ctop_kuaishou_audience_report_daily_gender_material","gender"),
+    BUSINESS("ctop_kuaishou_audience_report_daily_business", "business_interest_tags","ctop_kuaishou_audience_report_daily_business_material","business_interest_tags"),
+    PROVINCE("ctop_kuaishou_audience_report_daily_province", "province","ctop_kuaishou_audience_report_daily_province_material","province");
+
+    private String originTable;
+    private String originTableKey;
+    private String aggregationTable;
+    private String aggregationTableKey;
+
+    AudienceReportTableEnum(String originTable, String originTableKey, String aggregationTable, String aggregationTableKey) {
+        this.originTable = originTable;
+        this.originTableKey = originTableKey;
+        this.aggregationTable = aggregationTable;
+        this.aggregationTableKey = aggregationTableKey;
+    }
+
+    public String getOriginTable() {
+        return originTable;
+    }
+
+    public String getOriginTableKey() {
+        return originTableKey;
+    }
+
+    public String getAggregationTable() {
+        return aggregationTable;
+    }
+
+    public String getAggregationTableKey() {
+        return aggregationTableKey;
+    }
+}

+ 6 - 1
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/KuaishouAudienceReportDailyMapper.java

@@ -2,6 +2,7 @@ package cn.com.ctop.kuaishou.modules.report.mapper;
 
 
 import cn.com.ctop.kuaishou.modules.report.entity.KuaishouAudienceReportDaily;
 import cn.com.ctop.kuaishou.modules.report.entity.KuaishouAudienceReportDaily;
 import com.baomidou.mybatisplus.core.mapper.BaseMapper;
 import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import org.apache.ibatis.annotations.Param;
 
 
 /**
 /**
  * 人群分析报表
  * 人群分析报表
@@ -11,5 +12,9 @@ import com.baomidou.mybatisplus.core.mapper.BaseMapper;
  * @cersion: V1.0
  * @cersion: V1.0
  */
  */
 public interface KuaishouAudienceReportDailyMapper extends BaseMapper<KuaishouAudienceReportDaily> {
 public interface KuaishouAudienceReportDailyMapper extends BaseMapper<KuaishouAudienceReportDaily> {
-
+    void aggregationData(@Param(value = "date") String date,
+                         @Param(value = "originTable") String originTable,
+                         @Param(value = "originTableKey")String originTableKey,
+                         @Param(value = "aggregationTable")String aggregationTable,
+                         @Param(value = "aggregationTableKey")String aggregationTableKey);
 }
 }

+ 80 - 1
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/xml/KuaishouAudienceReportDailyMapper.xml

@@ -1,5 +1,84 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
 <!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
 <mapper namespace="cn.com.ctop.kuaishou.modules.report.mapper.KuaishouAudienceReportDailyMapper">
 <mapper namespace="cn.com.ctop.kuaishou.modules.report.mapper.KuaishouAudienceReportDailyMapper">
-
+    <insert id="aggregationData">
+        replace into ${aggregationTable} (project_id,
+                                                                  signature,
+                                                                  stat_date,
+                                                                  ${aggregationTableKey},
+                                                                  aclick,
+                                                                  bclick,
+                                                                  photo_click_cost,
+                                                                  download_started,
+                                                                  download_completed,
+                                                                  activation,
+                                                                  submit,
+                                                                  photo_show,
+                                                                  photo_click,
+                                                                  photo_click_ratio,
+                                                                  charge,
+                                                                  action_ratio,
+                                                                  impression_1k_cost,
+                                                                  action_cost,
+                                                                  event_pay_first_day,
+                                                                  event_pay_purchase_amount_first_day,
+                                                                  event_pay,
+                                                                  event_pay_purchase_amount,
+                                                                  event_register,
+                                                                  event_register_cost,
+                                                                  event_register_ratio,
+                                                                  event_jin_jian_app,
+                                                                  event_jin_jian_app_cost,
+                                                                  event_credit_grant_app,
+                                                                  event_credit_grant_app_cost,
+                                                                  event_order_paid,
+                                                                  event_order_paid_purchase_amount,
+                                                                  event_next_day_stay_cost,
+                                                                  event_next_day_stay_ratio,
+                                                                  event_order_paid_cost,
+                                                                  event_next_day_stay,
+                                                                  event_app_invoked,
+                                                                  event_credit_grant_app_ratio)
+        select project_id,
+               signature,
+               stat_date,
+               ${originTableKey},
+               sum(aclick),
+               sum(bclick),
+               round(sum(charge) / sum(photo_click), 2),
+               sum(download_started),
+               sum(download_completed),
+               sum(activation),
+               sum(submit),
+               sum(photo_show),
+               sum(photo_click),
+               round(sum(photo_click) / sum(photo_show) * 100, 2),
+               sum(charge),
+               round(sum(bclick) / sum(aclick) * 100, 2),
+               round(sum(charge) / sum(photo_show) * 1000, 2),
+               round(sum(charge) / sum(bclick), 2),
+               sum(event_pay_first_day),
+               sum(event_pay_purchase_amount_first_day),
+               sum(event_pay),
+               sum(event_pay_purchase_amount),
+               sum(event_register),
+               round(sum(charge) / sum(event_register), 2),
+               round(sum(event_register) / sum(activation) * 100, 2),
+               sum(event_jin_jian_app),
+               round(sum(charge) / sum(event_jin_jian_app),2),
+               sum(event_credit_grant_app),
+               round(sum(charge) / sum(event_credit_grant_app),2),
+               sum(event_order_paid),
+               sum(event_order_paid_purchase_amount),
+               round(sum(charge) / sum(event_next_day_stay),2),
+               round(sum(event_next_day_stay) / sum(activation) * 100, 2),
+               round(sum(charge) / sum(event_order_paid),2),
+               sum(event_next_day_stay),
+               sum(event_app_invoked),
+               round(sum(form_count) / sum(bclick) * 100, 2)
+        from ${originTable}
+        where stat_date = #{date}
+          and project_id = 458
+        group by signature, ${originTableKey};
+    </insert>
 </mapper>
 </mapper>

+ 2 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/service/IKuaishouAudienceReportDailyService.java

@@ -26,4 +26,6 @@ public interface IKuaishouAudienceReportDailyService extends IService<KuaishouAu
     void crowdAnalysisReport2(CtopOauthToken token, String date, String type, Long plan, Long unit, Integer page, Integer pageSize);
     void crowdAnalysisReport2(CtopOauthToken token, String date, String type, Long plan, Long unit, Integer page, Integer pageSize);
 
 
     void crowdAnalysisReport3(CtopOauthToken token, String date, String type, Long plan, JSONArray unit, Integer page, Integer pageSize);
     void crowdAnalysisReport3(CtopOauthToken token, String date, String type, Long plan, JSONArray unit, Integer page, Integer pageSize);
+
+    void aggregationData(String param);
 }
 }

+ 12 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/service/impl/KuaishouAudienceReportDailyServiceImpl.java

@@ -16,6 +16,7 @@ import cn.com.ctop.kuaishou.modules.report.entity.KuaishouAudienceReportDailyCit
 import cn.com.ctop.kuaishou.modules.report.entity.KuaishouAudienceReportDailyClient;
 import cn.com.ctop.kuaishou.modules.report.entity.KuaishouAudienceReportDailyClient;
 import cn.com.ctop.kuaishou.modules.report.entity.KuaishouAudienceReportDailyGender;
 import cn.com.ctop.kuaishou.modules.report.entity.KuaishouAudienceReportDailyGender;
 import cn.com.ctop.kuaishou.modules.report.entity.KuaishouAudienceReportDailyProvince;
 import cn.com.ctop.kuaishou.modules.report.entity.KuaishouAudienceReportDailyProvince;
+import cn.com.ctop.kuaishou.modules.report.enums.AudienceReportTableEnum;
 import cn.com.ctop.kuaishou.modules.report.mapper.KuaishouAudienceReportDailyAgeMapper;
 import cn.com.ctop.kuaishou.modules.report.mapper.KuaishouAudienceReportDailyAgeMapper;
 import cn.com.ctop.kuaishou.modules.report.mapper.KuaishouAudienceReportDailyBusinessMapper;
 import cn.com.ctop.kuaishou.modules.report.mapper.KuaishouAudienceReportDailyBusinessMapper;
 import cn.com.ctop.kuaishou.modules.report.mapper.KuaishouAudienceReportDailyCityMapper;
 import cn.com.ctop.kuaishou.modules.report.mapper.KuaishouAudienceReportDailyCityMapper;
@@ -405,5 +406,16 @@ public class KuaishouAudienceReportDailyServiceImpl extends ServiceImpl<Kuaishou
             log.error("插入人群分析报表数据异常,数据维度:{}", type, e);
             log.error("插入人群分析报表数据异常,数据维度:{}", type, e);
         }
         }
     }
     }
+
+    @Override
+    public void aggregationData(String date) {
+        kuaishouAudienceReportDailyMapper.aggregationData(date,AudienceReportTableEnum.CITY.getOriginTable(),AudienceReportTableEnum.CITY.getOriginTableKey(),AudienceReportTableEnum.CITY.getAggregationTable(),AudienceReportTableEnum.CITY.getAggregationTableKey());
+        kuaishouAudienceReportDailyMapper.aggregationData(date,AudienceReportTableEnum.AGE.getOriginTable(),AudienceReportTableEnum.AGE.getOriginTableKey(),AudienceReportTableEnum.AGE.getAggregationTable(),AudienceReportTableEnum.AGE.getAggregationTableKey());
+        kuaishouAudienceReportDailyMapper.aggregationData(date,AudienceReportTableEnum.CLIENT.getOriginTable(),AudienceReportTableEnum.CLIENT.getOriginTableKey(),AudienceReportTableEnum.CLIENT.getAggregationTable(),AudienceReportTableEnum.CLIENT.getAggregationTableKey());
+        kuaishouAudienceReportDailyMapper.aggregationData(date,AudienceReportTableEnum.GENDER.getOriginTable(),AudienceReportTableEnum.GENDER.getOriginTableKey(),AudienceReportTableEnum.GENDER.getAggregationTable(),AudienceReportTableEnum.GENDER.getAggregationTableKey());
+        kuaishouAudienceReportDailyMapper.aggregationData(date,AudienceReportTableEnum.BUSINESS.getOriginTable(),AudienceReportTableEnum.BUSINESS.getOriginTableKey(),AudienceReportTableEnum.BUSINESS.getAggregationTable(),AudienceReportTableEnum.BUSINESS.getAggregationTableKey());
+        kuaishouAudienceReportDailyMapper.aggregationData(date,AudienceReportTableEnum.PROVINCE.getOriginTable(),AudienceReportTableEnum.PROVINCE.getOriginTableKey(),AudienceReportTableEnum.PROVINCE.getAggregationTable(),AudienceReportTableEnum.PROVINCE.getAggregationTableKey());
+
+    }
 }
 }
 
 

+ 0 - 15
pom.xml

@@ -472,27 +472,12 @@
             </plugin>
             </plugin>
             <!-- 打包跳过测试 -->
             <!-- 打包跳过测试 -->
             <plugin>
             <plugin>
-
 		        <groupId>org.apache.maven.plugins</groupId>
 		        <groupId>org.apache.maven.plugins</groupId>
 		        <artifactId>maven-surefire-plugin</artifactId>
 		        <artifactId>maven-surefire-plugin</artifactId>
 		        <configuration>
 		        <configuration>
 		          <skipTests>true</skipTests>
 		          <skipTests>true</skipTests>
 		        </configuration>
 		        </configuration>
 	        </plugin>
 	        </plugin>
-<!--	         &lt;!&ndash; 避免font文件的二进制文件格式压缩破坏 &ndash;&gt;-->
-<!--	         <plugin>-->
-<!--                <groupId>org.apache.maven.plugins</groupId>-->
-<!--                <artifactId>maven-resources-plugin</artifactId>-->
-<!--                <configuration>-->
-<!--                    <nonFilteredFileExtensions>-->
-<!--                        <nonFilteredFileExtension>woff</nonFilteredFileExtension>-->
-<!--                        <nonFilteredFileExtension>woff2</nonFilteredFileExtension>-->
-<!--                        <nonFilteredFileExtension>eot</nonFilteredFileExtension>-->
-<!--                        <nonFilteredFileExtension>ttf</nonFilteredFileExtension>-->
-<!--                        <nonFilteredFileExtension>svg</nonFilteredFileExtension>-->
-<!--                    </nonFilteredFileExtensions>-->
-<!--                </configuration>-->
-<!--            </plugin>-->
         </plugins>
         </plugins>
         <resources>
         <resources>
             <resource>
             <resource>