Przeglądaj źródła

添加增量数据聚合任务

songyh 3 lat temu
rodzic
commit
86db73fb4a

+ 1 - 1
module-job-kuaishou/src/main/java/cn/com/ctop/job/kuaishou/handler/KuaishouDailyAudienceReportJob.java

@@ -97,7 +97,7 @@ public class KuaishouDailyAudienceReportJob {
         if (param==null || param.trim().length()==0) {
             param = date;
         }
-        kuaishouAudienceReportDailyService.aggregationData(param);
+        kuaishouAudienceReportDailyService.aggregationDataByProjectId(param,458L);
     }
 
 

+ 9 - 2
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/KuaishouAudienceReportDailyMapper.java

@@ -12,9 +12,16 @@ import org.apache.ibatis.annotations.Param;
  * @cersion: V1.0
  */
 public interface KuaishouAudienceReportDailyMapper extends BaseMapper<KuaishouAudienceReportDaily> {
-    void aggregationData(@Param(value = "date") String date,
+    void aggregationData(@Param(value = "projectId") Long projectId,
+                         @Param(value = "date") String date,
                          @Param(value = "originTable") String originTable,
                          @Param(value = "originTableKey")String originTableKey,
                          @Param(value = "aggregationTable")String aggregationTable,
-                         @Param(value = "aggregationTableKey")String aggregationTableKey);
+                         @Param(value = "aggregationTableKey")String aggregationTableKey,
+                         @Param(value = "startIndex")int startIndex);
+
+    Integer countTotal(@Param(value = "date")String date,
+                   @Param(value = "projectId")Long projectId,
+                   @Param(value = "originTable")String getOriginTable,
+                   @Param(value = "originTableKey")String getOriginTableKey);
 }

+ 11 - 2
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/xml/KuaishouAudienceReportDailyMapper.xml

@@ -78,7 +78,16 @@
                round(sum(form_count) / sum(bclick) * 100, 2)
         from ${originTable}
         where stat_date = #{date}
-          and project_id = 458
-        group by signature, ${originTableKey};
+          and project_id = #{projectId}
+        group by signature, ${originTableKey}
+        order by signature,${originTableKey}
+        limit #{startIndex},100000
     </insert>
+    <select id="countTotal" resultType="java.lang.Integer">
+        select count(1)
+        from ${originTable}
+        where stat_date = #{date}
+          and project_id = #{projectId}
+        group by signature, ${originTableKey}
+    </select>
 </mapper>

+ 1 - 1
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/service/IKuaishouAudienceReportDailyService.java

@@ -27,5 +27,5 @@ public interface IKuaishouAudienceReportDailyService extends IService<KuaishouAu
 
     void crowdAnalysisReport3(CtopOauthToken token, String date, String type, Long plan, JSONArray unit, Integer page, Integer pageSize);
 
-    void aggregationData(String param);
+    void aggregationDataByProjectId(String date, Long projectId);
 }

+ 21 - 7
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/service/impl/KuaishouAudienceReportDailyServiceImpl.java

@@ -408,14 +408,28 @@ public class KuaishouAudienceReportDailyServiceImpl extends ServiceImpl<Kuaishou
     }
 
     @Override
-    public void aggregationData(String date) {
-        kuaishouAudienceReportDailyMapper.aggregationData(date,AudienceReportTableEnum.CITY.getOriginTable(),AudienceReportTableEnum.CITY.getOriginTableKey(),AudienceReportTableEnum.CITY.getAggregationTable(),AudienceReportTableEnum.CITY.getAggregationTableKey());
-        kuaishouAudienceReportDailyMapper.aggregationData(date,AudienceReportTableEnum.AGE.getOriginTable(),AudienceReportTableEnum.AGE.getOriginTableKey(),AudienceReportTableEnum.AGE.getAggregationTable(),AudienceReportTableEnum.AGE.getAggregationTableKey());
-        kuaishouAudienceReportDailyMapper.aggregationData(date,AudienceReportTableEnum.CLIENT.getOriginTable(),AudienceReportTableEnum.CLIENT.getOriginTableKey(),AudienceReportTableEnum.CLIENT.getAggregationTable(),AudienceReportTableEnum.CLIENT.getAggregationTableKey());
-        kuaishouAudienceReportDailyMapper.aggregationData(date,AudienceReportTableEnum.GENDER.getOriginTable(),AudienceReportTableEnum.GENDER.getOriginTableKey(),AudienceReportTableEnum.GENDER.getAggregationTable(),AudienceReportTableEnum.GENDER.getAggregationTableKey());
-        kuaishouAudienceReportDailyMapper.aggregationData(date,AudienceReportTableEnum.BUSINESS.getOriginTable(),AudienceReportTableEnum.BUSINESS.getOriginTableKey(),AudienceReportTableEnum.BUSINESS.getAggregationTable(),AudienceReportTableEnum.BUSINESS.getAggregationTableKey());
-        kuaishouAudienceReportDailyMapper.aggregationData(date,AudienceReportTableEnum.PROVINCE.getOriginTable(),AudienceReportTableEnum.PROVINCE.getOriginTableKey(),AudienceReportTableEnum.PROVINCE.getAggregationTable(),AudienceReportTableEnum.PROVINCE.getAggregationTableKey());
+    public void aggregationDataByProjectId(String date, Long projectId) {
+        this.aggregationData(date,projectId,AudienceReportTableEnum.CITY.getOriginTable(),AudienceReportTableEnum.CITY.getOriginTableKey(),AudienceReportTableEnum.CITY.getAggregationTable(),AudienceReportTableEnum.CITY.getAggregationTableKey());
+        this.aggregationData(date,projectId,AudienceReportTableEnum.AGE.getOriginTable(),AudienceReportTableEnum.AGE.getOriginTableKey(),AudienceReportTableEnum.AGE.getAggregationTable(),AudienceReportTableEnum.AGE.getAggregationTableKey());
+        this.aggregationData(date,projectId,AudienceReportTableEnum.CLIENT.getOriginTable(),AudienceReportTableEnum.CLIENT.getOriginTableKey(),AudienceReportTableEnum.CLIENT.getAggregationTable(),AudienceReportTableEnum.CLIENT.getAggregationTableKey());
+        this.aggregationData(date,projectId,AudienceReportTableEnum.GENDER.getOriginTable(),AudienceReportTableEnum.GENDER.getOriginTableKey(),AudienceReportTableEnum.GENDER.getAggregationTable(),AudienceReportTableEnum.GENDER.getAggregationTableKey());
+        this.aggregationData(date,projectId,AudienceReportTableEnum.BUSINESS.getOriginTable(),AudienceReportTableEnum.BUSINESS.getOriginTableKey(),AudienceReportTableEnum.BUSINESS.getAggregationTable(),AudienceReportTableEnum.BUSINESS.getAggregationTableKey());
+        this.aggregationData(date,projectId,AudienceReportTableEnum.PROVINCE.getOriginTable(),AudienceReportTableEnum.PROVINCE.getOriginTableKey(),AudienceReportTableEnum.PROVINCE.getAggregationTable(),AudienceReportTableEnum.PROVINCE.getAggregationTableKey());
+    }
 
+    public void aggregationData(String date,Long projectId,String getOriginTable,String getOriginTableKey,String getAggregationTable,String getAggregationTableKey){
+        //查询总数据量
+        Integer count = kuaishouAudienceReportDailyMapper.countTotal(date,projectId,getOriginTable,getOriginTableKey);
+        if(null == count||count == 0){
+            return;
+        }
+        int startIndex = 0;
+        while (count>0){
+            kuaishouAudienceReportDailyMapper.aggregationData(projectId,date,AudienceReportTableEnum.CITY.getOriginTable(),AudienceReportTableEnum.CITY.getOriginTableKey(),AudienceReportTableEnum.CITY.getAggregationTable(),AudienceReportTableEnum.CITY.getAggregationTableKey(),startIndex);
+            startIndex+=100000;
+            count-=100000;
+        }
     }
 }
 
+