Pārlūkot izejas kodu

添加头条视频消耗数据清洗任务

songyh 4 gadi atpakaļ
vecāks
revīzija
0831183917

+ 5 - 0
module-job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedanceEtlVideoJob.java

@@ -41,4 +41,9 @@ public class BytedanceEtlVideoJob {
     public void updBytedanceVideoJob() throws Exception {
         etlBytedanceReportVideoDailyService.updBytedanceVideo();
     }
+
+    @XxlJob("etlBytedanceVideoJob")
+    public void etlBytedanceVideoJob() throws Exception {
+        etlBytedanceReportVideoDailyService.etlBytedanceVideo();
+    }
 }

+ 2 - 1
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/batch/service/impl/BytedanceAreaInfoServiceImpl.java

@@ -9,6 +9,7 @@ import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
+import javax.annotation.Resource;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -21,7 +22,7 @@ import java.util.List;
  */
 @Service
 public class BytedanceAreaInfoServiceImpl extends ServiceImpl<BytedanceAreaInfoMapper, BytedanceAreaInfo> implements IBytedanceAreaInfoService {
-    @Autowired
+    @Resource
     private BytedanceAreaInfoMapper bytedanceAreaInfoMapper;
     @Override
     public boolean hasChild(BytedanceAreaInfo areaInfo) {

+ 4 - 1
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/EtlBytedanceReportVideoDailyMapper.java

@@ -1,7 +1,9 @@
 package cn.com.ctop.toutiao.modules.report.mapper;
 
+import cn.com.ctop.toutiao.modules.material.entity.EtlBytedanceVideoInfo;
 import cn.com.ctop.toutiao.modules.report.entity.EtlBytedanceReportVideoDaily;
 import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import org.apache.ibatis.annotations.Param;
 
 import java.util.List;
 import java.util.Map;
@@ -19,6 +21,7 @@ public interface EtlBytedanceReportVideoDailyMapper extends BaseMapper<EtlByteda
     void updBytedanceVideoByMd5(String md5,String coverUrl,String url);
 
 
+    List<EtlBytedanceVideoInfo> etlBytedanceVideoInfo(@Param(value = "date") String date);
 
-
+    void replaceEtlVideoInfo(@Param(value = "videoInfo") EtlBytedanceVideoInfo videoInfo);
 }

+ 160 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/xml/EtlBytedanceReportVideoDailyMapper.xml

@@ -24,6 +24,166 @@
         ORDER BY
             total DESC;
     </select>
+    <select id="etlBytedanceVideoInfo"
+            resultType="cn.com.ctop.toutiao.modules.material.entity.EtlBytedanceVideoInfo">
+             select
+                      (case when cmi.url is not null then cmi.url else '' end) 'url',
+                          (CASE when cmi.cover_url is null then ''
+                                ELSE cmi.cover_url end) 'conver_url',
+                          (case when cmi.material_name is null then '' else cmi.material_name end)  'material_name',
+                          (case when cbvi.signature is null then '' else cbvi.signature end)'md5',
+                          report.material_id 'material_id',
+                          cp.project_name 'project_name',
+                          case when etlVideo.clip_name is null then '' else etlVideo.clip_name end 'clip_name',
+                          case when etlVideo.plan_name is null then '' else etlVideo.plan_name end 'plan_name',
+                          case when etlVideo.shot_name is null then '' else etlVideo.shot_name end 'shot_name',
+                          case when cmi.create_time is not null then cmi.create_time else '' end 'material_create_time',
+                          sum(report.cost) 'cost',
+                          sum(report.click) 'click_material',
+                          (case when sum(report.show_material)>0 and sum(report.click)>0 then round(sum(report.click)/sum(report.show_material)*100) else 0 end) 'ctr',
+                          sum(report.show_material) 'show_material',
+                          sum(report.convert_material) 'convert_material',
+                          (case when sum(report.convert_material)>0 and sum(report.click)>0 then round(sum(report.convert_material)/sum(report.click)*100) else 0 end) 'convert_rate',
+                          (case when sum(report.cost) >0 and sum(report.show_material)>0 then round(sum(report.cost)/sum(report.show_material)*1000,2) else 0 end) 'cpm',
+                          (case when sum(report.cost) >0 and sum(report.click)>0 then round(sum(report.cost)/sum(report.click),2) else 0 end) 'cpc',
+                          (case when sum(report.cost) >0 and sum(report.convert_material)>0 then round(sum(report.cost)/sum(report.convert_material),2) else 0 end) 'convert_cost',
+                          sum(report.download_start) 'download_start',
+                          sum(report.download_finish) 'download_finish',
+                          sum(report.active) 'active',
+                          sum(report.register) 'register',
+                          sum(report.next_day_open) 'next_day_open',
+                          sum(report.pay_count) 'active_pay_amount',
+                          (case when sum(report.cost) >0 and sum(report.pay_count)>0 then round(sum(report.cost)/sum(report.pay_count),2) else 0 end)  'active_pay_cost',
+                          (case when sum(report.cost) >0 and sum(report.next_day_open)>0 then round(sum(report.cost)/sum(report.next_day_open),2) else 0 end) 'next_day_open_cost',
+                          (case when sum(report.cost) >0 and sum(report.register)>0 then round(sum(report.cost)/sum(report.register),2) else 0 end) 'active_register_cost',
+                          (case when sum(report.cost) >0 and sum(report.active)>0 then round(sum(report.cost)/sum(report.active),2) else 0 end) 'active_cost',
+                          (case when sum(report.cost) >0 and sum(report.download_finish)>0 then round(sum(report.cost)/sum(report.download_finish),2) else 0 end) 'download_finish_cost',
+                          (case when sum(report.cost) >0 and sum(report.download_start)>0 then round(sum(report.cost)/sum(report.download_start),2) else 0 end) 'download_start_cost',
+                          (case when sum(report.download_start) >0 and sum(report.click)>0 then round(sum(report.download_start)/sum(report.click)*100,2) else 0 end) 'download_start_rate',
+                          (case when sum(report.download_finish) >0 and sum(report.download_start)>0 then round(sum(report.download_finish)/sum(report.download_start)*100,2) else 0 end) 'download_finish_rate',
+                          (case when sum(report.active) >0 and sum(report.click)>0 then round(sum(report.active)/sum(report.click)*100,2) else 0 end) 'active_rate',
+                          (case when sum(report.register) >0 and sum(report.active)>0 then round(sum(report.register)/sum(report.active)*100,2) else 0 end) 'active_register_rate',
+                          (case when sum(report.active) >0 and sum(report.next_day_open)>0 then round(sum(report.next_day_open)/sum(report.active)*100,2) else 0 end) 'next_day_open_rate',
+                          sum(report.form) 'form',
+                          (case when sum(report.cost) >0 and sum(report.valid_play)>0 then round(sum(report.cost)/sum(report.valid_play),2) else 0 end) 'valid_play_cost',
+                          sum(report.play75_feed_break) 'play75_feed_break',
+                          sum(report.play100_feed_break) 'play100_feed_break',
+                          sum(report.play25_feed_break) 'play25_feed_break',
+                          sum(report.play50_feed_break) 'play50_feed_break',
+                          sum(report.share_material) 'share_material',
+                          sum(report.follow) 'follow',
+                          sum(report.like_material) 'like_material',
+                          report.stat_datetime 'stat_datetime',
+                      report.account_id 'account_id'
+  from ctop_bytedance_report_material_daily report
+           left join ctop_user_allocation cua on cua.account_id = report.account_id
+           left join (select distinct account_id,material_id,signature from ctop_bytedance_video_info ) cbvi
+                     on report.account_id = cbvi.account_id and report.material_id = cbvi.material_id
+           left join ctop_bytedance_video_etl_info etlVideo on etlVideo.video_code = cbvi.signature
+           left join ctop_material_info cmi on cmi.code = cbvi.signature and cmi.project_id = cua.project_id
+           left join ctop_project cp on cp.id = cua.project_id
+  where 1 = 1
+    and report.stat_datetime = #{date}
+  group by report.material_id,report.account_id;
+    </select>
+    <select id="replaceEtlVideoInfo">
+        REPLACE INTO etl_report_bytedance_video(
+            url,
+            cover_url,
+            material_name,
+            md5,
+            material_id,
+            project_name ,
+            clip_name,
+            plan_name,
+            shot_name,
+            material_create_time,
+            cost,
+            click,
+            ctr,
+            material_show,
+            convert_material,
+            convert_rate,
+            cpm,
+            cpc,
+            convert_cost,
+            download_start,
+            download_finish,
+            active,
+            register,
+            next_day_open,
+            pay_count,
+            active_pay_cost,
+            next_day_open_cost,
+            active_register_cost,
+            active_cost,
+            download_finish_cost,
+            download_start_cost,
+            download_start_rate,
+            download_finish_rate,
+            active_rate,
+            active_register_rate,
+            next_day_open_rate,
+            form,
+            valid_play_cost,
+            play75_feed_break,
+            play100_feed_break,
+            play25_feed_break,
+            play50_feed_break,
+            share_material,
+            follow,
+            like_material,
+            stat_datetime,
+            account_id)
+    values (
+            #{videoInfo.url},
+            #{videoInfo.coverUrl},
+            #{videoInfo.materialName},
+            #{videoInfo.md5},
+            #{videoInfo.materialId},
+            #{videoInfo.projectName },
+            #{videoInfo.clipName},
+            #{videoInfo.planName},
+            #{videoInfo.shotName},
+            #{videoInfo.materialCreateTime},
+            #{videoInfo.cost},
+            #{videoInfo.click},
+            #{videoInfo.ctr},
+            #{videoInfo.materialShow},
+            #{videoInfo.convertMaterial},
+            #{videoInfo.convertRate},
+            #{videoInfo.cpm},
+            #{videoInfo.cpc},
+            #{videoInfo.convertCost},
+            #{videoInfo.downloadStart},
+            #{videoInfo.downloadFinish},
+            #{videoInfo.active},
+            #{videoInfo.register},
+            #{videoInfo.nextDayOpen},
+            #{videoInfo.payCount},
+            #{videoInfo.activePayCost},
+            #{videoInfo.nextDayOpenCost},
+            #{videoInfo.activeRegisterCost},
+            #{videoInfo.activeCost},
+            #{videoInfo.downloadFinishCost},
+            #{videoInfo.downloadStartCost},
+            #{videoInfo.downloadStartRate},
+            #{videoInfo.downloadFinishRate},
+            #{videoInfo.activeRate},
+            #{videoInfo.activeRegisterRate},
+            #{videoInfo.nextDayOpenRate},
+            #{videoInfo.form},
+            #{videoInfo.validPlayCost},
+            #{videoInfo.play75FeedBreak},
+            #{videoInfo.play100FeedBreak},
+            #{videoInfo.play25FeedBreak},
+            #{videoInfo.play50FeedBreak},
+            #{videoInfo.shareMaterial},
+            #{videoInfo.follow},
+            #{videoInfo.likeMaterial},
+            #{videoInfo.statDatetime},
+            #{videoInfo.accountId})
+    </select>
 
 
     <update id="updBytedanceVideoByMd5">

+ 1 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/IEtlBytedanceReportVideoDailyService.java

@@ -16,4 +16,5 @@ public interface IEtlBytedanceReportVideoDailyService extends IService<EtlByteda
     Result updBytedanceVideo() throws Exception;
 
 
+    Result etlBytedanceVideo() throws Exception;
 }

+ 42 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/EtlBytedanceReportVideoDailyServiceImpl.java

@@ -1,5 +1,6 @@
 package cn.com.ctop.toutiao.modules.report.service.impl;
 
+import cn.com.ctop.toutiao.modules.material.entity.EtlBytedanceVideoInfo;
 import cn.com.ctop.toutiao.modules.report.entity.EtlBytedanceReportVideoDaily;
 import cn.com.ctop.toutiao.modules.report.mapper.EtlBytedanceReportVideoDailyMapper;
 import cn.com.ctop.toutiao.modules.report.service.IEtlBytedanceReportVideoDailyService;
@@ -8,6 +9,7 @@ import com.google.api.client.util.Data;
 import com.google.common.base.Function;
 import org.checkerframework.checker.nullness.qual.Nullable;
 import org.jeecg.common.api.vo.Result;
+import org.jeecg.common.util.DateUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
@@ -17,6 +19,9 @@ import java.util.Date;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import java.util.function.BiFunction;
 
 /**
@@ -55,6 +60,43 @@ public class EtlBytedanceReportVideoDailyServiceImpl extends ServiceImpl<EtlByte
         return Result.ok(future.get());
     }
 
+    static ExecutorService executorService = Executors.newFixedThreadPool(20);
+    static CountDownLatch countDownLatch = null;
+    @Override
+    public Result etlBytedanceVideo() throws Exception{
+        String date = DateUtils.formatDate();
+        CompletableFuture future = CompletableFuture.supplyAsync(()->{
+            return etlBytedanceReportVideoDailyMapper.etlBytedanceVideoInfo(date);
+        }).handle(new BiFunction<List<EtlBytedanceVideoInfo>, Throwable, Object>() {
+            @Override
+            public Object apply(List<EtlBytedanceVideoInfo> list, Throwable throwable) {
+                if (throwable != null){
+                    return Result.error(throwable.getMessage());
+                }
+                if(null!=list&&!list.isEmpty()){
+                    countDownLatch = new CountDownLatch(list.size());
+                    list.forEach(videoInfo ->{
+                        executorService.submit(()->{
+                            try {
+                                etlBytedanceReportVideoDailyMapper.replaceEtlVideoInfo(videoInfo);
+                            }catch (Exception e){
+                            }finally {
+                                countDownLatch.countDown();
+                            }
+                        });
+                    });
+                }
+                try {
+                    countDownLatch.await();
+                } catch (InterruptedException e) {
+                    e.printStackTrace();
+                }
+                return Result.ok("数据清洗完成。");
+            }
+        });
+        return Result.ok(future.get());
+    }
+