Kaynağa Gözat

release of '视频报表etl清洗任务' on 2021.02.04

jiequan.bi 4 yıl önce
ebeveyn
işleme
4392bb16f1

+ 9 - 9
jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java

@@ -544,16 +544,16 @@ public class SampleTest {
     @Test
     public void etlBytedanceVideoReport() {
 
-        for (int i = 1; i <= 62; i++) {
-            etlBytedanceVideo(DateUtils.formatDate(DateUtils.addDay(new Date(), -i)));
-        }
-        //etlBytedanceVideo("2021-02-01");
+//        for (int i = 1; i <= 62; i++) {
+//            etlBytedanceVideo(DateUtils.formatDate(DateUtils.addDay(new Date(), -i)));
+//        }
+        etlBytedanceVideo("2021-02-03");
 
     }
 
     private void etlBytedanceVideo(String statDate) {
         List<CtopOauthToken> tokens = oauthTokenService.selectToutiaoToken();
-        executorService = Executors.newFixedThreadPool(30);
+        executorService = Executors.newFixedThreadPool(10);
         countDownLatch = new CountDownLatch(tokens.size());
         tokens.forEach(token -> executorService.submit(() -> {
             try {
@@ -576,10 +576,10 @@ public class SampleTest {
 
     @Test
     public void etlBytedanceVideoReportClean() {
-        for (int i = 1; i <= 1; i++) {
-            String statDate = DateUtils.formatDate(DateUtils.addDay(new Date(), -i));
-            ietlReportBytedanceVideoService.cleanETL(statDate);
-        }
+        //for (int i = 1; i <= 1; i++) {
+          //  String statDate = DateUtils.formatDate(DateUtils.addDay(new Date(), -i));
+            ietlReportBytedanceVideoService.cleanETL("2020-12-31");
+        //}
     }
 
     @Test

+ 31 - 1
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/mapper/xml/ETLReportBytedanceVideoMapper.xml

@@ -52,7 +52,7 @@
     <select id="queryRelatePersonByMd5Old"  resultType="com.alibaba.fastjson.JSONObject">
         SELECT
             clip_id clipId,
-            (select realname from sys_user where id=clip_id) as clipaname,
+            (select realname from sys_user where id=clip_id) as clipName,
             shot_id shotId,
             (select realname from sys_user where id=shot_id) as shotName,
             plan_id planId,
@@ -76,4 +76,34 @@
             account_id = #{accountId}
         LIMIT 1
     </select>
+
+    <select id="queryETLData"  resultType="cn.com.ctop.toutiao.modules.link.entity.ETLReportBytedanceVideo">
+        SELECT
+            t1.*,
+            t3.clip_id clipId,
+            (select realname from sys_user where id=t3.clip_id) as clipName,
+            t3.shot_id shotId,
+            (select realname from sys_user where id=t3.shot_id) as shotName,
+            t3.plan_id planId,
+            (select realname from sys_user where id=t3.plan_id) as planName,
+            t5.advertiser_id as advertiserId,
+            (SELECT `name` from ctop_advertiser where id= t5.advertiser_id limit 1) as advertiserName,
+            t5.project_id as projectId,
+            (SELECT `project_name` from ctop_project where id= t5.project_id limit 1) as projectName,
+            t5.auth_name as accountName,
+            t4.material_name as materialName,
+            t4.url,
+            t4.cover_url coverUrl,
+            t4.create_time as materialCreateTime,
+            t2.signature AS md5,
+            t2.video_url AS toutiaoUrl
+        FROM
+            (SELECT * from base_report_bytedance_video_daily where stat_datetime=#{statDate}) t1
+                LEFT JOIN (select material_id,signature,video_url from ctop_bytedance_video_info where material_id in (SELECT DISTINCT material_id from base_report_bytedance_video_daily where stat_datetime=#{statDate})
+            ) t2 ON t1.material_id = t2.material_id
+                LEFT JOIN ctop_material_ascription t3 on t2.signature=t3.code
+                LEFT JOIN ctop_material_info t4 on t4.code=t2.signature
+                LEFT JOIN (SELECT account_id, advertiser_id,project_id,auth_name from ctop_user_allocation where account_id in (SELECT DISTINCT account_id from base_report_bytedance_video_daily where stat_datetime=#{statDate})) t5 on t1.account_id=t5.account_id
+        GROUP BY t1.id
+    </select>
 </mapper>

+ 17 - 7
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/service/impl/ETLReportBytedanceVideoServiceImpl.java

@@ -37,7 +37,7 @@ public class ETLReportBytedanceVideoServiceImpl extends ServiceImpl<ETLReportByt
         return baseReportBytedanceVideoDailyService.list(queryWrapper);
     }
 
-    @Override
+/*    @Override
     public void cleanETL(String statDate) {
         List<BaseReportBytedanceVideoDaily> source = listByStatDate(statDate);
         if(source.isEmpty()){
@@ -62,7 +62,8 @@ public class ETLReportBytedanceVideoServiceImpl extends ServiceImpl<ETLReportByt
                 if(null!=materialVideo&&!materialVideo.isEmpty()) {
                     temp.setUrl(materialVideo.getString("url"));
                     temp.setCoverUrl(materialVideo.getString("coverUrl"));
-                    temp.setMaterialCreateTime(materialVideo.getDate("creteTime").toString());
+                    temp.setMaterialName(materialVideo.getString("materialName"));
+                    temp.setMaterialCreateTime(materialVideo.getDate("createTime").toString());
                 }
                 if(null!=accountDetail&&!accountDetail.isEmpty()){
                     temp.setAccountName(accountDetail.getString("accountName"));
@@ -83,13 +84,22 @@ public class ETLReportBytedanceVideoServiceImpl extends ServiceImpl<ETLReportByt
                 }
                 batch.add(temp);
             }
+            this.saveBatch(batch);
         });
-        boolean result = this.saveBatch(batch);
-        if(result){
-            XxlJobHelper.log("头条数据报表清洗成功,数据条数:{}",batch.size());
-        }else {
-            XxlJobHelper.log("头条数据报表清洗入库失败请重试,数据条数:{}",batch.size());
+    }*/
+
+    @Override
+    public void cleanETL(String statDate) {
+        List<ETLReportBytedanceVideo> etlReportBytedanceVideos = etlReportBytedanceVideoMapper.queryETLData(statDate);
+        if(etlReportBytedanceVideos.isEmpty()){
+            log.warn("{},头条etl报表清洗异常,base数据源为空",statDate);
+            return;
         }
+        boolean result = this.saveBatch(etlReportBytedanceVideos);
+        if(result)
+            log.info("{},头条etl报表清洗完成,数量:{}",statDate,etlReportBytedanceVideos.size());
+        else
+            log.error("{},头条etl报表清洗失败,请重试",statDate);
     }
 
 }

+ 2 - 2
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/xml/BytedanceVideoReportMapper.xml

@@ -12,7 +12,7 @@
         t1.plan_name AS plan,
         t1.material_name AS materialName,
         t1.project_name AS projectName,
-        t1.matertial_create_time AS createTime,
+        t1.material_create_time AS createTime,
         ${filed}
         FROM
         etl_report_bytedance_video t1
@@ -42,7 +42,7 @@
         t1.material_name AS materialName,
         t1.project_name AS projectName,
         t1.advertiser_name as advertiserName,
-        t1.matertial_create_time AS createTime,
+        t1.material_create_time AS createTime,
         ${filed}
         FROM
         etl_report_bytedance_video t1

+ 1 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/BytedanceVideoReportServiceImpl.java

@@ -88,6 +88,7 @@ public class BytedanceVideoReportServiceImpl implements IBytedanceVideoReportSer
             JSONObject metrics= videoReportJSONObject.getJSONObject("metrics");
             BaseReportBytedanceVideoDaily temp= JSONObject.toJavaObject(metrics,BaseReportBytedanceVideoDaily.class);
             temp.setMaterialShow(metrics.getLong("show"));
+            temp.setConvertMaterial(metrics.getLong("convert"));
             temp.setAccountId(oauthToken.getAccountId());
             temp.setMaterialId(materialId);
             temp.setStatDatetime(startDate);

+ 1 - 1
module-toutiao/src/main/resources/json_template/bytedance_video_report_mapping.json

@@ -32,7 +32,7 @@
     "comment": "转化成本"
   },
   "convertRate": {
-    "filed": "CASE WHEN sum(t1.convert_material) / sum(t1.cost) IS NULL THEN 0 ELSE concat(round(sum(t1.convert_material) / sum(t1.cost)*100,2),'%') end AS convertRate ",
+    "filed": "CASE WHEN sum(t1.convert_material) / sum(t1.material_show) IS NULL THEN 0 ELSE concat(round(sum(t1.convert_material) / sum(t1.material_show)*100,2),'%') end AS convertRate ",
     "comment": "转化率"
   },
   "downloadStart": {