Browse Source

头条数据清洗

yumeng 3 years ago
parent
commit
49ff49c96d

+ 1 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/entity/ETLReportBytedanceVideo.java

@@ -89,6 +89,7 @@ public class ETLReportBytedanceVideo {
 	@Excel(name = "编导名称", width = 15)
     @ApiModelProperty(value = "编导名称")
 	private String planName;
+	private String teamLeader;
 	/**materialCreateTime*/
 	private String materialCreateTime;
 	/**数据起始时间*/

+ 3 - 1
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/mapper/ETLReportBytedanceVideoMapper.java

@@ -39,10 +39,12 @@ public interface ETLReportBytedanceVideoMapper extends BaseMapper<ETLReportByted
     List<ETLReportBytedanceVideo> getBytedanceVideoShotInfo();
     List<ETLReportBytedanceVideo> getBytedanceVideoPlanInfo();
 
-    void updateClipInfo(@Param("clipId")String clipId, @Param("clipName")String clipName, @Param("md5")String md5);
+    void updateClipInfo(@Param("clipId")String clipId, @Param("clipName")String clipName,@Param("teamLeader")String teamLeader, @Param("md5")String md5);
     void updateShotInfo(@Param("shotId")String shotId, @Param("shotName")String shotName, @Param("md5")String md5);
     void updatePlanInfo(@Param("planId")String planId, @Param("planName")String planName, @Param("md5")String md5);
 
     void updateTeamLeader(@Param("date") String date);
     void updateDesignTeamLeader(@Param("date") String date);
+
+    String getLeaderNameById(@Param("clipId") String clipId);
 }

+ 14 - 6
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/mapper/xml/ETLReportBytedanceVideoMapper.xml

@@ -202,7 +202,11 @@
         </foreach>
     </insert>
     <update id="updateClipInfo">
-        update etl_report_bytedance_video set clip_id=#{clipId},clip_name=#{clipName} where md5=#{md5}
+        update etl_report_bytedance_video set
+         clip_id=#{clipId},
+         clip_name=#{clipName},
+         team_leader = #{teamLeader}
+         where md5=#{md5}
     </update>
     <update id="updateShotInfo">
         update etl_report_bytedance_video set shot_id=#{shotId},shot_name=#{shotName} where md5=#{md5}
@@ -355,14 +359,15 @@ limit 1
             resultType="cn.com.ctop.toutiao.modules.link.entity.ETLReportBytedanceVideo">
         SELECT DISTINCT
             video.md5 as 'md5',
-            cmi.clip_id as 'clip_id',
-            u.realname as 'clip_name'
+            cmi.clip_id as 'clipId',
+            u.realname as 'clipName',
+            cmi.leader_name as 'teamLeader'
         FROM
             etl_report_bytedance_video video
-                LEFT JOIN ctop_material_ascription cmi ON video.md5 = cmi.material_id
-                 LEFT JOIN sys_user u ON u.id = cmi.clip_id
+            LEFT JOIN ctop_material_ascription cmi ON video.md5 = cmi.material_id
+            LEFT JOIN sys_user u ON u.id = cmi.clip_id
         WHERE video.stat_datetime &gt;= #{date}
-           and cmi.clip_id IS NOT NULL
+          and cmi.clip_id IS NOT NULL
           AND video.clip_id = ''
           AND u.realname IS NOT NULL
 
@@ -397,4 +402,7 @@ limit 1
           AND video.plan_id = ''
           AND u.realname IS NOT NULL
     </select>
+    <select id="getLeaderNameById" resultType="java.lang.String">
+         SELECT leader_name  from  sys_user WHERE id =#{clipId}
+    </select>
 </mapper>

+ 35 - 27
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/service/impl/ETLReportBytedanceVideoServiceImpl.java

@@ -1,11 +1,11 @@
 package cn.com.ctop.toutiao.modules.link.service.impl;
 
+import cn.com.ctop.common.module.utils.Check;
 import cn.com.ctop.toutiao.modules.link.entity.BaseReportBytedanceVideoDaily;
 import cn.com.ctop.toutiao.modules.link.entity.ETLReportBytedanceVideo;
 import cn.com.ctop.toutiao.modules.link.mapper.ETLReportBytedanceVideoMapper;
 import cn.com.ctop.toutiao.modules.link.service.IBaseReportBytedanceVideoDailyService;
 import cn.com.ctop.toutiao.modules.link.service.IETLReportBytedanceVideoService;
-import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import lombok.extern.slf4j.Slf4j;
@@ -30,32 +30,32 @@ public class ETLReportBytedanceVideoServiceImpl extends ServiceImpl<ETLReportByt
 
     @Override
     public List<BaseReportBytedanceVideoDaily> listByStatDate(String statDate) {
-        QueryWrapper<BaseReportBytedanceVideoDaily>  queryWrapper= new QueryWrapper<>();
-        queryWrapper.eq("stat_datetime",statDate);
+        QueryWrapper<BaseReportBytedanceVideoDaily> queryWrapper = new QueryWrapper<>();
+        queryWrapper.eq("stat_datetime", statDate);
         return baseReportBytedanceVideoDailyService.list(queryWrapper);
     }
 
     @Override
     public void cleanETL(String statDate) {
         List<ETLReportBytedanceVideo> etlReportBytedanceVideos = etlReportBytedanceVideoMapper.queryETLData(statDate);
-        System.out.println(statDate+":"+etlReportBytedanceVideos.size());
-        if(etlReportBytedanceVideos.isEmpty()){
-            log.warn("{},头条etl报表清洗异常,base数据源为空",statDate);
+        System.out.println(statDate + ":" + etlReportBytedanceVideos.size());
+        if (etlReportBytedanceVideos.isEmpty()) {
+            log.warn("{},头条etl报表清洗异常,base数据源为空", statDate);
             return;
         }
         boolean result = this.saveBatch(etlReportBytedanceVideos);
-        if(result){
-            log.info("{},头条etl报表清洗完成,数量:{}",statDate,etlReportBytedanceVideos.size());
-        } else{
-            log.error("{},头条etl报表清洗失败,请重试",statDate);
+        if (result) {
+            log.info("{},头条etl报表清洗完成,数量:{}", statDate, etlReportBytedanceVideos.size());
+        } else {
+            log.error("{},头条etl报表清洗失败,请重试", statDate);
         }
     }
 
     @Override
-    public void cleanData(String date){
+    public void cleanData(String date) {
         List<ETLReportBytedanceVideo> videos = etlReportBytedanceVideoMapper.getDuplicateVideoInfo(date);
-        if(null!=videos&&!videos.isEmpty()){
-            videos.forEach(video->{
+        if (null != videos && !videos.isEmpty()) {
+            videos.forEach(video -> {
                 etlReportBytedanceVideoMapper.deleteDuplicateVideoInfo(video);
             });
         }
@@ -63,32 +63,40 @@ public class ETLReportBytedanceVideoServiceImpl extends ServiceImpl<ETLReportByt
 
     @Override
     public void etlBytedanceVideoClipIdJob(String date) {
-        List <ETLReportBytedanceVideo> clipInfos = etlReportBytedanceVideoMapper.getBytedanceVideoClipInfo(date);
-        if(null!=clipInfos&&!clipInfos.isEmpty()){
-            clipInfos.forEach(clipInfo->{
-                etlReportBytedanceVideoMapper.updateClipInfo(clipInfo.getClipId(),clipInfo.getClipName(),clipInfo.getMd5());
+        List<ETLReportBytedanceVideo> clipInfos = etlReportBytedanceVideoMapper.getBytedanceVideoClipInfo(date);
+        if (null != clipInfos && !clipInfos.isEmpty()) {
+            clipInfos.forEach(clipInfo -> {
+
+                String teamLeader = clipInfo.getTeamLeader();
+                if (Check.isNull(teamLeader)) {
+                    String leaderName = etlReportBytedanceVideoMapper.getLeaderNameById(clipInfo.getClipId());
+                    clipInfo.setTeamLeader(leaderName);
+                }
+
+                etlReportBytedanceVideoMapper.updateClipInfo(clipInfo.getClipId(), clipInfo.getClipName(), clipInfo.getTeamLeader(), clipInfo.getMd5());
+
             });
         }
-        etlReportBytedanceVideoMapper.updateDesignTeamLeader(date);
-        etlReportBytedanceVideoMapper.updateTeamLeader(date);
+      /*  etlReportBytedanceVideoMapper.updateDesignTeamLeader(date);
+        etlReportBytedanceVideoMapper.updateTeamLeader(date);*/
     }
 
     @Override
     public void etlBytedanceVideoPlanInfoJob() {
-        List <ETLReportBytedanceVideo> planInfos = etlReportBytedanceVideoMapper.getBytedanceVideoPlanInfo();
-        if(null!=planInfos&&!planInfos.isEmpty()){
-            planInfos.forEach(planInfo->{
-                etlReportBytedanceVideoMapper.updatePlanInfo(planInfo.getPlanId(),planInfo.getPlanName(),planInfo.getMd5());
+        List<ETLReportBytedanceVideo> planInfos = etlReportBytedanceVideoMapper.getBytedanceVideoPlanInfo();
+        if (null != planInfos && !planInfos.isEmpty()) {
+            planInfos.forEach(planInfo -> {
+                etlReportBytedanceVideoMapper.updatePlanInfo(planInfo.getPlanId(), planInfo.getPlanName(), planInfo.getMd5());
             });
         }
     }
 
     @Override
     public void etlBytedanceVideoShotInfoJob() {
-        List <ETLReportBytedanceVideo> shotInfos = etlReportBytedanceVideoMapper.getBytedanceVideoShotInfo();
-        if(null!=shotInfos&&!shotInfos.isEmpty()){
-            shotInfos.forEach(shotInfo->{
-                etlReportBytedanceVideoMapper.updateShotInfo(shotInfo.getShotId(),shotInfo.getShotName(),shotInfo.getMd5());
+        List<ETLReportBytedanceVideo> shotInfos = etlReportBytedanceVideoMapper.getBytedanceVideoShotInfo();
+        if (null != shotInfos && !shotInfos.isEmpty()) {
+            shotInfos.forEach(shotInfo -> {
+                etlReportBytedanceVideoMapper.updateShotInfo(shotInfo.getShotId(), shotInfo.getShotName(), shotInfo.getMd5());
             });
         }
     }