Kaynağa Gözat

修改素材清洗逻辑

yumeng 3 yıl önce
ebeveyn
işleme
1dec4c3155

+ 4 - 7
module-job-kuaishou/src/main/java/cn/com/ctop/job/kuaishou/handler/KuaishouEtlVideoJob.java

@@ -60,18 +60,16 @@ public class KuaishouEtlVideoJob {
     }
 
 
-
     /**
      * 清洗快手视频归属信息表
      */
     @XxlJob("etlFinalKuaishouVideoInfo")
     public void etlKuaishouVideoInfo() {
-        Date startDate = DateUtils.addDay(new Date(), -2);
-        for (int i = 0; i <= 2; i++) {
-            videoEtlInfoService.etlKuaishouVideoInfo(DateUtils.formatDate(DateUtils.addDay(startDate, i)));
-        }
-    }
+        String nowDate = DateUtils.getNowDate("yyyy-MM-dd");
+        String anotherDay = DateUtils.getAnotherDay("yyyy-MM-dd", nowDate, -1);
+        videoEtlInfoService.etlKuaishouVideoInfo(anotherDay);
 
+    }
 
     @XxlJob("cleanKuaishouMaterialReport")
     public void cleanKuaishouMaterialReport() {
@@ -89,7 +87,6 @@ public class KuaishouEtlVideoJob {
     }
 
 
-
     /**
      * 快手清洗素材账户报表信息
      */

+ 1 - 1
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/mapper/KuaiShouVideoGetMapper.java

@@ -58,7 +58,7 @@ public interface KuaiShouVideoGetMapper extends BaseMapper<KuaiShouVideoGet> {
 
     JSONObject getVideoByPhotoIdAndAccountId(@Param("accountId") Long accountId,@Param("photoId") Long photoId);
 
-    List<JSONObject> getVideoListByDate(@Param("startDate") Date startDate,@Param("endDate") Date endDate);
+    List<JSONObject> getVideoListByDate(@Param("date") String date);
 
     List<String> getDateList();
 }

+ 2 - 5
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/mapper/xml/KuaiShouVideoGetMapper.xml

@@ -356,13 +356,10 @@
      SELECT
      account_id as 'accountId',
      signature as 'signature',
-     photo_name as 'photoName',
-     url as 'url',
+     photo_url as 'url',
      cover_url as 'coverUrl',
      channel_type as 'channelType'
-    from ctop_kuaishou_video_get WHERE stat_date BETWEEN #{startDate} AND #{endDate}
-     group by signature
-
+    from ctop_kuaishou_report_daily_material WHERE stat_date = #{date} group by signature
     </select>
     <select id="getDateList" resultType="java.lang.String">
        SELECT

+ 2 - 2
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/impl/KuaiShouVideoGetServiceImpl.java

@@ -353,8 +353,8 @@ public class KuaiShouVideoGetServiceImpl extends ServiceImpl<KuaiShouVideoGetMap
     }
 
     @Override
-    public List<JSONObject> getVideoListByDate(Date startDate, Date endDate) {
-        return videoGetMapper.getVideoListByDate(startDate, endDate);
+    public List<JSONObject> getVideoListByDate(String date) {
+        return videoGetMapper.getVideoListByDate(date);
     }
 
     @Override

+ 4 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/material/mapper/EtlKuaishouVideoInfoMapper.java

@@ -20,4 +20,8 @@ public interface EtlKuaishouVideoInfoMapper extends BaseMapper<EtlKuaishouVideoI
     void replaceVideoInfo(@Param(value = "etlKuaishouVideoInfo") EtlKuaishouVideoInfo etlKuaishouVideoInfo);
 
     void addBatch(@Param("addList") List<EtlKuaishouVideoInfo> addList);
+
+    String getPhotoName(@Param("signature") String signature);
+
+    Integer getByMd5(@Param("signature") String signature);
 }

+ 6 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/material/mapper/xml/EtlKuaishouVideoInfoMapper.xml

@@ -89,4 +89,10 @@
     <select id="getVideoMinDate" resultType="java.lang.String">
         select frast_cost_time from application.material_farst_cost_time where signature = #{signature}
     </select>
+    <select id="getPhotoName" resultType="java.lang.String">
+        SELECT photo_name  from  ctop_etl_kuaishou_video_base_info  WHERE signature = #{signature} limit 1
+    </select>
+    <select id="getByMd5" resultType="java.lang.Integer">
+        SELECT count(1)  from  ctop_etl_kuaishou_video_info WHERE video_code = #{signature} AND state_date is not null;
+    </select>
 </mapper>

+ 11 - 5
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/material/service/impl/EtlKuaishouVideoInfoServiceImpl.java

@@ -49,14 +49,12 @@ public class EtlKuaishouVideoInfoServiceImpl extends ServiceImpl<EtlKuaishouVide
     private IMaterialInfoService materialInfoService;
     @Autowired
     private IUserCompanyService companyService;
-    ExecutorService executorService = Executors.newFixedThreadPool(5);
+    ExecutorService executorService = Executors.newFixedThreadPool(15);
     ExecutorService videoExecutorService = Executors.newFixedThreadPool(10);
 
     @Override
     public void etlKuaishouVideoInfo(String date) {
-        Date startDate = DateUtils.getStartDate(date);
-        Date endDate = DateUtils.getEndDate(date);
-        List<JSONObject> videoList = videoGetService.getVideoListByDate(startDate, endDate);
+        List<JSONObject> videoList = videoGetService.getVideoListByDate(date);
         if (Check.isNull(videoList)) {
             return;
         }
@@ -75,6 +73,11 @@ public class EtlKuaishouVideoInfoServiceImpl extends ServiceImpl<EtlKuaishouVide
                     if (Check.isNull(signature)) {
                         continue;
                     }
+                    Integer count = etlKuaishouVideoInfoMapper.getByMd5(signature);
+                    if (count > 0) {
+                        continue;
+                    }
+
                     try {
                         //1:查询素材名称
                         EtlKuaishouVideoInfo etlKuaishouVideoInfo = new EtlKuaishouVideoInfo();
@@ -82,7 +85,9 @@ public class EtlKuaishouVideoInfoServiceImpl extends ServiceImpl<EtlKuaishouVide
                         if (null != materialInfo) {
                             etlKuaishouVideoInfo.setVideoName(materialInfo.getMaterialName());
                         } else {
-                            etlKuaishouVideoInfo.setVideoName(videoJson.getString("photoName"));
+                            String photoName = etlKuaishouVideoInfoMapper.getPhotoName(signature);
+
+                            etlKuaishouVideoInfo.setVideoName(photoName);
                         }
                         etlKuaishouVideoInfo.setVideoCode(signature);
                         etlKuaishouVideoInfo.setUrl(videoJson.getString("url"));
@@ -186,6 +191,7 @@ public class EtlKuaishouVideoInfoServiceImpl extends ServiceImpl<EtlKuaishouVide
                 }
                 if (!Check.isNull(addList)) {
                     etlKuaishouVideoInfoMapper.addBatch(addList);
+                    log.info("视频数据清洗添加成功,日期:{}。条数:{}", date, addList.size());
 
                 }
             });