Browse Source

头条视频报表url异常清洗 优化

yangzian 3 năm trước cách đây
mục cha
commit
8cc2b0a3cf

+ 14 - 1
module-job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedanceEtlVideoJob.java

@@ -1,5 +1,7 @@
 package cn.com.ctop.job.bytedance.handler;
 package cn.com.ctop.job.bytedance.handler;
 
 
+import cn.com.ctop.common.module.entity.UserAllocation;
+import cn.com.ctop.common.module.service.IUserAllocationService;
 import cn.com.ctop.toutiao.modules.link.service.IETLReportBytedanceVideoService;
 import cn.com.ctop.toutiao.modules.link.service.IETLReportBytedanceVideoService;
 import cn.com.ctop.toutiao.modules.report.service.IBytedanceVideoEtlInfoService;
 import cn.com.ctop.toutiao.modules.report.service.IBytedanceVideoEtlInfoService;
 import cn.com.ctop.toutiao.modules.report.service.IEtlBytedanceReportVideoDailyService;
 import cn.com.ctop.toutiao.modules.report.service.IEtlBytedanceReportVideoDailyService;
@@ -8,7 +10,9 @@ import org.jeecg.common.util.DateUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 import org.springframework.stereotype.Component;
 
 
+import java.text.SimpleDateFormat;
 import java.util.Date;
 import java.util.Date;
+import java.util.List;
 
 
 @Component
 @Component
 public class BytedanceEtlVideoJob {
 public class BytedanceEtlVideoJob {
@@ -19,6 +23,10 @@ public class BytedanceEtlVideoJob {
     @Autowired
     @Autowired
     private IEtlBytedanceReportVideoDailyService etlBytedanceReportVideoDailyService;
     private IEtlBytedanceReportVideoDailyService etlBytedanceReportVideoDailyService;
 
 
+    @Autowired
+    private IUserAllocationService userAllocationService;
+
+
     @XxlJob("bytedanceEtlVideoJob")
     @XxlJob("bytedanceEtlVideoJob")
     public void execute() throws Exception {
     public void execute() throws Exception {
         Date startDate = DateUtils.addDay(new Date(),-2);
         Date startDate = DateUtils.addDay(new Date(),-2);
@@ -45,7 +53,12 @@ public class BytedanceEtlVideoJob {
      */
      */
     @XxlJob("updBytedanceVideoJob")
     @XxlJob("updBytedanceVideoJob")
     public void updBytedanceVideoJob() throws Exception {
     public void updBytedanceVideoJob() throws Exception {
-        etlBytedanceReportVideoDailyService.updBytedanceVideo();
+        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
+        String timeStr = sdf.format(new Date());
+        List<UserAllocation> userAllocationList = userAllocationService.list();
+        for (UserAllocation userAllocation : userAllocationList) {
+            etlBytedanceReportVideoDailyService.updBytedanceVideo(userAllocation.getAccountId(),userAllocation.getProjectId(),timeStr);
+        }
     }
     }
 
 
     @Autowired
     @Autowired

+ 1 - 1
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/EtlBytedanceReportVideoDailyMapper.java

@@ -17,7 +17,7 @@ import java.util.Map;
  */
  */
 public interface EtlBytedanceReportVideoDailyMapper extends BaseMapper<EtlBytedanceReportVideoDaily> {
 public interface EtlBytedanceReportVideoDailyMapper extends BaseMapper<EtlBytedanceReportVideoDaily> {
 
 
-    List<Map<String,Object>> getBytedanceVideoInfo();
+    List<Map<String,Object>> getBytedanceVideoInfo(@Param("accountId") Long accountId,@Param("projectId") Long projectId,@Param("timeStr") String timeStr);
 
 
     void updBytedanceVideoByMd5(String md5,String coverUrl,String url);
     void updBytedanceVideoByMd5(String md5,String coverUrl,String url);
 
 

+ 3 - 2
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/xml/EtlBytedanceReportVideoDailyMapper.xml

@@ -47,12 +47,13 @@
             count( 1 ) AS total
             count( 1 ) AS total
         FROM
         FROM
             etl_report_bytedance_video a
             etl_report_bytedance_video a
-                LEFT JOIN ( SELECT DISTINCT signature, poster_url, video_url FROM ctop_bytedance_video_info ) b ON b.signature = a.md5
-                LEFT JOIN ( SELECT DISTINCT CODE, url FROM ctop_material_info ) c ON c.CODE = a.md5
+                LEFT JOIN ( SELECT DISTINCT signature, poster_url, video_url FROM ctop_bytedance_video_info where account_id = #{accountId}) b ON b.signature = a.md5
+                LEFT JOIN ( SELECT DISTINCT CODE, url FROM ctop_material_info where project_id = #{projectId}) c ON c.CODE = a.md5
         WHERE
         WHERE
             a.url = ''
             a.url = ''
           AND a.toutiao_url = ''
           AND a.toutiao_url = ''
           AND a.md5 != ''
           AND a.md5 != ''
+          AND a.stat_datetime = #{timeStr}
         GROUP BY
         GROUP BY
             a.md5
             a.md5
         ORDER BY
         ORDER BY

+ 1 - 1
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/IEtlBytedanceReportVideoDailyService.java

@@ -13,7 +13,7 @@ import org.jeecg.common.api.vo.Result;
 public interface IEtlBytedanceReportVideoDailyService extends IService<EtlBytedanceReportVideoDaily> {
 public interface IEtlBytedanceReportVideoDailyService extends IService<EtlBytedanceReportVideoDaily> {
 
 
 
 
-    Result updBytedanceVideo() throws Exception;
+    Result updBytedanceVideo(Long accountId,Long projectId,String timeStr) throws Exception;
 
 
 
 
     void etlBytedanceVideo() throws Exception;
     void etlBytedanceVideo() throws Exception;

+ 2 - 2
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/EtlBytedanceReportVideoDailyServiceImpl.java

@@ -42,9 +42,9 @@ public class EtlBytedanceReportVideoDailyServiceImpl extends ServiceImpl<EtlByte
     private EtlBytedanceReportVideoDailyMapper etlBytedanceReportVideoDailyMapper;
     private EtlBytedanceReportVideoDailyMapper etlBytedanceReportVideoDailyMapper;
 
 
     @Override
     @Override
-    public Result updBytedanceVideo() throws Exception{
+    public Result updBytedanceVideo(Long accountId,Long projectId,String timeStr) throws Exception{
         CompletableFuture future = CompletableFuture.supplyAsync(()->{
         CompletableFuture future = CompletableFuture.supplyAsync(()->{
-           return etlBytedanceReportVideoDailyMapper.getBytedanceVideoInfo();
+           return etlBytedanceReportVideoDailyMapper.getBytedanceVideoInfo(accountId,projectId,timeStr);
         }).handle(new BiFunction<List<Map<String, Object>>, Throwable, Object>() {
         }).handle(new BiFunction<List<Map<String, Object>>, Throwable, Object>() {
             @Override
             @Override
             public Object apply(List<Map<String, Object>> list, Throwable throwable) {
             public Object apply(List<Map<String, Object>> list, Throwable throwable) {