Kaynağa Gözat

Merge branch 'ydev'

yangzian 3 yıl önce
ebeveyn
işleme
8be7f4d6cd

+ 2 - 2
module-job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/ByteDanceVideoReportJob.java

@@ -20,7 +20,7 @@ public class ByteDanceVideoReportJob {
     private IByteDanceVideoReportDailyService byteDanceVideoReportDailyService;
 
     // TODO 待删除
-    @XxlJob("byteDanceVideoReport")
+   /* @XxlJob("byteDanceVideoReport")
     public void execute() throws Exception {
         String getDate2 = DateUtils.formatDate(DateUtils.addDay(new Date(), -2));
         XxlJobHelper.log("头条视频报表清洗数据定时任务开始, 执行时间为:" + getDate2);
@@ -29,5 +29,5 @@ public class ByteDanceVideoReportJob {
         String getDate = DateUtils.formatDate(DateUtils.addDay(new Date(), -1));
         XxlJobHelper.log("头条视频报表清洗数据定时任务开始, 执行时间为:" + getDate);
         byteDanceVideoReportDailyService.videoInfoList(getDate, getDate);
-    }
+    }*/
 }

+ 15 - 1
module-job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedanceEtlVideoJob.java

@@ -1,6 +1,8 @@
 package cn.com.ctop.job.bytedance.handler;
 
+import cn.com.ctop.common.module.entity.CtopOauthToken;
 import cn.com.ctop.common.module.entity.UserAllocation;
+import cn.com.ctop.common.module.service.ICtopOauthTokenService;
 import cn.com.ctop.common.module.service.IUserAllocationService;
 import cn.com.ctop.toutiao.modules.link.service.IETLReportBytedanceVideoService;
 import cn.com.ctop.toutiao.modules.report.service.IBytedanceVideoEtlInfoService;
@@ -13,10 +15,15 @@ import org.springframework.stereotype.Component;
 import java.text.SimpleDateFormat;
 import java.util.Date;
 import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 
 @Component
 public class BytedanceEtlVideoJob {
 
+    static ExecutorService executorService = Executors.newFixedThreadPool(5);
+
+
     @Autowired
     private IBytedanceVideoEtlInfoService bytedanceVideoEtlInfoService;
 
@@ -26,6 +33,9 @@ public class BytedanceEtlVideoJob {
     @Autowired
     private IUserAllocationService userAllocationService;
 
+    @Autowired
+    private ICtopOauthTokenService tokenService;
+
 
     @XxlJob("bytedanceEtlVideoJob")
     public void execute() throws Exception {
@@ -97,6 +107,10 @@ public class BytedanceEtlVideoJob {
      */
     @XxlJob("formatBytedance")
     public void etlBytedanceVideoJob() throws Exception {
-        etlBytedanceReportVideoDailyService.etlBytedanceVideo();
+        //查询所有账户-头条
+        List<CtopOauthToken> tokens = tokenService.selectToutiaoToken();
+        tokens.forEach(token -> executorService.submit(() -> {
+            etlBytedanceReportVideoDailyService.etlBytedanceVideo(token.getAccountId());
+        }));
     }
 }

+ 1 - 1
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/EtlBytedanceReportVideoDailyMapper.java

@@ -22,7 +22,7 @@ public interface EtlBytedanceReportVideoDailyMapper extends BaseMapper<EtlByteda
     void updBytedanceVideoByMd5(String md5,String coverUrl,String url);
 
 
-    List<ETLReportBytedanceVideo> etlBytedanceVideoInfo(@Param(value = "date") String date);
+    List<ETLReportBytedanceVideo> etlBytedanceVideoInfo(@Param(value = "date") String date,@Param("accountId") Long accountId);
 
     void replaceEtlVideoInfo(@Param(value = "videoInfo") ETLReportBytedanceVideo videoInfo);
 

+ 1 - 1
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/xml/ByteDanceVideoReportDailyMapper.xml

@@ -1095,7 +1095,7 @@
         from ctop_bytedance_report_material_daily a
         left join ctop_user_allocation b on a.account_id = b.account_id
         left join user_company c on b.user_id = c.user_id
-        left join (select signature,video_url,material_id from ctop_bytedance_video_info group by material_id) d on
+        left join (select signature,video_url,material_id from ctop_bytedance_video_info where account_id = #{accountId} group by material_id) d on
         a.material_id = d.material_id
         where c.company_id = #{companyId}
         and stat_datetime &gt;= #{startDate}

+ 6 - 2
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/xml/EtlBytedanceReportVideoDailyMapper.xml

@@ -69,6 +69,7 @@
                           (case when cbvi.signature is null then '' else cbvi.signature end)'md5',
                           report.material_id 'material_id',
                           cp.project_name 'project_name',
+                          cp.id as 'project_id',
                           case when etlVideo.clip_name is null then '' else etlVideo.clip_name end 'clip_name',
                           case when etlVideo.plan_name is null then '' else etlVideo.plan_name end 'plan_name',
                           case when etlVideo.shot_name is null then '' else etlVideo.shot_name end 'shot_name',
@@ -119,6 +120,7 @@
            left join ctop_project cp on cp.id = cua.project_id
   where 1 = 1
     and report.stat_datetime = #{date}
+    and report.account_id = #{accountId}
   group by report.material_id,report.account_id;
     </select>
     <select id="replaceEtlVideoInfo">
@@ -128,7 +130,8 @@
             material_name,
             md5,
             material_id,
-            project_name ,
+            project_name,
+            project_id,
             clip_name,
             plan_name,
             shot_name,
@@ -176,7 +179,8 @@
             #{videoInfo.materialName},
             #{videoInfo.md5},
             #{videoInfo.materialId},
-            #{videoInfo.projectName },
+            #{videoInfo.projectName},
+            #{videoInfo.projectId},
             #{videoInfo.clipName},
             #{videoInfo.planName},
             #{videoInfo.shotName},

+ 1 - 1
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/IEtlBytedanceReportVideoDailyService.java

@@ -16,5 +16,5 @@ public interface IEtlBytedanceReportVideoDailyService extends IService<EtlByteda
     Result updBytedanceVideo(Long accountId,Long projectId,String timeStr) throws Exception;
 
 
-    void etlBytedanceVideo() throws Exception;
+    void etlBytedanceVideo(Long accountId);
 }

+ 23 - 15
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/ByteDanceVideoReportDailyServiceImpl.java

@@ -1,6 +1,8 @@
 package cn.com.ctop.toutiao.modules.report.service.impl;
 
 import cn.com.ctop.common.module.constant.CtopRoleCodeConstant;
+import cn.com.ctop.common.module.entity.CtopOauthToken;
+import cn.com.ctop.common.module.mapper.CtopOauthTokenMapper;
 import cn.com.ctop.toutiao.modules.report.DTO.AccountVideoDTO;
 import cn.com.ctop.toutiao.modules.report.entity.ByteDanceVideoReportDaily;
 import cn.com.ctop.toutiao.modules.report.mapper.ByteDanceVideoReportDailyMapper;
@@ -35,6 +37,9 @@ public class ByteDanceVideoReportDailyServiceImpl extends ServiceImpl<ByteDanceV
     @Resource
     private ByteDanceVideoReportDailyMapper byteDanceVideoReportDailyMapper;
 
+    @Resource
+    private CtopOauthTokenMapper ctopOauthTokenMapper;
+
     @Override
     public void videoInfoList(String startDate, String endDate){
         //华南
@@ -51,23 +56,26 @@ public class ByteDanceVideoReportDailyServiceImpl extends ServiceImpl<ByteDanceV
         wrapper.le("stat_datetime",endDate);
         wrapper.eq("company_id", companyId);
         byteDanceVideoReportDailyMapper.delete(wrapper);
-
-        List<ByteDanceVideoReportDaily> mList = byteDanceVideoReportDailyMapper.videoInfoList(startDate, endDate, companyId, null);
-        if(null!=mList&&!mList.isEmpty()){
-            List<ByteDanceVideoReportDaily> reports = new ArrayList<>();
-            for(ByteDanceVideoReportDaily report :mList){
-                if(null == report.getMaterialName()|| "".equals(report.getMaterialName().trim())){
-                    report.setMaterialName("-");
-                }
-                if(null == report.getProjectName()|| "".equals(report.getProjectName().trim())){
-                    report.setProjectName("-");
-                }
-                if(null == report.getAdvertiserName()|| "".equals(report.getAdvertiserName().trim())){
-                    report.setAdvertiserName("-");
+        //获取所有头条账户
+        List<CtopOauthToken> listToken = ctopOauthTokenMapper.selectToutiaoToken();
+        for (CtopOauthToken token : listToken) {
+            List<ByteDanceVideoReportDaily> mList = byteDanceVideoReportDailyMapper.videoInfoList(startDate, endDate, companyId, token.getAccountId());
+            if(null!=mList&&!mList.isEmpty()){
+                List<ByteDanceVideoReportDaily> reports = new ArrayList<>();
+                for(ByteDanceVideoReportDaily report :mList){
+                    if(null == report.getMaterialName()|| "".equals(report.getMaterialName().trim())){
+                        report.setMaterialName("-");
+                    }
+                    if(null == report.getProjectName()|| "".equals(report.getProjectName().trim())){
+                        report.setProjectName("-");
+                    }
+                    if(null == report.getAdvertiserName()|| "".equals(report.getAdvertiserName().trim())){
+                        report.setAdvertiserName("-");
+                    }
+                    reports.add(report);
                 }
-                reports.add(report);
+                byteDanceVideoReportDailyMapper.insertBatch(reports);
             }
-            byteDanceVideoReportDailyMapper.insertBatch(reports);
         }
     }
 

+ 2 - 2
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/EtlBytedanceReportVideoDailyServiceImpl.java

@@ -66,9 +66,9 @@ public class EtlBytedanceReportVideoDailyServiceImpl extends ServiceImpl<EtlByte
     static ExecutorService executorService = Executors.newFixedThreadPool(20);
     static CountDownLatch countDownLatch = null;
     @Override
-    public void etlBytedanceVideo()throws Exception {
+    public void etlBytedanceVideo(Long accountId){
         String date = DateUtils.formatDate(DateUtils.addDay(new Date(), -1));
-        List<ETLReportBytedanceVideo> list = etlBytedanceReportVideoDailyMapper.etlBytedanceVideoInfo(date);
+        List<ETLReportBytedanceVideo> list = etlBytedanceReportVideoDailyMapper.etlBytedanceVideoInfo(date,accountId);
         if (null != list && !list.isEmpty()) {
             countDownLatch = new CountDownLatch(list.size());
             list.forEach(videoInfo -> executorService.submit(() -> {