Sfoglia il codice sorgente

Merge remote-tracking branch 'origin/master'

yumeng 4 anni fa
parent
commit
a3465945ff
38 ha cambiato i file con 1046 aggiunte e 333 eliminazioni
  1. 2 0
      jeecg-boot-module-system/src/main/java/org/jeecg/JeecgApplication.java
  2. 1 0
      jeecg-boot-module-system/src/main/java/org/jeecg/config/ShiroConfig.java
  3. 25 0
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/TestController.java
  4. 10 10
      jeecg-boot-module-system/src/main/resources/application-wps.yml
  5. 90 56
      jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java
  6. 3 0
      module-common/src/main/java/cn/com/ctop/common/module/service/ISysUserService.java
  7. 6 0
      module-common/src/main/java/cn/com/ctop/common/module/service/impl/SysUserServiceImpl.java
  8. 146 146
      module-common/src/main/java/cn/com/ctop/common/module/utils/JavacvUtil.java
  9. 21 1
      module-job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/ByteDanceRefreshTokenJob.java
  10. 73 0
      module-job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedanceCheckAccountReportJob.java
  11. 19 1
      module-job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedanceEtlVideoJob.java
  12. 1 1
      module-job-kuaishou/src/main/java/cn/com/ctop/job/kuaishou/handler/KuaishouProjectVideoGetJob.java
  13. 1 1
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/controller/KuaishouProjectVideoGetController.java
  14. 2 2
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/mapper/xml/KuaiShouVideoGetMapper.xml
  15. 1 1
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/IKuaishouProjectVideoGetService.java
  16. 5 3
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/impl/KuaishouProjectVideoGetServiceImpl.java
  17. 10 4
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/material/service/impl/EtlKuaishouVideoInfoServiceImpl.java
  18. 8 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/mapper/ETLReportBytedanceVideoMapper.java
  19. 55 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/mapper/xml/ETLReportBytedanceVideoMapper.xml
  20. 6 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/service/IETLReportBytedanceVideoService.java
  21. 30 51
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/service/impl/ETLReportBytedanceVideoServiceImpl.java
  22. 166 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/controller/BytedanceCheckReportTaskInfoController.java
  23. 67 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/entity/BytedanceCheckReportTaskInfo.java
  24. 7 4
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/BytedanceCampaignDailyReportMapper.java
  25. 14 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/BytedanceCheckReportTaskInfoMapper.java
  26. 5 4
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/BytedanceReportMaterialDailyMapper.java
  27. 12 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/xml/BytedanceCampaignDailyReportMapper.xml
  28. 5 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/xml/BytedanceCheckReportTaskInfoMapper.xml
  29. 24 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/xml/BytedanceReportMaterialDailyMapper.xml
  30. 2 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/IBytedanceAdvertiserDailyReportService.java
  31. 2 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/IBytedanceCampaignDailyReportService.java
  32. 24 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/IBytedanceCheckReportTaskInfoService.java
  33. 5 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/IBytedanceReportMaterialDailyService.java
  34. 11 39
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/BytedanceAdvertiserDailyReportServiceImpl.java
  35. 13 7
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/BytedanceCampaignDailyReportServiceImpl.java
  36. 159 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/BytedanceCheckReportTaskInfoServiceImpl.java
  37. 13 2
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/BytedanceReportMaterialDailyServiceImpl.java
  38. 2 0
      pom.xml

+ 2 - 0
jeecg-boot-module-system/src/main/java/org/jeecg/JeecgApplication.java

@@ -7,6 +7,8 @@ import org.apache.tomcat.util.scan.StandardJarScanner;
 import org.jeecg.common.constant.AccountReportConstants;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.data.mongo.MongoRepositoriesAutoConfiguration;
+import org.springframework.boot.autoconfigure.data.redis.RedisRepositoriesAutoConfiguration;
 import org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactory;
 import org.springframework.context.ConfigurableApplicationContext;
 import org.springframework.context.annotation.Bean;

+ 1 - 0
jeecg-boot-module-system/src/main/java/org/jeecg/config/ShiroConfig.java

@@ -103,6 +103,7 @@ public class ShiroConfig {
         filterChainDefinitionMap.put("/ctop/byteDance/homePage/report/*", "anon");
         filterChainDefinitionMap.put("/ctop/material/top/cost", "anon");
         filterChainDefinitionMap.put("/dimension/report/*", "anon");
+        filterChainDefinitionMap.put("/userAllocation/updateAuthName", "anon");
 
         filterChainDefinitionMap.put("/v1/**", "anon");
 

+ 25 - 0
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/TestController.java

@@ -29,6 +29,7 @@ import cn.com.ctop.manage.modules.material.service.IMaterialUploadService;
 import cn.com.ctop.toutiao.modules.link.service.IETLReportBytedanceVideoService;
 import cn.com.ctop.toutiao.modules.material.service.IByteDanceAdvertiserDataService;
 import cn.com.ctop.toutiao.modules.material.service.IBytedanceEffectVideoInfoService;
+import cn.com.ctop.toutiao.modules.report.entity.BytedanceCheckReportTaskInfo;
 import cn.com.ctop.toutiao.modules.report.service.*;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
@@ -1541,4 +1542,28 @@ public class TestController {
         ResultMapUtils.setResultMap(result, StatusCode.COMMON_SUCCESS);
         return result;
     }
+    @Autowired
+    private IBytedanceCheckReportTaskInfoService checkReportTaskInfoService;
+
+    @GetMapping("testInitCheckDataTask")
+    public Map<String, Object> testInitCheckDataTask(String date) throws ParseException {
+        Map<String, Object> result = new HashMap<>();
+        checkReportTaskInfoService.initTask(date);
+        ResultMapUtils.setResultMap(result, StatusCode.COMMON_SUCCESS);
+        return result;
+    }
+
+
+    @GetMapping("testCheckCost")
+    public Map<String, Object> testCheckCost(String date) throws ParseException {
+        Map<String, Object> result = new HashMap<>();
+        List<BytedanceCheckReportTaskInfo>taskInfos = checkReportTaskInfoService.getListByParams(null,"2021-06-01",0);
+        if(null!=taskInfos&&!taskInfos.isEmpty()){
+            for (BytedanceCheckReportTaskInfo taskInfo:taskInfos) {
+                checkReportTaskInfoService.checkCost(taskInfo);
+            }
+        }
+        ResultMapUtils.setResultMap(result, StatusCode.COMMON_SUCCESS);
+        return result;
+    }
 }

+ 10 - 10
jeecg-boot-module-system/src/main/resources/application-wps.yml

@@ -95,21 +95,21 @@ spring:
         # 通过connectProperties属性来打开mergeSql功能;慢SQL记录
         connectionProperties: druid.stat.mergeSql\=true;druid.stat.slowSqlMillis\=5000
       datasource:
-        master:
-          url: jdbc:mysql://139.186.27.96:4000/jiaoyang?characterEncoding=UTF-8&useUnicode=true&useSSL=false
-          username: data
-          password: hcst@2021
-          driver-class-name: com.mysql.jdbc.Driver
+#        master:
+#          url: jdbc:mysql://139.186.27.96:4000/jiaoyang?characterEncoding=UTF-8&useUnicode=true&useSSL=false
+#          username: data
+#          password: hcst@2021
+#          driver-class-name: com.mysql.jdbc.Driver
 #        master:
 #          url: jdbc:mysql://139.186.151.174:3306/jeecg-boot?characterEncoding=UTF-8&useUnicode=true
 #          username: hcst
 #          password: hcst@2020
 #          driver-class-name: com.mysql.jdbc.Driver
-#        master:
-#          url: jdbc:mysql://139.186.27.96:4000/jeecg-boot?characterEncoding=UTF-8&useUnicode=true&useSSL=false
-#          username: data
-#          password: hcst@2021
-#          driver-class-name: com.mysql.jdbc.Driver
+        master:
+          url: jdbc:mysql://139.186.27.96:4000/jeecg-boot?characterEncoding=UTF-8&useUnicode=true&useSSL=false
+          username: data
+          password: hcst@2021
+          driver-class-name: com.mysql.jdbc.Driver
   #redis 配置
   redis:
     database: 0

+ 90 - 56
jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java

@@ -9,13 +9,14 @@ import cn.com.ctop.kuaishou.modules.batch.service.IKuaishouInterfaceService;
 import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouWebInterfaceService;
 import cn.com.ctop.kuaishou.modules.report.mapper.EtlKuaishouAccountMaterialReportDailyMapper;
 import cn.com.ctop.kuaishou.modules.report.service.IKuaishouReportDailyImageService;
-import cn.com.ctop.oa.modules.service.IAppchatCreateService;
 import cn.com.ctop.oa.modules.service.IWechatNoListService;
 import cn.com.ctop.oa.modules.service.IWechatUserListService;
 import cn.com.ctop.toutiao.modules.link.service.IETLReportBytedanceVideoService;
 import cn.com.ctop.toutiao.modules.material.service.IByteDanceAdvertiserDataService;
 import cn.com.ctop.toutiao.modules.material.service.IByteDanceCampaignService;
+import cn.com.ctop.toutiao.modules.report.entity.BytedanceCheckReportTaskInfo;
 import cn.com.ctop.toutiao.modules.report.service.*;
+import com.xxl.job.core.context.XxlJobHelper;
 import lombok.extern.slf4j.Slf4j;
 import org.jeecg.common.util.DateUtils;
 import org.jeecg.common.util.RedisUtil;
@@ -41,19 +42,45 @@ import java.util.concurrent.Executors;
 @Slf4j
 public class SampleTest {
     @Autowired
+    private IBytedanceVideoEtlInfoService bytedanceVideoEtlInfoService;
+    @Autowired
+    IBytedanceVideoReportService bytedanceVideoReportService;
+    @Autowired
+    IETLReportBytedanceVideoService ietlReportBytedanceVideoService;
+    @Autowired
+    IKuaishouReportDailyImageService kuaishouReportDailyImageService;
+    @Autowired
+    IRuleByteDanceAccountService ruleByteDanceAccountService;
+    @Autowired
     private ICtopOauthTokenService tokenService;
     @Autowired
     private IBindAccountLoginService bindAccountLoginService;
     @Autowired
     private IKuaishouWebInterfaceService kuaishouWebInterfaceService;
     @Autowired
-    private IKuaishouInterfaceService kuaishouInterfaceService;
+    private IWechatUserListService wechatUserInfoService;
     @Autowired
-    private RedisUtil redisUtil;
+    private IByteDanceAdvertiserDataService advertiserDataService;
     @Autowired
-    private IAppchatCreateService appchatCreateService;
+    private IWechatNoListService wechatNoListService;
     @Autowired
-    private IWechatUserListService wechatUserInfoService;
+    private IETLReportBytedanceVideoService bytedanceVideoService;
+    @Autowired
+    private IByteDanceVideoReportDailyService videoReportDailyService;
+    @Resource
+    private EtlKuaishouAccountMaterialReportDailyMapper etlKuaishouAccountMaterialReportDailyMapper;
+    @Autowired
+    private IByteDanceCampaignService campaignService;
+    @Autowired
+    private IReportService reportService;
+    @Resource
+    private IBytedanceCheckReportTaskInfoService checkReportTaskInfoService;
+    @Autowired
+    private IEtlBytedanceReportVideoDailyService etlBytedanceReportVideoDailyService;
+    //    @Autowired
+    //    private IAppchatCreateService appchatCreateService;
+
+
 
 
     @Test
@@ -72,6 +99,7 @@ public class SampleTest {
 
         // appchatCreateService.createAppChat(null, null, null);
    //     appchatCreateService.getChatInfo(null);
+//        appchatCreateService.getChatInfo(null);
        /* redisUtil.set("11sdfhgjksdhfjksdhfjksdhfkjsjkfhkjsddhfjksdhfkjhsjkfhjkshfjksdhfjksdhfjkshdjkfhsjkdfhkjsd1","hahahaha",5);
         Object o = redisUtil.get("11sdfhgjksdhfjksdhfjksdhfkjsjkfhkjsddhfjksdhfkjhsjkfhjkshfjksdhfjksdhfjkshdjkfhsjkdfhkjsd1");
         System.err.println(o);*/
@@ -112,11 +140,11 @@ public class SampleTest {
         Date parse = dateFormat.parse("2021-04-02");
         kuaishouInterfaceService.getAdvertiserReportDaily(tokenByAccountId, parse, parse, null);*/
     }
-
-
     static ExecutorService executorService = null;
     static CountDownLatch countDownLatch = null;
 
+
+
     @Test
     public void deleteKuaishouComment() {
         Long start = System.currentTimeMillis();
@@ -143,11 +171,6 @@ public class SampleTest {
         }
     }
 
-    @Autowired
-    private IBytedanceReportService bytedanceReportService;
-
-    @Autowired
-    private IByteDanceAdvertiserDataService advertiserDataService;
 
     @Test
     public void loadBytedanceVideoReportData() {
@@ -155,10 +178,7 @@ public class SampleTest {
         advertiserDataService.getMaterialList(token);
     }
 
-    @Autowired
-    private IWechatNoListService wechatNoListService;
-    @Autowired
-    private IETLReportBytedanceVideoService bytedanceVideoService;
+
 
     @Test
     public void etlBytedanceVideoInfo() {
@@ -183,8 +203,6 @@ public class SampleTest {
     }
 
 
-    @Autowired
-    private IByteDanceVideoReportDailyService videoReportDailyService;
 
     @Test
     public void formatVideoReportData() {
@@ -195,17 +213,6 @@ public class SampleTest {
         }
     }
 
-    @Autowired
-    private IBytedanceVideoEtlInfoService bytedanceVideoEtlInfoService;
-    @Autowired
-    IBytedanceVideoReportService bytedanceVideoReportService;
-    @Autowired
-    IETLReportBytedanceVideoService ietlReportBytedanceVideoService;
-    @Autowired
-    IKuaishouReportDailyImageService kuaishouReportDailyImageService;
-    @Autowired
-    IRuleByteDanceAccountService ruleByteDanceAccountService;
-
     @Test
     public void loadBDAccoutData() {
         Date startDate = DateUtils.addDay(new Date(), -2000);
@@ -215,8 +222,6 @@ public class SampleTest {
         }
     }
 
-    @Resource
-    private EtlKuaishouAccountMaterialReportDailyMapper etlKuaishouAccountMaterialReportDailyMapper;
 
     @Test
     public void etlKuaishouVideoInfo() {
@@ -224,27 +229,20 @@ public class SampleTest {
     }
 
     @Autowired
-    private IByteDanceCampaignService campaignService;
+    private IBytedanceReportService bytedanceReportService;
 
     @Test
     public void testLoadBytedanceCampaign() {
-        List<CtopOauthToken> tokens = tokenService.selectToutiaoToken();
-        for (CtopOauthToken token : tokens) {
-            campaignService.getAdvertiserCampaign(token, null, null);
-        }
+        CtopOauthToken token = tokenService.getTokenByAccountId(1681669009084429L);
+        bytedanceReportService.bytedanceMaterialReport(token, "2021-05-01", "2021-05-06");
     }
 
-    @Autowired
-    private IEtlBytedanceReportVideoDailyService etlBytedanceReportVideoDailyService;
 
     @Test
     public void etlBytedanceVideoReportInfo() throws Exception {
         etlBytedanceReportVideoDailyService.etlBytedanceVideo();
     }
 
-    @Autowired
-    private IReportService reportService;
-
     @Test
     public void loadBytedanceAccountReport() {
         /**
@@ -253,20 +251,56 @@ public class SampleTest {
          * 1696734332089352
          * 1696734333251598
          */
-        CtopOauthToken token1 = tokenService.getTokenByAccountId(1692555408132110L);
-        reportService.getAdvertiserReport(token1, DateUtils.parseDate("2021-05-01", "yyyy-MM-dd"), DateUtils.parseDate("2021-05-12", "yyyy-MM-dd"), CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
-        CtopOauthToken token2 = tokenService.getTokenByAccountId(1692555631713287L);
-        reportService.getAdvertiserReport(token2, DateUtils.parseDate("2021-05-01", "yyyy-MM-dd"), DateUtils.parseDate("2021-05-12", "yyyy-MM-dd"), CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
-        CtopOauthToken token3 = tokenService.getTokenByAccountId(1696734332089352L);
-        reportService.getAdvertiserReport(token3, DateUtils.parseDate("2021-05-01", "yyyy-MM-dd"), DateUtils.parseDate("2021-05-12", "yyyy-MM-dd"), CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
-        CtopOauthToken token4 = tokenService.getTokenByAccountId(1696734333251598L);
-        reportService.getAdvertiserReport(token4, DateUtils.parseDate("2021-05-01", "yyyy-MM-dd"), DateUtils.parseDate("2021-05-12", "yyyy-MM-dd"), CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
-
-        CtopOauthToken token5 = tokenService.getTokenByAccountId(10129666L);
-        kuaishouInterfaceService.getAdvertiserReportDaily(token5, DateUtils.parseDate("2021-05-07", "yyyy-MM-dd"), DateUtils.parseDate("2021-05-11", "yyyy-MM-dd"), null);
-        CtopOauthToken token6 = tokenService.getTokenByAccountId(10097961L);
-        kuaishouInterfaceService.getAdvertiserReportDaily(token6, DateUtils.parseDate("2021-05-07", "yyyy-MM-dd"), DateUtils.parseDate("2021-05-11", "yyyy-MM-dd"), null);
-        CtopOauthToken token7 = tokenService.getTokenByAccountId(10098039L);
-        kuaishouInterfaceService.getAdvertiserReportDaily(token7, DateUtils.parseDate("2021-05-07", "yyyy-MM-dd"), DateUtils.parseDate("2021-05-11", "yyyy-MM-dd"), null);
+        CtopOauthToken token1 = tokenService.getTokenByAccountId(1693648201498637L);
+        reportService.getAdvertiserReport(token1, DateUtils.parseDate("2021-04-01", "yyyy-MM-dd"), DateUtils.parseDate("2021-04-12", "yyyy-MM-dd"), CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
+    }
+
+    @Test
+    public void checkCostData(){
+        String date = "2021-06-01";
+        for (int i=0;i<14;i++){
+            String getDate = DateUtils.addDay(date,i);
+            //1:初始化
+            checkReportTaskInfoService.initTask(getDate);
+            //2: checkCost
+            List<BytedanceCheckReportTaskInfo>taskInfos = checkReportTaskInfoService.getListByParams(null,getDate,0);
+            if(null!=taskInfos&&!taskInfos.isEmpty()){
+                for (BytedanceCheckReportTaskInfo task:taskInfos) {
+                    checkReportTaskInfoService.checkCost(task);
+                }
+                log.info("{}:数据获取完成",getDate);
+            }
+        }
+// 3:重复异常数据
+    }
+
+    @Test
+    public void refreshReportData()throws Exception{
+        //1:查询异常任务数据
+        List<BytedanceCheckReportTaskInfo>taskInfos = checkReportTaskInfoService.getListByParams(null,null,50001);
+        if(null!=taskInfos&&!taskInfos.isEmpty()){
+            countDownLatch = new CountDownLatch(taskInfos.size());
+            taskInfos.forEach(taskInfo -> {
+                try {
+                    checkReportTaskInfoService.reloadData(taskInfo);
+                    checkReportTaskInfoService.checkCost(taskInfo);
+                }catch (Exception e){
+                }finally {
+                    countDownLatch.countDown();
+                }
+            });
+            countDownLatch.await();
+            XxlJobHelper.log("{}数据更新完成");
+        }
     }
+
+    @Test
+    public void testfreshBytedanceUrl(){
+        etlReportBytedanceVideoService.etlBytedanceVideoClipIdJob();
+        etlReportBytedanceVideoService.etlBytedanceVideoPlanInfoJob();
+        etlReportBytedanceVideoService.etlBytedanceVideoShotInfoJob();
+    }
+
+    @Autowired
+    private IETLReportBytedanceVideoService etlReportBytedanceVideoService;
 }

+ 3 - 0
module-common/src/main/java/cn/com/ctop/common/module/service/ISysUserService.java

@@ -1,5 +1,6 @@
 package cn.com.ctop.common.module.service;
 
+import cn.com.ctop.common.module.entity.SysRole;
 import cn.com.ctop.common.module.entity.SysUser;
 import cn.com.ctop.common.module.model.SysUserSysDepartModel;
 import cn.com.ctop.common.module.vo.SysUserVo;
@@ -269,4 +270,6 @@ public interface ISysUserService extends IService<SysUser> {
     List<SysUser> queryPageList(SysUser user);
 
     Map<String, String> getUserMapByMap(Map<String, String> userIdMap);
+
+    String getRoleCodeByUserId(String userId);
 }

+ 6 - 0
module-common/src/main/java/cn/com/ctop/common/module/service/impl/SysUserServiceImpl.java

@@ -308,6 +308,12 @@ public class SysUserServiceImpl extends ServiceImpl<SysUserMapper, SysUser> impl
     }
 
     @Override
+    public String getRoleCodeByUserId(String userId) {
+        return sysRoleMapper.getRoleCodeByUserId(userId);
+    }
+
+
+    @Override
     public Map<String, String> getDepNamesByUserIds(List<String> userIds) {
         List<SysUserDepVo> list = this.baseMapper.getDepNamesByUserIds(userIds);
         Map<String, String> res = new HashMap<String, String>();

+ 146 - 146
module-common/src/main/java/cn/com/ctop/common/module/utils/JavacvUtil.java

@@ -1,146 +1,146 @@
-package cn.com.ctop.common.module.utils;
-
-import com.google.common.collect.Lists;
-import org.bytedeco.javacpp.opencv_core;
-import org.bytedeco.javacv.FFmpegFrameGrabber;
-import org.bytedeco.javacv.Frame;
-import org.bytedeco.javacv.Java2DFrameConverter;
-import org.opencv.core.CvType;
-import org.opencv.core.Mat;
-import org.opencv.core.MatOfByte;
-import org.opencv.imgcodecs.Imgcodecs;
-import org.opencv.imgproc.Imgproc;
-
-import javax.imageio.ImageIO;
-import java.awt.*;
-import java.awt.image.BufferedImage;
-import java.awt.image.DataBufferByte;
-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.List;
-
-public class JavacvUtil {
-    /**
-     * 视频文件指定时间段的帧截取
-     * @param file
-     * @param start
-     * @param end
-     */
-    public static List<File> videoIntercept(File file, Integer start, Integer end) {
-        Frame frame = null;
-        List<File> files = Lists.newArrayList();
-        FFmpegFrameGrabber fFmpegFrameGrabber = new FFmpegFrameGrabber(file);
-        String filePath = "D://video//images//";
-        String fileTargetName = "movie";
-        try {
-            fFmpegFrameGrabber.start();
-            int ftp = fFmpegFrameGrabber.getLengthInFrames();
-            for (int i=0 ; i < ftp ; i++){
-                if( i >= start && i <= end){
-                    frame = fFmpegFrameGrabber.grabImage();
-                    doExecuteFrame(frame, filePath, fileTargetName, i ,files);
-                }
-            }
-            fFmpegFrameGrabber.stop();
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-        return files;
-    }
-
-    public static void doExecuteFrame(Frame frame, String targetFilePath, String targetFileName, int index ,List<File> files) {
-        if ( frame == null || frame.image == null) {
-            return;
-        }
-        Java2DFrameConverter converter = new Java2DFrameConverter();
-        String imageMat = "jpg";
-        String fileName = targetFilePath + targetFileName + "_" + index + "." + imageMat;
-        BufferedImage bi = converter.getBufferedImage(frame);
-
-//        Mat img = Imgcodecs.imread("");
-//        Mat imgHSV = new Mat(img.rows(), img.cols(), CvType.CV_8UC3);
-//        Mat img2 = new Mat(img.rows(), img.cols(), CvType.CV_8UC3);
-
-        //转成HSV空间
-//        Imgproc.cvtColor(img, imgHSV, Imgproc.COLOR_BGR2HSV);
-//        //转回BGR空间
-//        Imgproc.cvtColor(imgHSV, img2, Imgproc.COLOR_HSV2BGR);
-
-        File output = new File(fileName);
-        files.add(output);
-        try{
-            ImageIO.write(bi, imageMat, output);
-        } catch (IOException e) {
-            e.printStackTrace();
-        }
-    }
-
-    public static void main(String[] args) {
-        List<File> files = videoIntercept(new File("C:\\Users\\宋英豪\\Desktop\\nfpHmswW.mp4"), 10, 100);
-    }
-
-    /**
-     * Mat转换成BufferedImage
-     *
-     * @param matrix
-     *            要转换的Mat
-     * @param fileExtension
-     *            格式为 ".jpg", ".png", etc
-     * @return
-     */
-    public static BufferedImage Mat2BufImg (Mat matrix, String fileExtension) {
-        // convert the matrix into a matrix of bytes appropriate for
-        // this file extension
-        MatOfByte mob = new MatOfByte();
-        Imgcodecs.imencode(fileExtension, matrix, mob);
-        // convert the "matrix of bytes" into a byte array
-        byte[] byteArray = mob.toArray();
-        BufferedImage bufImage = null;
-        try {
-            InputStream in = new ByteArrayInputStream(byteArray);
-            bufImage = ImageIO.read(in);
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-        return bufImage;
-    }
-
-    /**
-     * BufferedImage转换成Mat
-     *
-     * @param original
-     *            要转换的BufferedImage
-     * @param imgType
-     *            bufferedImage的类型 如 BufferedImage.TYPE_3BYTE_BGR
-     * @param matType
-     *            转换成mat的type 如 CvType.CV_8UC3
-     */
-    public static Mat BufImg2Mat (BufferedImage original, int imgType, int matType) {
-        if (original == null) {
-            throw new IllegalArgumentException("original == null");
-        }
-
-        // Don't convert if it already has correct type
-        if (original.getType() != imgType) {
-
-            // Create a buffered image
-            BufferedImage image = new BufferedImage(original.getWidth(), original.getHeight(), imgType);
-
-            // Draw the image onto the new buffer
-            Graphics2D g = image.createGraphics();
-            try {
-                g.setComposite(AlphaComposite.Src);
-                g.drawImage(original, 0, 0, null);
-            } finally {
-                g.dispose();
-            }
-        }
-
-        byte[] pixels = ((DataBufferByte) original.getRaster().getDataBuffer()).getData();
-        Mat mat = Mat.eye(original.getHeight(), original.getWidth(), matType);
-        mat.put(0, 0, pixels);
-        return mat;
-    }
-}
+//package cn.com.ctop.common.module.utils;
+//
+//import com.google.common.collect.Lists;
+//import org.bytedeco.javacpp.opencv_core;
+//import org.bytedeco.javacv.FFmpegFrameGrabber;
+//import org.bytedeco.javacv.Frame;
+//import org.bytedeco.javacv.Java2DFrameConverter;
+//import org.opencv.core.CvType;
+//import org.opencv.core.Mat;
+//import org.opencv.core.MatOfByte;
+//import org.opencv.imgcodecs.Imgcodecs;
+//import org.opencv.imgproc.Imgproc;
+//
+//import javax.imageio.ImageIO;
+//import java.awt.*;
+//import java.awt.image.BufferedImage;
+//import java.awt.image.DataBufferByte;
+//import java.io.ByteArrayInputStream;
+//import java.io.File;
+//import java.io.IOException;
+//import java.io.InputStream;
+//import java.util.List;
+//
+//public class JavacvUtil {
+//    /**
+//     * 视频文件指定时间段的帧截取
+//     * @param file
+//     * @param start
+//     * @param end
+//     */
+//    public static List<File> videoIntercept(File file, Integer start, Integer end) {
+//        Frame frame = null;
+//        List<File> files = Lists.newArrayList();
+//        FFmpegFrameGrabber fFmpegFrameGrabber = new FFmpegFrameGrabber(file);
+//        String filePath = "D://video//images//";
+//        String fileTargetName = "movie";
+//        try {
+//            fFmpegFrameGrabber.start();
+//            int ftp = fFmpegFrameGrabber.getLengthInFrames();
+//            for (int i=0 ; i < ftp ; i++){
+//                if( i >= start && i <= end){
+//                    frame = fFmpegFrameGrabber.grabImage();
+//                    doExecuteFrame(frame, filePath, fileTargetName, i ,files);
+//                }
+//            }
+//            fFmpegFrameGrabber.stop();
+//        } catch (Exception e) {
+//            e.printStackTrace();
+//        }
+//        return files;
+//    }
+//
+//    public static void doExecuteFrame(Frame frame, String targetFilePath, String targetFileName, int index ,List<File> files) {
+//        if ( frame == null || frame.image == null) {
+//            return;
+//        }
+//        Java2DFrameConverter converter = new Java2DFrameConverter();
+//        String imageMat = "jpg";
+//        String fileName = targetFilePath + targetFileName + "_" + index + "." + imageMat;
+//        BufferedImage bi = converter.getBufferedImage(frame);
+//
+////        Mat img = Imgcodecs.imread("");
+////        Mat imgHSV = new Mat(img.rows(), img.cols(), CvType.CV_8UC3);
+////        Mat img2 = new Mat(img.rows(), img.cols(), CvType.CV_8UC3);
+//
+//        //转成HSV空间
+////        Imgproc.cvtColor(img, imgHSV, Imgproc.COLOR_BGR2HSV);
+////        //转回BGR空间
+////        Imgproc.cvtColor(imgHSV, img2, Imgproc.COLOR_HSV2BGR);
+//
+//        File output = new File(fileName);
+//        files.add(output);
+//        try{
+//            ImageIO.write(bi, imageMat, output);
+//        } catch (IOException e) {
+//            e.printStackTrace();
+//        }
+//    }
+//
+//    public static void main(String[] args) {
+//        List<File> files = videoIntercept(new File("C:\\Users\\宋英豪\\Desktop\\nfpHmswW.mp4"), 10, 100);
+//    }
+//
+//    /**
+//     * Mat转换成BufferedImage
+//     *
+//     * @param matrix
+//     *            要转换的Mat
+//     * @param fileExtension
+//     *            格式为 ".jpg", ".png", etc
+//     * @return
+//     */
+//    public static BufferedImage Mat2BufImg (Mat matrix, String fileExtension) {
+//        // convert the matrix into a matrix of bytes appropriate for
+//        // this file extension
+//        MatOfByte mob = new MatOfByte();
+//        Imgcodecs.imencode(fileExtension, matrix, mob);
+//        // convert the "matrix of bytes" into a byte array
+//        byte[] byteArray = mob.toArray();
+//        BufferedImage bufImage = null;
+//        try {
+//            InputStream in = new ByteArrayInputStream(byteArray);
+//            bufImage = ImageIO.read(in);
+//        } catch (Exception e) {
+//            e.printStackTrace();
+//        }
+//        return bufImage;
+//    }
+//
+//    /**
+//     * BufferedImage转换成Mat
+//     *
+//     * @param original
+//     *            要转换的BufferedImage
+//     * @param imgType
+//     *            bufferedImage的类型 如 BufferedImage.TYPE_3BYTE_BGR
+//     * @param matType
+//     *            转换成mat的type 如 CvType.CV_8UC3
+//     */
+//    public static Mat BufImg2Mat (BufferedImage original, int imgType, int matType) {
+//        if (original == null) {
+//            throw new IllegalArgumentException("original == null");
+//        }
+//
+//        // Don't convert if it already has correct type
+//        if (original.getType() != imgType) {
+//
+//            // Create a buffered image
+//            BufferedImage image = new BufferedImage(original.getWidth(), original.getHeight(), imgType);
+//
+//            // Draw the image onto the new buffer
+//            Graphics2D g = image.createGraphics();
+//            try {
+//                g.setComposite(AlphaComposite.Src);
+//                g.drawImage(original, 0, 0, null);
+//            } finally {
+//                g.dispose();
+//            }
+//        }
+//
+//        byte[] pixels = ((DataBufferByte) original.getRaster().getDataBuffer()).getData();
+//        Mat mat = Mat.eye(original.getHeight(), original.getWidth(), matType);
+//        mat.put(0, 0, pixels);
+//        return mat;
+//    }
+//}

+ 21 - 1
module-job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/ByteDanceRefreshTokenJob.java

@@ -1,16 +1,22 @@
 package cn.com.ctop.job.bytedance.handler;
 
+import cn.com.ctop.common.module.entity.UserAllocation;
 import cn.com.ctop.common.module.service.IRefreshTokenService;
+import cn.com.ctop.common.module.service.IUserAllocationService;
+import cn.com.ctop.common.module.utils.HttpUtils;
 import com.xxl.job.core.handler.annotation.XxlJob;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
+import java.util.HashMap;
+import java.util.List;
+
 @Component
 public class ByteDanceRefreshTokenJob {
 
     @Autowired
     private IRefreshTokenService refreshTokenService;
-
+    private static final String REFRESH_AUTHNAME_URL = "http://api.tjyourong.com.cn/jeecg-boot/ctop/userAllocation/updateAuthName?";
     /**
      * 刷新token
      * @throws Exception
@@ -19,4 +25,18 @@ public class ByteDanceRefreshTokenJob {
     public void execute() throws Exception {
         refreshTokenService.byteDanceRefresh(null);
     }
+
+    @Autowired
+    private IUserAllocationService userAllocationService;
+    @XxlJob("refreshAuthName")
+    public void refreshAuthName() throws Exception {
+        List<UserAllocation> userAllocationList = userAllocationService.listByMediaId(null,0);
+        if(null!=userAllocationList&&!userAllocationList.isEmpty()){
+            userAllocationList.forEach(userAllocation -> {
+                String url = REFRESH_AUTHNAME_URL+"accountId="+userAllocation.getAccountId()+"&mediaId="+userAllocation.getMediaId();
+                HttpUtils.httpGet(url,new HashMap<>(),new HashMap<>());
+            });
+        }
+        refreshTokenService.byteDanceRefresh(null);
+    }
 }

+ 73 - 0
module-job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedanceCheckAccountReportJob.java

@@ -0,0 +1,73 @@
+package cn.com.ctop.job.bytedance.handler;
+
+import cn.com.ctop.common.module.utils.Check;
+import cn.com.ctop.toutiao.modules.report.entity.BytedanceCheckReportTaskInfo;
+import cn.com.ctop.toutiao.modules.report.service.IBytedanceCheckReportTaskInfoService;
+import com.xxl.job.core.context.XxlJobHelper;
+import com.xxl.job.core.handler.annotation.XxlJob;
+import org.jeecg.common.util.DateUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.Resource;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+@Component
+public class BytedanceCheckAccountReportJob {
+    private ExecutorService executorService = Executors.newFixedThreadPool(10);
+    private CountDownLatch countDownLatch;
+    @Autowired
+    private IBytedanceCheckReportTaskInfoService checkReportTaskInfoService;
+    @XxlJob("initCheckReportJob")
+    public void initCheckReportJob() throws Exception {
+        String params = DateUtils.formatDate();
+        String date = XxlJobHelper.getJobParam();
+        if (date==null || date.trim().length()==0) {
+            date = params;
+        }
+        checkReportTaskInfoService.initTask(date);
+        //查询初始化的任务
+        List<BytedanceCheckReportTaskInfo>taskInfos = checkReportTaskInfoService.getListByParams(null,date,0);
+        if(null!=taskInfos&&!taskInfos.isEmpty()){
+            countDownLatch = new CountDownLatch(taskInfos.size());
+            taskInfos.forEach(taskInfo -> {
+                try {
+                    checkReportTaskInfoService.checkCost(taskInfo);
+                }catch (Exception e){
+                }finally {
+                    countDownLatch.countDown();
+                }
+            });
+            countDownLatch.await();
+            XxlJobHelper.log("{}数据初始化完成",date);
+        }
+    }
+
+    @XxlJob("refreshCheckReportJob")
+    public void refreshCheckReportJob() throws Exception {
+        String params = DateUtils.formatDate();
+        String date = XxlJobHelper.getJobParam();
+        if (date==null || date.trim().length()==0) {
+            date = params;
+        }
+        //1:查询异常任务数据
+        List<BytedanceCheckReportTaskInfo>taskInfos = checkReportTaskInfoService.getListByParams(null,date,50001);
+        if(null!=taskInfos&&!taskInfos.isEmpty()){
+            countDownLatch = new CountDownLatch(taskInfos.size());
+            taskInfos.forEach(taskInfo -> {
+                try {
+                    checkReportTaskInfoService.reloadData(taskInfo);
+                    checkReportTaskInfoService.checkCost(taskInfo);
+                }catch (Exception e){
+                }finally {
+                    countDownLatch.countDown();
+                }
+            });
+            countDownLatch.await();
+            XxlJobHelper.log("{}数据更新完成",date);
+        }
+    }
+}

+ 19 - 1
module-job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedanceEtlVideoJob.java

@@ -1,5 +1,6 @@
 package cn.com.ctop.job.bytedance.handler;
 
+import cn.com.ctop.toutiao.modules.link.service.IETLReportBytedanceVideoService;
 import cn.com.ctop.toutiao.modules.report.service.IBytedanceVideoEtlInfoService;
 import cn.com.ctop.toutiao.modules.report.service.IEtlBytedanceReportVideoDailyService;
 import com.xxl.job.core.handler.annotation.XxlJob;
@@ -42,7 +43,24 @@ public class BytedanceEtlVideoJob {
         etlBytedanceReportVideoDailyService.updBytedanceVideo();
     }
 
-    @XxlJob("etlBytedanceVideoJob")
+    @Autowired
+    private IETLReportBytedanceVideoService etlReportBytedanceVideoService;
+    @XxlJob("etlBytedanceVideoClipInfoJob")
+    public void etlBytedanceVideoClipInfoJob() throws Exception {
+        etlReportBytedanceVideoService.etlBytedanceVideoClipIdJob();
+    }
+
+    @XxlJob("etlBytedanceVideoPlanInfoJob")
+    public void etlBytedanceVideoPlanInfoJob() throws Exception {
+        etlReportBytedanceVideoService.etlBytedanceVideoPlanInfoJob();
+    }
+
+    @XxlJob("etlBytedanceVideoShotInfoJob")
+    public void etlBytedanceVideoShotInfoJob() throws Exception {
+        etlReportBytedanceVideoService.etlBytedanceVideoShotInfoJob();
+    }
+
+    @XxlJob("formatBytedance")
     public void etlBytedanceVideoJob() throws Exception {
         etlBytedanceReportVideoDailyService.etlBytedanceVideo();
     }

+ 1 - 1
module-job-kuaishou/src/main/java/cn/com/ctop/job/kuaishou/handler/KuaishouProjectVideoGetJob.java

@@ -33,7 +33,7 @@ public class KuaishouProjectVideoGetJob {
         //2快手,1开启的项目
         List<Project> projects = projectService.listByMediaType(2, 0);
         projects.forEach(project -> executorService.submit(() -> {
-            projectVideoGetService.clearProjectVideo(project.getId(),1);
+            projectVideoGetService.clearProjectVideo(project.getId(),1,null);
         }));
         XxlJobHelper.log("清洗项目素材完成");
     }

+ 1 - 1
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/controller/KuaishouProjectVideoGetController.java

@@ -241,7 +241,7 @@ public class KuaishouProjectVideoGetController {
     @GetMapping(value = "/clearProjectVideo")
     public Result<Object> clearProjectVideo(Long projectId) {
         try {
-            kuaishouProjectVideoGetService.clearProjectVideo(projectId,1);
+            kuaishouProjectVideoGetService.clearProjectVideo(projectId,1,null);
         } catch (Exception e) {
             log.error(e.getMessage(), e);
         }

+ 2 - 2
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/mapper/xml/KuaiShouVideoGetMapper.xml

@@ -306,8 +306,8 @@
         INNER JOIN ctop_user_allocation t2 ON t1.account_id = t2.account_id
         WHERE t1.`status` =0
         AND t2.project_id=#{projectId}
-        <if test="updateTime!=null">
-            AND t1.update_time >=#{updateTime}
+        <if test="updateTime!=null and updateTime!=''">
+            AND t1.update_time &gt;=#{updateTime}
         </if>
         group by t1.signature
     </select>

+ 1 - 1
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/IKuaishouProjectVideoGetService.java

@@ -17,7 +17,7 @@ public interface IKuaishouProjectVideoGetService extends IService<KuaishouProjec
     /**
      * 清洗项目数据
      */
-    void clearProjectVideo(Long projectId, Integer pageStart);
+    void clearProjectVideo(Long projectId, Integer pageStart,String updateTime);
 
     Result<Object> queryPageList(KuaishouProjectVideoGet kuaishouProjectVideoGet, Integer pageNo, Integer pageSize);
 

+ 5 - 3
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/impl/KuaishouProjectVideoGetServiceImpl.java

@@ -62,10 +62,12 @@ public class KuaishouProjectVideoGetServiceImpl extends ServiceImpl<KuaishouProj
     private IKuaiShouVideoGetService videoGetService;
 
     @Override
-    public void clearProjectVideo(Long projectId, Integer pageNo) {
+    public void clearProjectVideo(Long projectId, Integer pageNo, String updateTime) {
         try {
             log.info("清洗快手项目素材数据,projectId:{},pageNo:{}", projectId, pageNo);
-            String updateTime = projectVideoGetMapper.queryLatestUpdateTime(projectId);
+            if (Check.isNull(updateTime) && pageNo == 1) {
+                updateTime = projectVideoGetMapper.queryLatestUpdateTime(projectId);
+            }
             PageHelper.startPage(pageNo, 1500);
             List<KuaishouProjectVideoGet> list = videoGetMapper.selectProjectVideo(projectId, updateTime);
             if (Check.isNull(list)) {
@@ -73,7 +75,7 @@ public class KuaishouProjectVideoGetServiceImpl extends ServiceImpl<KuaishouProj
             }
             projectVideoGetMapper.replaceBatch(list);
             Thread.sleep(500);
-            clearProjectVideo(projectId, pageNo + 1);
+            clearProjectVideo(projectId, pageNo + 1, updateTime);
         } catch (Exception e) {
             log.error("清洗项目数据异常", e);
         }

+ 10 - 4
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/material/service/impl/EtlKuaishouVideoInfoServiceImpl.java

@@ -2,6 +2,7 @@ package cn.com.ctop.kuaishou.modules.material.service.impl;
 
 import cn.com.ctop.common.module.entity.MaterialInfo;
 import cn.com.ctop.common.module.entity.SysUser;
+import cn.com.ctop.common.module.mapper.SysUserMapper;
 import cn.com.ctop.common.module.service.IMaterialAscriptionService;
 import cn.com.ctop.common.module.service.IMaterialInfoService;
 import cn.com.ctop.common.module.service.ISysUserService;
@@ -10,6 +11,7 @@ import cn.com.ctop.kuaishou.modules.batch.service.IKuaiShouVideoGetService;
 import cn.com.ctop.kuaishou.modules.material.entity.EtlKuaishouVideoInfo;
 import cn.com.ctop.kuaishou.modules.material.mapper.EtlKuaishouVideoInfoMapper;
 import cn.com.ctop.kuaishou.modules.material.service.IEtlKuaishouVideoInfoService;
+import cn.com.ctop.kuaishou.modules.report.mapper.KuaishouReportDailyAccountMapper;
 import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
@@ -44,9 +46,7 @@ public class EtlKuaishouVideoInfoServiceImpl extends ServiceImpl<EtlKuaishouVide
     private ISysUserService sysUserService;
     @Autowired
     private IMaterialInfoService materialInfoService;
-
     ExecutorService executorService = Executors.newFixedThreadPool(200);
-
     @Override
     public void etlKuaishouVideoInfo(String date) {
         //1:查询当前日期内的所有存在md5码信息的视频数据
@@ -82,8 +82,14 @@ public class EtlKuaishouVideoInfoServiceImpl extends ServiceImpl<EtlKuaishouVide
                                 //根据id查询剪辑所属的设计负责人
                                 SysUser user = sysUserService.getById(clipId);
                                 if(null!=user){
-                                    etlKuaishouVideoInfo.setDesignTeamLeaderId(user.getLeaderId());
-                                    etlKuaishouVideoInfo.setDesignTeamLeaderName(user.getLeaderName());
+                                    String roleCode = sysUserService.getRoleCodeByUserId(user.getId());
+                                    if(null!=roleCode&&roleCode.trim().equals("designTeamLeader")){
+                                        etlKuaishouVideoInfo.setDesignTeamLeaderId(user.getId());
+                                        etlKuaishouVideoInfo.setDesignTeamLeaderName(user.getRealname());
+                                    }else{
+                                        etlKuaishouVideoInfo.setDesignTeamLeaderId(user.getLeaderId());
+                                        etlKuaishouVideoInfo.setDesignTeamLeaderName(user.getLeaderName());
+                                    }
                                 }
                             }
                             String clipName = ascription.getString("clipName");

+ 8 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/mapper/ETLReportBytedanceVideoMapper.java

@@ -34,4 +34,12 @@ public interface ETLReportBytedanceVideoMapper extends BaseMapper<ETLReportByted
     List<ETLReportBytedanceVideo> getDuplicateVideoInfo(@Param("date")String date);
 
     void deleteDuplicateVideoInfo(@Param("video")ETLReportBytedanceVideo video);
+
+    List<ETLReportBytedanceVideo> getBytedanceVideoClipInfo();
+    List<ETLReportBytedanceVideo> getBytedanceVideoShotInfo();
+    List<ETLReportBytedanceVideo> getBytedanceVideoPlanInfo();
+
+    void updateClipInfo(@Param("clipId")String clipId, @Param("clipName")String clipName, @Param("md5")String md5);
+    void updateShotInfo(@Param("shotId")String shotId, @Param("shotName")String shotName, @Param("md5")String md5);
+    void updatePlanInfo(@Param("planId")String planId, @Param("planName")String planName, @Param("md5")String md5);
 }

+ 55 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/mapper/xml/ETLReportBytedanceVideoMapper.xml

@@ -201,6 +201,16 @@
             )
         </foreach>
     </insert>
+    <update id="updateClipInfo">
+        update etl_report_bytedance_video set clip_id=#{clipId},clip_name=#{clipName} where md5=#{md5}
+    </update>
+    <update id="updateShotInfo">
+        update etl_report_bytedance_video set shot_id=#{shotId},shot_name=#{shotName} where md5=#{md5}
+    </update>
+    <update id="updatePlanInfo">
+        update etl_report_bytedance_video set plan_id=#{planId},plan_name=#{planName} where md5=#{md5}
+    </update>
+
     <delete id="deleteDuplicateVideoInfo">
         delete from
                     etl_report_bytedance_video
@@ -319,4 +329,49 @@
         from etl_report_bytedance_video where stat_datetime =#{date} and cost>0 GROUP BY stat_datetime,material_id,account_id
         HAVING total >1 ORDER BY total DESC;
     </select>
+    <select id="getBytedanceVideoClipInfo"
+            resultType="cn.com.ctop.toutiao.modules.link.entity.ETLReportBytedanceVideo">
+        SELECT DISTINCT
+            video.md5 as 'md5',
+            cmi.clip_id as 'clip_id',
+            u.realname as 'clip_name'
+        FROM
+            etl_report_bytedance_video video
+                LEFT JOIN ctop_material_ascription cmi ON video.md5 = cmi.
+                CODE LEFT JOIN sys_user u ON u.id = cmi.clip_id
+        WHERE
+            cmi.clip_id IS NOT NULL
+          AND video.clip_id = ''
+          AND u.realname IS NOT NULL
+    </select>
+    <select id="getBytedanceVideoShotInfo"
+            resultType="cn.com.ctop.toutiao.modules.link.entity.ETLReportBytedanceVideo">
+        SELECT DISTINCT
+            video.md5 as 'md5',
+            cmi.shot_id as 'shot_id',
+            u.realname as 'shot_name'
+        FROM
+            etl_report_bytedance_video video
+                LEFT JOIN ctop_material_ascription cmi ON video.md5 = cmi.
+                CODE LEFT JOIN sys_user u ON u.id = cmi.shot_id
+        WHERE
+            cmi.shot_id IS NOT NULL
+          AND video.shot_id = ''
+          AND u.realname IS NOT NULL
+    </select>
+    <select id="getBytedanceVideoPlanInfo"
+            resultType="cn.com.ctop.toutiao.modules.link.entity.ETLReportBytedanceVideo">
+        SELECT DISTINCT
+            video.md5 as 'md5',
+            cmi.plan_id as 'plan_id',
+            u.realname as 'plan_name'
+        FROM
+            etl_report_bytedance_video video
+                LEFT JOIN ctop_material_ascription cmi ON video.md5 = cmi.
+                CODE LEFT JOIN sys_user u ON u.id = cmi.plan_id
+        WHERE
+            cmi.plan_id IS NOT NULL
+          AND video.plan_id = ''
+          AND u.realname IS NOT NULL
+    </select>
 </mapper>

+ 6 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/service/IETLReportBytedanceVideoService.java

@@ -16,4 +16,10 @@ public interface IETLReportBytedanceVideoService extends IService<ETLReportByted
     void cleanETL(String statDate);
 
     void cleanData(String data);
+
+    void etlBytedanceVideoClipIdJob();
+
+    void etlBytedanceVideoPlanInfoJob();
+
+    void etlBytedanceVideoShotInfoJob();
 }

+ 30 - 51
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/service/impl/ETLReportBytedanceVideoServiceImpl.java

@@ -35,57 +35,6 @@ public class ETLReportBytedanceVideoServiceImpl extends ServiceImpl<ETLReportByt
         return baseReportBytedanceVideoDailyService.list(queryWrapper);
     }
 
-/*    @Override
-    public void cleanETL(String statDate) {
-        List<BaseReportBytedanceVideoDaily> source = listByStatDate(statDate);
-        if(source.isEmpty()){
-            XxlJobHelper.log("头条数据报表清洗ETL异常,请检查数据源");
-        }
-        List<ETLReportBytedanceVideo> batch = new ArrayList<>();
-        source.forEach(it->{
-            JSONObject videoInfo= etlReportBytedanceVideoMapper.queryVideoInfoByMaterialId(it.getMaterialId());
-            if(null==videoInfo||videoInfo.isEmpty()){
-                XxlJobHelper.log("头条数据报表清洗ETL异常,未关联到video_info,materialId:{}",it.getMaterialId());
-            }else {
-                //TODO 暂时从老数据库查询,新素材库怕同步数据有偏差
-                //JSONObject materialVideo= etlReportBytedanceVideoMapper.queryMaterialVideoByMd5(videoInfo.getString("md5"));
-                JSONObject materialVideo= etlReportBytedanceVideoMapper.queryMaterialVideoByMd5Old(videoInfo.getString("md5"));
-                //JSONObject relatePerson=etlReportBytedanceVideoMapper.queryRelatePersonByMd5(videoInfo.getString("md5"));
-                JSONObject relatePerson=etlReportBytedanceVideoMapper.queryRelatePersonByMd5Old(videoInfo.getString("md5"));
-
-                JSONObject accountDetail=etlReportBytedanceVideoMapper.queryAccountDetailByAccountId(it.getAccountId());
-                ETLReportBytedanceVideo temp = JSONObject.toJavaObject(JSONObject.parseObject(JSON.toJSON(it).toString()),ETLReportBytedanceVideo.class);
-                temp.setMd5(videoInfo.getString("md5"));
-                temp.setToutiaoUrl(videoInfo.getString("url"));
-                if(null!=materialVideo&&!materialVideo.isEmpty()) {
-                    temp.setUrl(materialVideo.getString("url"));
-                    temp.setCoverUrl(materialVideo.getString("coverUrl"));
-                    temp.setMaterialName(materialVideo.getString("materialName"));
-                    temp.setMaterialCreateTime(materialVideo.getDate("createTime").toString());
-                }
-                if(null!=accountDetail&&!accountDetail.isEmpty()){
-                    temp.setAccountName(accountDetail.getString("accountName"));
-                    temp.setProjectId(accountDetail.getLong("projectId"));
-                    temp.setProjectName(accountDetail.getString("projectName"));
-                    temp.setProductId(accountDetail.getLong("productId"));
-                    temp.setProductName(accountDetail.getString("productName"));
-                    temp.setAdvertiserId(accountDetail.getString("advertiserId"));
-                    temp.setAdvertiserName(accountDetail.getString("advertiserName"));
-                }
-                if(null!=relatePerson&&!relatePerson.isEmpty()){
-                    temp.setClipId(relatePerson.getString("clipId"));
-                    temp.setClipName(relatePerson.getString("clipName"));
-                    temp.setShotId(relatePerson.getString("shotId"));
-                    temp.setShotName(relatePerson.getString("shotName"));
-                    temp.setPlanId(relatePerson.getString("planId"));
-                    temp.setPlanName(relatePerson.getString("planName"));
-                }
-                batch.add(temp);
-            }
-            this.saveBatch(batch);
-        });
-    }*/
-
     @Override
     public void cleanETL(String statDate) {
         List<ETLReportBytedanceVideo> etlReportBytedanceVideos = etlReportBytedanceVideoMapper.queryETLData(statDate);
@@ -112,4 +61,34 @@ public class ETLReportBytedanceVideoServiceImpl extends ServiceImpl<ETLReportByt
         }
     }
 
+    @Override
+    public void etlBytedanceVideoClipIdJob() {
+        List <ETLReportBytedanceVideo> clipInfos = etlReportBytedanceVideoMapper.getBytedanceVideoClipInfo();
+        if(null!=clipInfos&&!clipInfos.isEmpty()){
+            clipInfos.forEach(clipInfo->{
+                etlReportBytedanceVideoMapper.updateClipInfo(clipInfo.getClipId(),clipInfo.getClipName(),clipInfo.getMd5());
+            });
+        }
+    }
+
+    @Override
+    public void etlBytedanceVideoPlanInfoJob() {
+        List <ETLReportBytedanceVideo> planInfos = etlReportBytedanceVideoMapper.getBytedanceVideoPlanInfo();
+        if(null!=planInfos&&!planInfos.isEmpty()){
+            planInfos.forEach(planInfo->{
+                etlReportBytedanceVideoMapper.updatePlanInfo(planInfo.getPlanId(),planInfo.getPlanName(),planInfo.getMd5());
+            });
+        }
+    }
+
+    @Override
+    public void etlBytedanceVideoShotInfoJob() {
+        List <ETLReportBytedanceVideo> shotInfos = etlReportBytedanceVideoMapper.getBytedanceVideoShotInfo();
+        if(null!=shotInfos&&!shotInfos.isEmpty()){
+            shotInfos.forEach(shotInfo->{
+                etlReportBytedanceVideoMapper.updateShotInfo(shotInfo.getShotId(),shotInfo.getShotName(),shotInfo.getMd5());
+            });
+        }
+    }
+
 }

+ 166 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/controller/BytedanceCheckReportTaskInfoController.java

@@ -0,0 +1,166 @@
+package cn.com.ctop.toutiao.modules.report.controller;
+
+import cn.com.ctop.common.module.utils.QueryGenerator;
+import cn.com.ctop.toutiao.modules.report.entity.BytedanceCheckReportTaskInfo;
+import cn.com.ctop.toutiao.modules.report.service.IBytedanceCheckReportTaskInfoService;
+import com.alibaba.fastjson.JSON;
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import com.baomidou.mybatisplus.core.metadata.IPage;
+import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
+import lombok.extern.slf4j.Slf4j;
+import org.jeecg.common.api.vo.Result;
+import org.jeecg.common.util.oConvertUtils;
+import org.jeecgframework.poi.excel.ExcelImportUtil;
+import org.jeecgframework.poi.excel.def.NormalExcelConstants;
+import org.jeecgframework.poi.excel.entity.ExportParams;
+import org.jeecgframework.poi.excel.entity.ImportParams;
+import org.jeecgframework.poi.excel.view.JeecgEntityExcelView;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.*;
+import org.springframework.web.multipart.MultipartFile;
+import org.springframework.web.multipart.MultipartHttpServletRequest;
+import org.springframework.web.servlet.ModelAndView;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URLDecoder;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * 头条数据检查任务记录
+ * @author jeecg-boot
+ * @date   2021-06-07
+ * @version V1.0
+ */
+@Slf4j
+@Api(tags="头条数据检查任务记录")
+@RestController
+@RequestMapping("/ctop/bytedanceCheckReportTaskInfo")
+public class BytedanceCheckReportTaskInfoController {
+	@Autowired
+	private IBytedanceCheckReportTaskInfoService bytedanceCheckReportTaskInfoService;
+
+	/**
+	  * 分页列表查询
+	 * @param bytedanceCheckReportTaskInfo
+	 * @param pageNo
+	 * @param pageSize
+	 * @param req
+	 * @return
+	 */
+	@ApiOperation(value="头条数据检查任务记录-分页列表查询", notes="头条数据检查任务记录-分页列表查询")
+	@GetMapping(value = "/list")
+	public Result<IPage<BytedanceCheckReportTaskInfo>> queryPageList(BytedanceCheckReportTaskInfo bytedanceCheckReportTaskInfo,
+																	 @RequestParam(name="pageNo", defaultValue="1") Integer pageNo,
+																	 @RequestParam(name="pageSize", defaultValue="10") Integer pageSize,
+																	 HttpServletRequest req) {
+		Result<IPage<BytedanceCheckReportTaskInfo>> result = new Result<>();
+		QueryWrapper<BytedanceCheckReportTaskInfo> queryWrapper = QueryGenerator.initQueryWrapper(bytedanceCheckReportTaskInfo, req.getParameterMap());
+		Page<BytedanceCheckReportTaskInfo> page = new Page<>(pageNo, pageSize);
+		IPage<BytedanceCheckReportTaskInfo> pageList = bytedanceCheckReportTaskInfoService.page(page, queryWrapper);
+		result.setSuccess(true);
+		result.setResult(pageList);
+		return result;
+	}
+
+	/**
+	  *   添加
+	 * @param bytedanceCheckReportTaskInfo
+	 * @return
+	 */
+	@ApiOperation(value="头条数据检查任务记录-添加", notes="头条数据检查任务记录-添加")
+	@PostMapping(value = "/add")
+	public Result<BytedanceCheckReportTaskInfo> add(@RequestBody BytedanceCheckReportTaskInfo bytedanceCheckReportTaskInfo) {
+		Result<BytedanceCheckReportTaskInfo> result = new Result<>();
+		try {
+			bytedanceCheckReportTaskInfoService.save(bytedanceCheckReportTaskInfo);
+			result.success("添加成功!");
+		} catch (Exception e) {
+			log.error(e.getMessage(),e);
+			result.error500("操作失败");
+		}
+		return result;
+	}
+
+	/**
+	  *  编辑
+	 * @param bytedanceCheckReportTaskInfo
+	 * @return
+	 */
+	@ApiOperation(value="头条数据检查任务记录-编辑", notes="头条数据检查任务记录-编辑")
+	@PutMapping(value = "/edit")
+	public Result<BytedanceCheckReportTaskInfo> edit(@RequestBody BytedanceCheckReportTaskInfo bytedanceCheckReportTaskInfo) {
+		Result<BytedanceCheckReportTaskInfo> result = new Result<BytedanceCheckReportTaskInfo>();
+		BytedanceCheckReportTaskInfo bytedanceCheckReportTaskInfoEntity = bytedanceCheckReportTaskInfoService.getById(bytedanceCheckReportTaskInfo.getId());
+		if(bytedanceCheckReportTaskInfoEntity==null) {
+			result.error500("未找到对应实体");
+		}else {
+			boolean ok = bytedanceCheckReportTaskInfoService.updateById(bytedanceCheckReportTaskInfo);
+			if(ok) {
+				result.success("修改成功!");
+			}
+		}
+
+		return result;
+	}
+
+	/**
+	  *   通过id删除
+	 * @param id
+	 * @return
+	 */
+	@ApiOperation(value="头条数据检查任务记录-通过id删除", notes="头条数据检查任务记录-通过id删除")
+	@DeleteMapping(value = "/delete")
+	public Result<?> delete(@RequestParam(name="id") String id) {
+		try {
+			bytedanceCheckReportTaskInfoService.removeById(id);
+		} catch (Exception e) {
+			log.error("删除失败",e.getMessage());
+			return Result.error("删除失败!");
+		}
+		return Result.ok("删除成功!");
+	}
+
+	/**
+	 *  批量删除
+	 * @param ids
+	 * @return
+	 */
+	@ApiOperation(value="头条数据检查任务记录-批量删除", notes="头条数据检查任务记录-批量删除")
+	@DeleteMapping(value = "/deleteBatch")
+	public Result<BytedanceCheckReportTaskInfo> deleteBatch(@RequestParam(name="ids") String ids) {
+		Result<BytedanceCheckReportTaskInfo> result = new Result<>();
+		if(ids==null || "".equals(ids.trim())) {
+			result.error500("参数不识别!");
+		}else {
+			this.bytedanceCheckReportTaskInfoService.removeByIds(Arrays.asList(ids.split(",")));
+			result.success("删除成功!");
+		}
+		return result;
+	}
+
+	/**
+	  * 通过id查询
+	 * @param id
+	 * @return
+	 */
+	@ApiOperation(value="头条数据检查任务记录-通过id查询", notes="头条数据检查任务记录-通过id查询")
+	@GetMapping(value = "/queryById")
+	public Result<BytedanceCheckReportTaskInfo> queryById(@RequestParam(name="id",required=true) String id) {
+		Result<BytedanceCheckReportTaskInfo> result = new Result<>();
+		BytedanceCheckReportTaskInfo bytedanceCheckReportTaskInfo = bytedanceCheckReportTaskInfoService.getById(id);
+		if(bytedanceCheckReportTaskInfo==null) {
+			result.error500("未找到对应实体");
+		}else {
+			result.setResult(bytedanceCheckReportTaskInfo);
+			result.setSuccess(true);
+		}
+		return result;
+	}
+}

+ 67 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/entity/BytedanceCheckReportTaskInfo.java

@@ -0,0 +1,67 @@
+package cn.com.ctop.toutiao.modules.report.entity;
+
+import java.io.Serializable;
+import java.math.BigDecimal;
+import java.util.Date;
+
+import cn.com.ctop.common.module.entity.CtopOauthToken;
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import com.baomidou.mybatisplus.annotation.TableField;
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.experimental.Accessors;
+import com.fasterxml.jackson.annotation.JsonFormat;
+import org.springframework.format.annotation.DateTimeFormat;
+import org.jeecgframework.poi.excel.annotation.Excel;
+
+/**
+ * 头条数据检查任务记录
+ * @author jeecg-boot
+ * @date   2021-06-07
+ * @version V1.0
+ */
+@Data
+@TableName("ctop_bytedance_check_report_task_info")
+public class BytedanceCheckReportTaskInfo {
+    
+	/**id*/
+	@TableId(type = IdType.AUTO)
+	private Long id;
+	/**accountId*/
+	private Long accountId;
+	private BigDecimal accountCost;
+	private BigDecimal campaignCost;
+	private BigDecimal matCost;
+	private BigDecimal imageCost;
+	/**stateDate*/
+	private String stateDate;
+	/**status*/
+	private Integer status;
+	/**desc*/
+	private String description;
+	/**createTime*/
+	private Date createTime;
+	/**updateTime*/
+	private Date updateTime;
+	private Integer costStatus;
+
+	public BytedanceCheckReportTaskInfo(){
+
+	}
+	public BytedanceCheckReportTaskInfo(CtopOauthToken token, String date) {
+		this.accountId = token.getAccountId();
+		this.accountCost = BigDecimal.ZERO;
+		this.campaignCost = BigDecimal.ZERO;
+		this.matCost = BigDecimal.ZERO;
+		this.imageCost = BigDecimal.ZERO;
+		this.status = 0;
+		this.costStatus = 0;
+		this.stateDate = date;
+		this.createTime = new Date();
+		this.updateTime = new Date();
+	}
+}

+ 7 - 4
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/BytedanceCampaignDailyReportMapper.java

@@ -1,14 +1,17 @@
 package cn.com.ctop.toutiao.modules.report.mapper;
 
 import cn.com.ctop.toutiao.modules.report.entity.BytedanceCampaignDailyReport;
+import cn.com.ctop.toutiao.modules.report.entity.BytedanceCheckReportTaskInfo;
 import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import org.apache.ibatis.annotations.Param;
 
 /**
- * @Description: 广告组日报表信息
- * @Author: jeecg-boot
- * @Date: 2019-09-11
- * @Version: V1.0
+ * 广告组日报表信息
+ * @author jeecg-boot
+ * 2019-09-11
+ * @version V1.0
  */
 public interface BytedanceCampaignDailyReportMapper extends BaseMapper<BytedanceCampaignDailyReport> {
 
+    BytedanceCheckReportTaskInfo checkCampaignCostByParams(@Param(value = "accountId") Long accountId, @Param(value = "stateDate")String stateDate);
 }

+ 14 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/BytedanceCheckReportTaskInfoMapper.java

@@ -0,0 +1,14 @@
+package cn.com.ctop.toutiao.modules.report.mapper;
+
+import cn.com.ctop.toutiao.modules.report.entity.BytedanceCheckReportTaskInfo;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+/**
+ * 头条数据检查任务记录
+ * @author jeecg-boot
+ * 2021-06-07
+ * @version V1.0
+ */
+public interface BytedanceCheckReportTaskInfoMapper extends BaseMapper<BytedanceCheckReportTaskInfo> {
+
+}

+ 5 - 4
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/BytedanceReportMaterialDailyMapper.java

@@ -3,10 +3,7 @@ package cn.com.ctop.toutiao.modules.report.mapper;
 import cn.com.ctop.common.module.vo.BytedanceImageCostVO;
 import cn.com.ctop.common.module.vo.BytedanceImageCostVOEntity;
 import cn.com.ctop.toutiao.modules.material.vo.BytedanceVideoVo;
-import cn.com.ctop.toutiao.modules.report.entity.BytedanceReportMaterialDaily;
-import cn.com.ctop.toutiao.modules.report.entity.BytedanceReportMaterialRetry;
-import cn.com.ctop.toutiao.modules.report.entity.BytedanceReportVideoMaterialDaily;
-import cn.com.ctop.toutiao.modules.report.entity.BytedanceVideoEtlInfo;
+import cn.com.ctop.toutiao.modules.report.entity.*;
 import com.alibaba.fastjson.JSONArray;
 import com.baomidou.mybatisplus.core.mapper.BaseMapper;
 import org.apache.ibatis.annotations.Param;
@@ -63,4 +60,8 @@ public interface BytedanceReportMaterialDailyMapper extends BaseMapper<Bytedance
     List<BytedanceVideoEtlInfo> getSignatureByDate(@Param(value = "date") String date);
 
     BytedanceVideoEtlInfo getEtlDataByParams(@Param("signature")String signature, @Param("date")String date);
+
+    BytedanceCheckReportTaskInfo checkMatCostByParams(@Param("accountId")Long accountId, @Param("stateDate")String stateDate);
+
+    BytedanceCheckReportTaskInfo checkImageCostByParams(@Param("accountId")Long accountId, @Param("stateDate")String stateDate);
 }

+ 12 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/xml/BytedanceCampaignDailyReportMapper.xml

@@ -2,4 +2,16 @@
 <!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
 <mapper namespace="cn.com.ctop.toutiao.modules.report.mapper.BytedanceCampaignDailyReportMapper">
 
+    <select id="checkCampaignCostByParams"
+            resultType="cn.com.ctop.toutiao.modules.report.entity.BytedanceCheckReportTaskInfo">
+        select
+               sum(cost) as campaign_cost,
+               stat_datetime as state_date
+        from
+             ctop_bytedance_report_campaign_daily
+        where
+              advertiser_id = #{accountId}
+          and
+              stat_datetime = #{stateDate}
+    </select>
 </mapper>

+ 5 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/xml/BytedanceCheckReportTaskInfoMapper.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="cn.com.ctop.toutiao.modules.report.mapper.BytedanceCheckReportTaskInfoMapper">
+
+</mapper>

+ 24 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/xml/BytedanceReportMaterialDailyMapper.xml

@@ -450,6 +450,30 @@ where a.plane_id is not null and a.plane_name is null
             report.stat_datetime&lt;= #{date}
         group by video.signature;
     </select>
+    <select id="checkMatCostByParams"
+            resultType="cn.com.ctop.toutiao.modules.report.entity.BytedanceCheckReportTaskInfo">
+        select
+               sum(cost) as mat_cost,
+               stat_datetime as state_date
+        from
+             ctop_bytedance_report_material_daily
+        where
+              account_id = #{accountId}
+          and
+              stat_datetime = #{stateDate}
+    </select>
+    <select id="checkImageCostByParams"
+            resultType="cn.com.ctop.toutiao.modules.report.entity.BytedanceCheckReportTaskInfo">
+        select
+            sum(cost) as image_cost,
+            stat_datetime as state_date
+        from
+            ctop_bytedance_report_image_daily
+        where
+            account_id = #{accountId}
+          and
+            stat_datetime = #{stateDate}
+    </select>
 
     <insert id="replaceIntoVideoMaterialBatch">
         REPLACE INTO ctop_bytedance_report_video_material_daily

+ 2 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/IBytedanceAdvertiserDailyReportService.java

@@ -1,6 +1,7 @@
 package cn.com.ctop.toutiao.modules.report.service;
 
 import cn.com.ctop.toutiao.modules.report.entity.BytedanceAdvertiserDailyReport;
+import cn.com.ctop.toutiao.modules.report.entity.BytedanceCheckReportTaskInfo;
 import com.baomidou.mybatisplus.extension.service.IService;
 
 /**
@@ -11,4 +12,5 @@ import com.baomidou.mybatisplus.extension.service.IService;
  */
 public interface IBytedanceAdvertiserDailyReportService extends IService<BytedanceAdvertiserDailyReport> {
 
+    BytedanceAdvertiserDailyReport getByParams(Long accountId, String stateDate);
 }

+ 2 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/IBytedanceCampaignDailyReportService.java

@@ -1,6 +1,7 @@
 package cn.com.ctop.toutiao.modules.report.service;
 
 import cn.com.ctop.toutiao.modules.report.entity.BytedanceCampaignDailyReport;
+import cn.com.ctop.toutiao.modules.report.entity.BytedanceCheckReportTaskInfo;
 import com.baomidou.mybatisplus.extension.service.IService;
 
 /**
@@ -11,4 +12,5 @@ import com.baomidou.mybatisplus.extension.service.IService;
  */
 public interface IBytedanceCampaignDailyReportService extends IService<BytedanceCampaignDailyReport> {
 
+    BytedanceCheckReportTaskInfo checkCampaignCostByParams(Long accountId, String stateDate);
 }

+ 24 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/IBytedanceCheckReportTaskInfoService.java

@@ -0,0 +1,24 @@
+package cn.com.ctop.toutiao.modules.report.service;
+
+import cn.com.ctop.toutiao.modules.report.entity.BytedanceCheckReportTaskInfo;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+import java.util.List;
+
+/**
+ * 头条数据检查任务记录
+ * @author jeecg-boot
+ * 2021-06-07
+ * @version V1.0
+ */
+public interface IBytedanceCheckReportTaskInfoService extends IService<BytedanceCheckReportTaskInfo> {
+    void initTask(String date);
+
+    BytedanceCheckReportTaskInfo getInfoByParams(Long accountId, String date);
+
+    List<BytedanceCheckReportTaskInfo> getListByParams(Long accountId, String date,Integer status);
+
+    void checkCost(BytedanceCheckReportTaskInfo taskInfo);
+
+    void reloadData(BytedanceCheckReportTaskInfo taskInfo);
+}

+ 5 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/IBytedanceReportMaterialDailyService.java

@@ -1,6 +1,7 @@
 package cn.com.ctop.toutiao.modules.report.service;
 
 import cn.com.ctop.toutiao.modules.material.vo.BytedanceVideoVo;
+import cn.com.ctop.toutiao.modules.report.entity.BytedanceCheckReportTaskInfo;
 import cn.com.ctop.toutiao.modules.report.entity.BytedanceReportMaterialDaily;
 import cn.com.ctop.toutiao.modules.report.entity.BytedanceVideoEtlInfo;
 import com.baomidou.mybatisplus.extension.service.IService;
@@ -19,4 +20,8 @@ public interface IBytedanceReportMaterialDailyService extends IService<Bytedance
     List<BytedanceVideoEtlInfo> getSignatureByDate(String date);
 
     BytedanceVideoEtlInfo getEtlDataByParams(String signature, String date);
+
+    BytedanceCheckReportTaskInfo checkMatCostByParams(Long accountId, String stateDate);
+
+    BytedanceCheckReportTaskInfo checkImageCostByParams(Long accountId, String stateDate);
 }

+ 11 - 39
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/BytedanceAdvertiserDailyReportServiceImpl.java

@@ -4,6 +4,7 @@ import cn.com.ctop.toutiao.modules.report.DTO.AccountReportDTO;
 import cn.com.ctop.toutiao.modules.report.entity.BytedanceAdvertiserDailyReport;
 import cn.com.ctop.toutiao.modules.report.mapper.BytedanceAdvertiserDailyReportMapper;
 import cn.com.ctop.toutiao.modules.report.service.IBytedanceAdvertiserDailyReportService;
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import org.springframework.context.annotation.Primary;
 import org.springframework.stereotype.Service;
@@ -21,45 +22,16 @@ import java.util.List;
 @Primary
 
 public class BytedanceAdvertiserDailyReportServiceImpl extends ServiceImpl<BytedanceAdvertiserDailyReportMapper, BytedanceAdvertiserDailyReport> implements IBytedanceAdvertiserDailyReportService {
-
-    public void bytedanceAccountReportStatistics(AccountReportDTO accountReportDTO){
-        String startDate = accountReportDTO.getStartDate();
-        String endDate = accountReportDTO.getEndDate();
-        BigDecimal discount = accountReportDTO.getDiscount();  //仅头条内广可用 折扣
-        BigDecimal point = accountReportDTO.getPoint();  //仅头条可用 返点
-        List<Long> accountIds = accountReportDTO.getAccountIds();
-        Integer statHour = accountReportDTO.getStatHour();
-        Integer type = accountReportDTO.getType();  //1今天  2昨天和其他单天 3近一周、15天、1月、3月按钮 4六个月、一年 5输入框输入的除了单天的时间段
-
-        //type=1分时操作
-        if(type == 1){
-
-        }else if(type == 2){
-
-        }else if(type == 3 || type == 5){
-
+    @Override
+    public BytedanceAdvertiserDailyReport getByParams(Long accountId, String stateDate) {
+        QueryWrapper<BytedanceAdvertiserDailyReport> queryWrapper = new QueryWrapper<>();
+        if(null!=accountId&&accountId!=0){
+            queryWrapper.eq("advertiser_id",accountId);
         }
-
-
-
-
-
-
-
-    }
-
-
-
-
-    public void bytedanceAccountReportListAndCollect(AccountReportDTO accountReportDTO){
-
-
-
-
-
-
-
+        if(null!=stateDate&&!"".equals(stateDate.trim())){
+            queryWrapper.eq("stat_datetime",stateDate+" 00:00:00");
+        }
+        queryWrapper.orderByDesc("id").last("limit 1");
+        return this.getOne(queryWrapper);
     }
-
-
 }

+ 13 - 7
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/BytedanceCampaignDailyReportServiceImpl.java

@@ -1,21 +1,27 @@
 package cn.com.ctop.toutiao.modules.report.service.impl;
 
 import cn.com.ctop.toutiao.modules.report.entity.BytedanceCampaignDailyReport;
+import cn.com.ctop.toutiao.modules.report.entity.BytedanceCheckReportTaskInfo;
 import cn.com.ctop.toutiao.modules.report.mapper.BytedanceCampaignDailyReportMapper;
 import cn.com.ctop.toutiao.modules.report.service.IBytedanceCampaignDailyReportService;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import org.springframework.context.annotation.Primary;
 import org.springframework.stereotype.Service;
 
+import javax.annotation.Resource;
+
 /**
- * @Description: 广告组日报表信息
- * @Author: jeecg-boot
- * @Date: 2019-09-11
- * @Version: V1.0
+ * 广告组日报表信息
+ * @author jeecg-boot
+ * @date 2019-09-11
+ * @version V1.0
  */
 @Service
-@Primary
-
 public class BytedanceCampaignDailyReportServiceImpl extends ServiceImpl<BytedanceCampaignDailyReportMapper, BytedanceCampaignDailyReport> implements IBytedanceCampaignDailyReportService {
-
+    @Resource
+    private BytedanceCampaignDailyReportMapper campaignDailyReportMapper;
+    @Override
+    public BytedanceCheckReportTaskInfo checkCampaignCostByParams(Long accountId, String stateDate) {
+        return campaignDailyReportMapper.checkCampaignCostByParams(accountId,stateDate);
+    }
 }

+ 159 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/BytedanceCheckReportTaskInfoServiceImpl.java

@@ -0,0 +1,159 @@
+package cn.com.ctop.toutiao.modules.report.service.impl;
+
+import cn.com.ctop.common.module.entity.CtopOauthToken;
+import cn.com.ctop.common.module.service.ICtopOauthTokenService;
+import cn.com.ctop.common.module.utils.CtopAdConstant;
+import cn.com.ctop.toutiao.modules.report.entity.BytedanceAdvertiserDailyReport;
+import cn.com.ctop.toutiao.modules.report.entity.BytedanceCheckReportTaskInfo;
+import cn.com.ctop.toutiao.modules.report.mapper.BytedanceCheckReportTaskInfoMapper;
+import cn.com.ctop.toutiao.modules.report.service.*;
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.jeecg.common.util.DateUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.math.BigDecimal;
+import java.util.Date;
+import java.util.List;
+
+/**
+ * 头条数据检查任务记录
+ * @author jeecg-boot
+ * 2021-06-07
+ * @version V1.0
+ */
+@Service
+public class BytedanceCheckReportTaskInfoServiceImpl extends ServiceImpl<BytedanceCheckReportTaskInfoMapper, BytedanceCheckReportTaskInfo> implements IBytedanceCheckReportTaskInfoService {
+    @Autowired
+    private ICtopOauthTokenService tokenService;
+    @Autowired
+    private IBytedanceAdvertiserDailyReportService advertiserDailyReportService;
+    @Autowired
+    private IBytedanceCampaignDailyReportService campaignDailyReportService;
+    @Autowired
+    private IReportService reportService;
+    @Autowired
+    private IBytedanceReportMaterialDailyService materialDailyService;
+    @Autowired
+    private IBytedanceReportService bytedanceReportService;
+
+    @Override
+    public void initTask(String date) {
+        List<CtopOauthToken> tokens = tokenService.selectToutiaoToken();
+        if(null != tokens&&!tokens.isEmpty()){
+            tokens.forEach(token->{
+                this.saveOrUpdateTaskInfo(token,date);
+            });
+        }
+    }
+
+    private void saveOrUpdateTaskInfo(CtopOauthToken token, String date) {
+        BytedanceCheckReportTaskInfo taskInfo = this.getInfoByParams(token.getAccountId(),date);
+        if(null == taskInfo){
+            taskInfo = new BytedanceCheckReportTaskInfo(token,date);
+            this.save(taskInfo);
+        }
+    }
+
+    @Override
+    public BytedanceCheckReportTaskInfo getInfoByParams(Long accountId, String date) {
+        QueryWrapper<BytedanceCheckReportTaskInfo> queryWrapper = new QueryWrapper<>();
+        if(null!=accountId&&accountId!=0){
+            queryWrapper.eq("account_id",accountId);
+        }
+        if(null!=date&&!"".equals(date.trim())){
+            queryWrapper.eq("state_date",date);
+        }
+        queryWrapper.orderByDesc("id").last("limit 1");
+        return this.getOne(queryWrapper);
+    }
+
+    @Override
+    public List<BytedanceCheckReportTaskInfo> getListByParams(Long accountId, String date,Integer status) {
+        QueryWrapper<BytedanceCheckReportTaskInfo> queryWrapper = new QueryWrapper<>();
+        if(null!=accountId&&accountId!=0){
+            queryWrapper.eq("account_id",accountId);
+        }
+        if(null!=date&&!"".equals(date.trim())){
+            queryWrapper.eq("state_date",date);
+        }
+        if(null!=status){
+            queryWrapper.eq("status",status);
+        }
+        queryWrapper.orderByDesc("id");
+        return this.list(queryWrapper);
+    }
+
+    @Override
+    public void checkCost(BytedanceCheckReportTaskInfo taskInfo){
+        CtopOauthToken token = tokenService.getTokenByAccountId(taskInfo.getAccountId());
+        if(null == token){
+            taskInfo.setStatus(-1);
+            taskInfo.setDescription("token数据不存在!");
+            taskInfo.setCostStatus(-1);
+            taskInfo.setUpdateTime(new Date());
+            this.saveOrUpdate(taskInfo);
+            return;
+        }
+        //1:获取账户消耗数据
+        BytedanceAdvertiserDailyReport dailyReport = advertiserDailyReportService.getByParams(taskInfo.getAccountId(),taskInfo.getStateDate());
+        if(null!=dailyReport){
+            taskInfo.setAccountCost(dailyReport.getCost());
+        }
+        //2:获取组消耗数据
+        BytedanceCheckReportTaskInfo campaignTaskInfo = campaignDailyReportService.checkCampaignCostByParams(taskInfo.getAccountId(),taskInfo.getStateDate());
+        if(null!=campaignTaskInfo){
+            taskInfo.setCampaignCost(campaignTaskInfo.getCampaignCost());
+        }
+        //3:获取素材消耗数据
+        BytedanceCheckReportTaskInfo matTaskInfo = materialDailyService.checkMatCostByParams(taskInfo.getAccountId(),taskInfo.getStateDate());
+        if(null!=matTaskInfo){
+            taskInfo.setMatCost(matTaskInfo.getMatCost());
+        }
+        //4:获取图片消耗数据
+        BytedanceCheckReportTaskInfo imageTaskInfo = materialDailyService.checkImageCostByParams(taskInfo.getAccountId(),taskInfo.getStateDate());
+        if(null!=imageTaskInfo){
+            taskInfo.setImageCost(imageTaskInfo.getImageCost());
+        }
+        if(taskInfo.getAccountCost().compareTo(taskInfo.getCampaignCost())!=0){
+            taskInfo.setDescription("账户消耗与组消耗不匹配");
+            taskInfo.setStatus(50001);
+            taskInfo.setCostStatus(-1);
+            this.saveOrUpdate(taskInfo);
+            return;
+        }
+        if(taskInfo.getImageCost().add(taskInfo.getMatCost()).subtract(taskInfo.getAccountCost()).abs().compareTo(new BigDecimal("10"))>0){
+            taskInfo.setDescription("账户消耗与素材消耗不匹配");
+            taskInfo.setStatus(50001);
+            taskInfo.setCostStatus(-2);
+            this.saveOrUpdate(taskInfo);
+            return;
+        }
+
+        taskInfo.setDescription("消耗数据正常");
+        taskInfo.setStatus(1);
+        taskInfo.setCostStatus(1);
+        this.saveOrUpdate(taskInfo);
+    }
+
+    @Override
+    public void reloadData(BytedanceCheckReportTaskInfo taskInfo){
+        CtopOauthToken token = tokenService.getTokenByAccountId(taskInfo.getAccountId());
+        BigDecimal accountCost = taskInfo.getAccountCost();
+        BigDecimal campaignCost = taskInfo.getCampaignCost();
+        BigDecimal imageCost = taskInfo.getImageCost();
+        BigDecimal matCost = taskInfo.getMatCost();
+        //账户消耗!=组消耗
+        if(accountCost.compareTo(campaignCost)!=0){
+            reportService.getAdvertiserReport(token, DateUtils.parseDate(taskInfo.getStateDate(),"yyyy-MM-dd"),DateUtils.parseDate(taskInfo.getStateDate(),"yyyy-MM-dd"), CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
+            reportService.getAdvertiserCampaignReport(token, DateUtils.parseDate(taskInfo.getStateDate(),"yyyy-MM-dd"),DateUtils.parseDate(taskInfo.getStateDate(),"yyyy-MM-dd"), CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
+        }
+        if(imageCost.add(matCost).subtract(campaignCost).abs().compareTo(BigDecimal.ZERO)>0){
+            reportService.getAdvertiserReport(token, DateUtils.parseDate(taskInfo.getStateDate(),"yyyy-MM-dd"),DateUtils.parseDate(taskInfo.getStateDate(),"yyyy-MM-dd"), CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
+            reportService.getAdvertiserCampaignReport(token, DateUtils.parseDate(taskInfo.getStateDate(),"yyyy-MM-dd"),DateUtils.parseDate(taskInfo.getStateDate(),"yyyy-MM-dd"), CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
+            bytedanceReportService.bytedanceMaterialReport(token, taskInfo.getStateDate(), taskInfo.getStateDate());
+        }
+    }
+
+}

+ 13 - 2
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/BytedanceReportMaterialDailyServiceImpl.java

@@ -1,16 +1,17 @@
 package cn.com.ctop.toutiao.modules.report.service.impl;
 
 import cn.com.ctop.toutiao.modules.material.vo.BytedanceVideoVo;
+import cn.com.ctop.toutiao.modules.report.entity.BytedanceCheckReportTaskInfo;
 import cn.com.ctop.toutiao.modules.report.entity.BytedanceReportMaterialDaily;
 import cn.com.ctop.toutiao.modules.report.entity.BytedanceVideoEtlInfo;
 import cn.com.ctop.toutiao.modules.report.mapper.BytedanceReportMaterialDailyMapper;
 import cn.com.ctop.toutiao.modules.report.service.IBytedanceReportMaterialDailyService;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.annotation.Primary;
 import org.springframework.stereotype.Service;
 
+import javax.annotation.Resource;
 import java.util.List;
 
 /**
@@ -23,7 +24,7 @@ import java.util.List;
 @Primary
 
 public class BytedanceReportMaterialDailyServiceImpl extends ServiceImpl<BytedanceReportMaterialDailyMapper, BytedanceReportMaterialDaily> implements IBytedanceReportMaterialDailyService {
-    @Autowired
+    @Resource
     private BytedanceReportMaterialDailyMapper bytedanceReportMaterialDailyMapper;
     @Override
     public List<BytedanceReportMaterialDaily> getListByParams(String date) {
@@ -50,5 +51,15 @@ public class BytedanceReportMaterialDailyServiceImpl extends ServiceImpl<Bytedan
         return bytedanceReportMaterialDailyMapper.getEtlDataByParams(signature,date);
     }
 
+    @Override
+    public BytedanceCheckReportTaskInfo checkMatCostByParams(Long accountId, String stateDate) {
+        return bytedanceReportMaterialDailyMapper.checkMatCostByParams(accountId,stateDate);
+    }
+
+    @Override
+    public BytedanceCheckReportTaskInfo checkImageCostByParams(Long accountId, String stateDate) {
+        return bytedanceReportMaterialDailyMapper.checkImageCostByParams(accountId,stateDate);
+    }
+
 
 }

+ 2 - 0
pom.xml

@@ -66,11 +66,13 @@
 
         <repository>
             <id>maven-3rdparty</id>
+            <name>maven-3rdparty</name>
             <url>https://packages.atlassian.com/maven-3rdparty/</url>
         </repository>
 
         <repository>
             <id>ctop</id>
+            <name>ctop</name>
             <url>http://118.24.244.213:8081/repository/3rd_part</url>
         </repository>
     </repositories>