songyh 4 rokov pred
rodič
commit
2aaf9858f0
18 zmenil súbory, kde vykonal 450 pridanie a 228 odobranie
  1. 1 0
      jeecg-boot-module-system/src/main/java/org/jeecg/config/ShiroConfig.java
  2. 1 1
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/TestController.java
  3. 48 13
      jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java
  4. 3 0
      module-common/src/main/java/cn/com/ctop/common/module/service/ISysUserService.java
  5. 6 0
      module-common/src/main/java/cn/com/ctop/common/module/service/impl/SysUserServiceImpl.java
  6. 146 146
      module-common/src/main/java/cn/com/ctop/common/module/utils/JavacvUtil.java
  7. 21 1
      module-job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/ByteDanceRefreshTokenJob.java
  8. 73 0
      module-job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedanceCheckAccountReportJob.java
  9. 19 1
      module-job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedanceEtlVideoJob.java
  10. 10 4
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/material/service/impl/EtlKuaishouVideoInfoServiceImpl.java
  11. 8 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/mapper/ETLReportBytedanceVideoMapper.java
  12. 55 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/mapper/xml/ETLReportBytedanceVideoMapper.xml
  13. 6 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/service/IETLReportBytedanceVideoService.java
  14. 30 51
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/service/impl/ETLReportBytedanceVideoServiceImpl.java
  15. 3 1
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/entity/BytedanceCheckReportTaskInfo.java
  16. 1 1
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/IBytedanceCheckReportTaskInfoService.java
  17. 15 7
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/BytedanceCheckReportTaskInfoServiceImpl.java
  18. 4 2
      pom.xml

+ 1 - 0
jeecg-boot-module-system/src/main/java/org/jeecg/config/ShiroConfig.java

@@ -103,6 +103,7 @@ public class ShiroConfig {
         filterChainDefinitionMap.put("/ctop/byteDance/homePage/report/*", "anon");
         filterChainDefinitionMap.put("/ctop/material/top/cost", "anon");
         filterChainDefinitionMap.put("/dimension/report/*", "anon");
+        filterChainDefinitionMap.put("/userAllocation/updateAuthName", "anon");
 
         filterChainDefinitionMap.put("/v1/**", "anon");
 

+ 1 - 1
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/TestController.java

@@ -1557,7 +1557,7 @@ public class TestController {
     @GetMapping("testCheckCost")
     public Map<String, Object> testCheckCost(String date) throws ParseException {
         Map<String, Object> result = new HashMap<>();
-        List<BytedanceCheckReportTaskInfo>taskInfos = checkReportTaskInfoService.getListByParams(null,"2021-06-01");
+        List<BytedanceCheckReportTaskInfo>taskInfos = checkReportTaskInfoService.getListByParams(null,"2021-06-01",0);
         if(null!=taskInfos&&!taskInfos.isEmpty()){
             for (BytedanceCheckReportTaskInfo taskInfo:taskInfos) {
                 checkReportTaskInfoService.checkCost(taskInfo);

+ 48 - 13
jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java

@@ -16,6 +16,7 @@ import cn.com.ctop.toutiao.modules.material.service.IByteDanceAdvertiserDataServ
 import cn.com.ctop.toutiao.modules.material.service.IByteDanceCampaignService;
 import cn.com.ctop.toutiao.modules.report.entity.BytedanceCheckReportTaskInfo;
 import cn.com.ctop.toutiao.modules.report.service.*;
+import com.xxl.job.core.context.XxlJobHelper;
 import lombok.extern.slf4j.Slf4j;
 import org.jeecg.common.util.DateUtils;
 import org.jeecg.common.util.RedisUtil;
@@ -216,13 +217,13 @@ public class SampleTest {
         etlKuaishouAccountMaterialReportDailyMapper.etlKuaishouAccountMaterialReportDailyData(DateUtils.formatDate());
     }
 
+    @Autowired
+    private IBytedanceReportService bytedanceReportService;
 
     @Test
     public void testLoadBytedanceCampaign() {
-        List<CtopOauthToken> tokens = tokenService.selectToutiaoToken();
-        for (CtopOauthToken token : tokens) {
-            campaignService.getAdvertiserCampaign(token, null, null);
-        }
+        CtopOauthToken token = tokenService.getTokenByAccountId(1681669009084429L);
+        bytedanceReportService.bytedanceMaterialReport(token, "2021-05-01", "2021-05-06");
     }
 
 
@@ -245,16 +246,50 @@ public class SampleTest {
 
     @Test
     public void checkCostData(){
-        //1:初始化
-//        checkReportTaskInfoService.initTask("2021-06-01");
-        //2: checkCost
-        List<BytedanceCheckReportTaskInfo>taskInfos = checkReportTaskInfoService.getListByParams(null,"2021-06-01");
-        if(null!=taskInfos&&!taskInfos.isEmpty()){
-            for (BytedanceCheckReportTaskInfo task:taskInfos) {
-                checkReportTaskInfoService.checkCost(task);
+        String date = "2021-06-01";
+        for (int i=0;i<14;i++){
+            String getDate = DateUtils.addDay(date,i);
+            //1:初始化
+            checkReportTaskInfoService.initTask(getDate);
+            //2: checkCost
+            List<BytedanceCheckReportTaskInfo>taskInfos = checkReportTaskInfoService.getListByParams(null,getDate,0);
+            if(null!=taskInfos&&!taskInfos.isEmpty()){
+                for (BytedanceCheckReportTaskInfo task:taskInfos) {
+                    checkReportTaskInfoService.checkCost(task);
+                }
+                log.info("{}:数据获取完成",getDate);
             }
-            System.out.println("数据获取完成");
         }
-        //3:重复异常数据
+// 3:重复异常数据
+    }
+
+    @Test
+    public void refreshReportData()throws Exception{
+        //1:查询异常任务数据
+        List<BytedanceCheckReportTaskInfo>taskInfos = checkReportTaskInfoService.getListByParams(null,null,50001);
+        if(null!=taskInfos&&!taskInfos.isEmpty()){
+            countDownLatch = new CountDownLatch(taskInfos.size());
+            taskInfos.forEach(taskInfo -> {
+                try {
+                    checkReportTaskInfoService.reloadData(taskInfo);
+                    checkReportTaskInfoService.checkCost(taskInfo);
+                }catch (Exception e){
+                }finally {
+                    countDownLatch.countDown();
+                }
+            });
+            countDownLatch.await();
+            XxlJobHelper.log("{}数据更新完成");
+        }
     }
+    
+    @Test
+    public void testfreshBytedanceUrl(){
+        etlReportBytedanceVideoService.etlBytedanceVideoClipIdJob();
+        etlReportBytedanceVideoService.etlBytedanceVideoPlanInfoJob();
+        etlReportBytedanceVideoService.etlBytedanceVideoShotInfoJob();
+    }
+
+    @Autowired
+    private IETLReportBytedanceVideoService etlReportBytedanceVideoService;
 }

+ 3 - 0
module-common/src/main/java/cn/com/ctop/common/module/service/ISysUserService.java

@@ -1,5 +1,6 @@
 package cn.com.ctop.common.module.service;
 
+import cn.com.ctop.common.module.entity.SysRole;
 import cn.com.ctop.common.module.entity.SysUser;
 import cn.com.ctop.common.module.model.SysUserSysDepartModel;
 import cn.com.ctop.common.module.vo.SysUserVo;
@@ -269,4 +270,6 @@ public interface ISysUserService extends IService<SysUser> {
     List<SysUser> queryPageList(SysUser user);
 
     Map<String, String> getUserMapByMap(Map<String, String> userIdMap);
+
+    String getRoleCodeByUserId(String userId);
 }

+ 6 - 0
module-common/src/main/java/cn/com/ctop/common/module/service/impl/SysUserServiceImpl.java

@@ -308,6 +308,12 @@ public class SysUserServiceImpl extends ServiceImpl<SysUserMapper, SysUser> impl
     }
 
     @Override
+    public String getRoleCodeByUserId(String userId) {
+        return sysRoleMapper.getRoleCodeByUserId(userId);
+    }
+
+
+    @Override
     public Map<String, String> getDepNamesByUserIds(List<String> userIds) {
         List<SysUserDepVo> list = this.baseMapper.getDepNamesByUserIds(userIds);
         Map<String, String> res = new HashMap<String, String>();

+ 146 - 146
module-common/src/main/java/cn/com/ctop/common/module/utils/JavacvUtil.java

@@ -1,146 +1,146 @@
-package cn.com.ctop.common.module.utils;
-
-import com.google.common.collect.Lists;
-import org.bytedeco.javacpp.opencv_core;
-import org.bytedeco.javacv.FFmpegFrameGrabber;
-import org.bytedeco.javacv.Frame;
-import org.bytedeco.javacv.Java2DFrameConverter;
-import org.opencv.core.CvType;
-import org.opencv.core.Mat;
-import org.opencv.core.MatOfByte;
-import org.opencv.imgcodecs.Imgcodecs;
-import org.opencv.imgproc.Imgproc;
-
-import javax.imageio.ImageIO;
-import java.awt.*;
-import java.awt.image.BufferedImage;
-import java.awt.image.DataBufferByte;
-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.List;
-
-public class JavacvUtil {
-    /**
-     * 视频文件指定时间段的帧截取
-     * @param file
-     * @param start
-     * @param end
-     */
-    public static List<File> videoIntercept(File file, Integer start, Integer end) {
-        Frame frame = null;
-        List<File> files = Lists.newArrayList();
-        FFmpegFrameGrabber fFmpegFrameGrabber = new FFmpegFrameGrabber(file);
-        String filePath = "D://video//images//";
-        String fileTargetName = "movie";
-        try {
-            fFmpegFrameGrabber.start();
-            int ftp = fFmpegFrameGrabber.getLengthInFrames();
-            for (int i=0 ; i < ftp ; i++){
-                if( i >= start && i <= end){
-                    frame = fFmpegFrameGrabber.grabImage();
-                    doExecuteFrame(frame, filePath, fileTargetName, i ,files);
-                }
-            }
-            fFmpegFrameGrabber.stop();
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-        return files;
-    }
-
-    public static void doExecuteFrame(Frame frame, String targetFilePath, String targetFileName, int index ,List<File> files) {
-        if ( frame == null || frame.image == null) {
-            return;
-        }
-        Java2DFrameConverter converter = new Java2DFrameConverter();
-        String imageMat = "jpg";
-        String fileName = targetFilePath + targetFileName + "_" + index + "." + imageMat;
-        BufferedImage bi = converter.getBufferedImage(frame);
-
-//        Mat img = Imgcodecs.imread("");
-//        Mat imgHSV = new Mat(img.rows(), img.cols(), CvType.CV_8UC3);
-//        Mat img2 = new Mat(img.rows(), img.cols(), CvType.CV_8UC3);
-
-        //转成HSV空间
-//        Imgproc.cvtColor(img, imgHSV, Imgproc.COLOR_BGR2HSV);
-//        //转回BGR空间
-//        Imgproc.cvtColor(imgHSV, img2, Imgproc.COLOR_HSV2BGR);
-
-        File output = new File(fileName);
-        files.add(output);
-        try{
-            ImageIO.write(bi, imageMat, output);
-        } catch (IOException e) {
-            e.printStackTrace();
-        }
-    }
-
-    public static void main(String[] args) {
-        List<File> files = videoIntercept(new File("C:\\Users\\宋英豪\\Desktop\\nfpHmswW.mp4"), 10, 100);
-    }
-
-    /**
-     * Mat转换成BufferedImage
-     *
-     * @param matrix
-     *            要转换的Mat
-     * @param fileExtension
-     *            格式为 ".jpg", ".png", etc
-     * @return
-     */
-    public static BufferedImage Mat2BufImg (Mat matrix, String fileExtension) {
-        // convert the matrix into a matrix of bytes appropriate for
-        // this file extension
-        MatOfByte mob = new MatOfByte();
-        Imgcodecs.imencode(fileExtension, matrix, mob);
-        // convert the "matrix of bytes" into a byte array
-        byte[] byteArray = mob.toArray();
-        BufferedImage bufImage = null;
-        try {
-            InputStream in = new ByteArrayInputStream(byteArray);
-            bufImage = ImageIO.read(in);
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-        return bufImage;
-    }
-
-    /**
-     * BufferedImage转换成Mat
-     *
-     * @param original
-     *            要转换的BufferedImage
-     * @param imgType
-     *            bufferedImage的类型 如 BufferedImage.TYPE_3BYTE_BGR
-     * @param matType
-     *            转换成mat的type 如 CvType.CV_8UC3
-     */
-    public static Mat BufImg2Mat (BufferedImage original, int imgType, int matType) {
-        if (original == null) {
-            throw new IllegalArgumentException("original == null");
-        }
-
-        // Don't convert if it already has correct type
-        if (original.getType() != imgType) {
-
-            // Create a buffered image
-            BufferedImage image = new BufferedImage(original.getWidth(), original.getHeight(), imgType);
-
-            // Draw the image onto the new buffer
-            Graphics2D g = image.createGraphics();
-            try {
-                g.setComposite(AlphaComposite.Src);
-                g.drawImage(original, 0, 0, null);
-            } finally {
-                g.dispose();
-            }
-        }
-
-        byte[] pixels = ((DataBufferByte) original.getRaster().getDataBuffer()).getData();
-        Mat mat = Mat.eye(original.getHeight(), original.getWidth(), matType);
-        mat.put(0, 0, pixels);
-        return mat;
-    }
-}
+//package cn.com.ctop.common.module.utils;
+//
+//import com.google.common.collect.Lists;
+//import org.bytedeco.javacpp.opencv_core;
+//import org.bytedeco.javacv.FFmpegFrameGrabber;
+//import org.bytedeco.javacv.Frame;
+//import org.bytedeco.javacv.Java2DFrameConverter;
+//import org.opencv.core.CvType;
+//import org.opencv.core.Mat;
+//import org.opencv.core.MatOfByte;
+//import org.opencv.imgcodecs.Imgcodecs;
+//import org.opencv.imgproc.Imgproc;
+//
+//import javax.imageio.ImageIO;
+//import java.awt.*;
+//import java.awt.image.BufferedImage;
+//import java.awt.image.DataBufferByte;
+//import java.io.ByteArrayInputStream;
+//import java.io.File;
+//import java.io.IOException;
+//import java.io.InputStream;
+//import java.util.List;
+//
+//public class JavacvUtil {
+//    /**
+//     * 视频文件指定时间段的帧截取
+//     * @param file
+//     * @param start
+//     * @param end
+//     */
+//    public static List<File> videoIntercept(File file, Integer start, Integer end) {
+//        Frame frame = null;
+//        List<File> files = Lists.newArrayList();
+//        FFmpegFrameGrabber fFmpegFrameGrabber = new FFmpegFrameGrabber(file);
+//        String filePath = "D://video//images//";
+//        String fileTargetName = "movie";
+//        try {
+//            fFmpegFrameGrabber.start();
+//            int ftp = fFmpegFrameGrabber.getLengthInFrames();
+//            for (int i=0 ; i < ftp ; i++){
+//                if( i >= start && i <= end){
+//                    frame = fFmpegFrameGrabber.grabImage();
+//                    doExecuteFrame(frame, filePath, fileTargetName, i ,files);
+//                }
+//            }
+//            fFmpegFrameGrabber.stop();
+//        } catch (Exception e) {
+//            e.printStackTrace();
+//        }
+//        return files;
+//    }
+//
+//    public static void doExecuteFrame(Frame frame, String targetFilePath, String targetFileName, int index ,List<File> files) {
+//        if ( frame == null || frame.image == null) {
+//            return;
+//        }
+//        Java2DFrameConverter converter = new Java2DFrameConverter();
+//        String imageMat = "jpg";
+//        String fileName = targetFilePath + targetFileName + "_" + index + "." + imageMat;
+//        BufferedImage bi = converter.getBufferedImage(frame);
+//
+////        Mat img = Imgcodecs.imread("");
+////        Mat imgHSV = new Mat(img.rows(), img.cols(), CvType.CV_8UC3);
+////        Mat img2 = new Mat(img.rows(), img.cols(), CvType.CV_8UC3);
+//
+//        //转成HSV空间
+////        Imgproc.cvtColor(img, imgHSV, Imgproc.COLOR_BGR2HSV);
+////        //转回BGR空间
+////        Imgproc.cvtColor(imgHSV, img2, Imgproc.COLOR_HSV2BGR);
+//
+//        File output = new File(fileName);
+//        files.add(output);
+//        try{
+//            ImageIO.write(bi, imageMat, output);
+//        } catch (IOException e) {
+//            e.printStackTrace();
+//        }
+//    }
+//
+//    public static void main(String[] args) {
+//        List<File> files = videoIntercept(new File("C:\\Users\\宋英豪\\Desktop\\nfpHmswW.mp4"), 10, 100);
+//    }
+//
+//    /**
+//     * Mat转换成BufferedImage
+//     *
+//     * @param matrix
+//     *            要转换的Mat
+//     * @param fileExtension
+//     *            格式为 ".jpg", ".png", etc
+//     * @return
+//     */
+//    public static BufferedImage Mat2BufImg (Mat matrix, String fileExtension) {
+//        // convert the matrix into a matrix of bytes appropriate for
+//        // this file extension
+//        MatOfByte mob = new MatOfByte();
+//        Imgcodecs.imencode(fileExtension, matrix, mob);
+//        // convert the "matrix of bytes" into a byte array
+//        byte[] byteArray = mob.toArray();
+//        BufferedImage bufImage = null;
+//        try {
+//            InputStream in = new ByteArrayInputStream(byteArray);
+//            bufImage = ImageIO.read(in);
+//        } catch (Exception e) {
+//            e.printStackTrace();
+//        }
+//        return bufImage;
+//    }
+//
+//    /**
+//     * BufferedImage转换成Mat
+//     *
+//     * @param original
+//     *            要转换的BufferedImage
+//     * @param imgType
+//     *            bufferedImage的类型 如 BufferedImage.TYPE_3BYTE_BGR
+//     * @param matType
+//     *            转换成mat的type 如 CvType.CV_8UC3
+//     */
+//    public static Mat BufImg2Mat (BufferedImage original, int imgType, int matType) {
+//        if (original == null) {
+//            throw new IllegalArgumentException("original == null");
+//        }
+//
+//        // Don't convert if it already has correct type
+//        if (original.getType() != imgType) {
+//
+//            // Create a buffered image
+//            BufferedImage image = new BufferedImage(original.getWidth(), original.getHeight(), imgType);
+//
+//            // Draw the image onto the new buffer
+//            Graphics2D g = image.createGraphics();
+//            try {
+//                g.setComposite(AlphaComposite.Src);
+//                g.drawImage(original, 0, 0, null);
+//            } finally {
+//                g.dispose();
+//            }
+//        }
+//
+//        byte[] pixels = ((DataBufferByte) original.getRaster().getDataBuffer()).getData();
+//        Mat mat = Mat.eye(original.getHeight(), original.getWidth(), matType);
+//        mat.put(0, 0, pixels);
+//        return mat;
+//    }
+//}

+ 21 - 1
module-job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/ByteDanceRefreshTokenJob.java

@@ -1,16 +1,22 @@
 package cn.com.ctop.job.bytedance.handler;
 
+import cn.com.ctop.common.module.entity.UserAllocation;
 import cn.com.ctop.common.module.service.IRefreshTokenService;
+import cn.com.ctop.common.module.service.IUserAllocationService;
+import cn.com.ctop.common.module.utils.HttpUtils;
 import com.xxl.job.core.handler.annotation.XxlJob;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
+import java.util.HashMap;
+import java.util.List;
+
 @Component
 public class ByteDanceRefreshTokenJob {
 
     @Autowired
     private IRefreshTokenService refreshTokenService;
-
+    private static final String REFRESH_AUTHNAME_URL = "http://api.tjyourong.com.cn/jeecg-boot/ctop/userAllocation/updateAuthName?";
     /**
      * 刷新token
      * @throws Exception
@@ -19,4 +25,18 @@ public class ByteDanceRefreshTokenJob {
     public void execute() throws Exception {
         refreshTokenService.byteDanceRefresh(null);
     }
+
+    @Autowired
+    private IUserAllocationService userAllocationService;
+    @XxlJob("refreshAuthName")
+    public void refreshAuthName() throws Exception {
+        List<UserAllocation> userAllocationList = userAllocationService.listByMediaId(null,0);
+        if(null!=userAllocationList&&!userAllocationList.isEmpty()){
+            userAllocationList.forEach(userAllocation -> {
+                String url = REFRESH_AUTHNAME_URL+"accountId="+userAllocation.getAccountId()+"&mediaId="+userAllocation.getMediaId();
+                HttpUtils.httpGet(url,new HashMap<>(),new HashMap<>());
+            });
+        }
+        refreshTokenService.byteDanceRefresh(null);
+    }
 }

+ 73 - 0
module-job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedanceCheckAccountReportJob.java

@@ -0,0 +1,73 @@
+package cn.com.ctop.job.bytedance.handler;
+
+import cn.com.ctop.common.module.utils.Check;
+import cn.com.ctop.toutiao.modules.report.entity.BytedanceCheckReportTaskInfo;
+import cn.com.ctop.toutiao.modules.report.service.IBytedanceCheckReportTaskInfoService;
+import com.xxl.job.core.context.XxlJobHelper;
+import com.xxl.job.core.handler.annotation.XxlJob;
+import org.jeecg.common.util.DateUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.Resource;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+@Component
+public class BytedanceCheckAccountReportJob {
+    private ExecutorService executorService = Executors.newFixedThreadPool(10);
+    private CountDownLatch countDownLatch;
+    @Autowired
+    private IBytedanceCheckReportTaskInfoService checkReportTaskInfoService;
+    @XxlJob("initCheckReportJob")
+    public void initCheckReportJob() throws Exception {
+        String params = DateUtils.formatDate();
+        String date = XxlJobHelper.getJobParam();
+        if (date==null || date.trim().length()==0) {
+            date = params;
+        }
+        checkReportTaskInfoService.initTask(date);
+        //查询初始化的任务
+        List<BytedanceCheckReportTaskInfo>taskInfos = checkReportTaskInfoService.getListByParams(null,date,0);
+        if(null!=taskInfos&&!taskInfos.isEmpty()){
+            countDownLatch = new CountDownLatch(taskInfos.size());
+            taskInfos.forEach(taskInfo -> {
+                try {
+                    checkReportTaskInfoService.checkCost(taskInfo);
+                }catch (Exception e){
+                }finally {
+                    countDownLatch.countDown();
+                }
+            });
+            countDownLatch.await();
+            XxlJobHelper.log("{}数据初始化完成",date);
+        }
+    }
+
+    @XxlJob("refreshCheckReportJob")
+    public void refreshCheckReportJob() throws Exception {
+        String params = DateUtils.formatDate();
+        String date = XxlJobHelper.getJobParam();
+        if (date==null || date.trim().length()==0) {
+            date = params;
+        }
+        //1:查询异常任务数据
+        List<BytedanceCheckReportTaskInfo>taskInfos = checkReportTaskInfoService.getListByParams(null,date,50001);
+        if(null!=taskInfos&&!taskInfos.isEmpty()){
+            countDownLatch = new CountDownLatch(taskInfos.size());
+            taskInfos.forEach(taskInfo -> {
+                try {
+                    checkReportTaskInfoService.reloadData(taskInfo);
+                    checkReportTaskInfoService.checkCost(taskInfo);
+                }catch (Exception e){
+                }finally {
+                    countDownLatch.countDown();
+                }
+            });
+            countDownLatch.await();
+            XxlJobHelper.log("{}数据更新完成",date);
+        }
+    }
+}

+ 19 - 1
module-job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedanceEtlVideoJob.java

@@ -1,5 +1,6 @@
 package cn.com.ctop.job.bytedance.handler;
 
+import cn.com.ctop.toutiao.modules.link.service.IETLReportBytedanceVideoService;
 import cn.com.ctop.toutiao.modules.report.service.IBytedanceVideoEtlInfoService;
 import cn.com.ctop.toutiao.modules.report.service.IEtlBytedanceReportVideoDailyService;
 import com.xxl.job.core.handler.annotation.XxlJob;
@@ -42,7 +43,24 @@ public class BytedanceEtlVideoJob {
         etlBytedanceReportVideoDailyService.updBytedanceVideo();
     }
 
-    @XxlJob("etlBytedanceVideoJob")
+    @Autowired
+    private IETLReportBytedanceVideoService etlReportBytedanceVideoService;
+    @XxlJob("etlBytedanceVideoClipInfoJob")
+    public void etlBytedanceVideoClipInfoJob() throws Exception {
+        etlReportBytedanceVideoService.etlBytedanceVideoClipIdJob();
+    }
+
+    @XxlJob("etlBytedanceVideoPlanInfoJob")
+    public void etlBytedanceVideoPlanInfoJob() throws Exception {
+        etlReportBytedanceVideoService.etlBytedanceVideoPlanInfoJob();
+    }
+
+    @XxlJob("etlBytedanceVideoShotInfoJob")
+    public void etlBytedanceVideoShotInfoJob() throws Exception {
+        etlReportBytedanceVideoService.etlBytedanceVideoShotInfoJob();
+    }
+
+    @XxlJob("formatBytedance")
     public void etlBytedanceVideoJob() throws Exception {
         etlBytedanceReportVideoDailyService.etlBytedanceVideo();
     }

+ 10 - 4
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/material/service/impl/EtlKuaishouVideoInfoServiceImpl.java

@@ -2,6 +2,7 @@ package cn.com.ctop.kuaishou.modules.material.service.impl;
 
 import cn.com.ctop.common.module.entity.MaterialInfo;
 import cn.com.ctop.common.module.entity.SysUser;
+import cn.com.ctop.common.module.mapper.SysUserMapper;
 import cn.com.ctop.common.module.service.IMaterialAscriptionService;
 import cn.com.ctop.common.module.service.IMaterialInfoService;
 import cn.com.ctop.common.module.service.ISysUserService;
@@ -10,6 +11,7 @@ import cn.com.ctop.kuaishou.modules.batch.service.IKuaiShouVideoGetService;
 import cn.com.ctop.kuaishou.modules.material.entity.EtlKuaishouVideoInfo;
 import cn.com.ctop.kuaishou.modules.material.mapper.EtlKuaishouVideoInfoMapper;
 import cn.com.ctop.kuaishou.modules.material.service.IEtlKuaishouVideoInfoService;
+import cn.com.ctop.kuaishou.modules.report.mapper.KuaishouReportDailyAccountMapper;
 import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
@@ -44,9 +46,7 @@ public class EtlKuaishouVideoInfoServiceImpl extends ServiceImpl<EtlKuaishouVide
     private ISysUserService sysUserService;
     @Autowired
     private IMaterialInfoService materialInfoService;
-
     ExecutorService executorService = Executors.newFixedThreadPool(200);
-
     @Override
     public void etlKuaishouVideoInfo(String date) {
         //1:查询当前日期内的所有存在md5码信息的视频数据
@@ -82,8 +82,14 @@ public class EtlKuaishouVideoInfoServiceImpl extends ServiceImpl<EtlKuaishouVide
                                 //根据id查询剪辑所属的设计负责人
                                 SysUser user = sysUserService.getById(clipId);
                                 if(null!=user){
-                                    etlKuaishouVideoInfo.setDesignTeamLeaderId(user.getLeaderId());
-                                    etlKuaishouVideoInfo.setDesignTeamLeaderName(user.getLeaderName());
+                                    String roleCode = sysUserService.getRoleCodeByUserId(user.getId());
+                                    if(null!=roleCode&&roleCode.trim().equals("designTeamLeader")){
+                                        etlKuaishouVideoInfo.setDesignTeamLeaderId(user.getId());
+                                        etlKuaishouVideoInfo.setDesignTeamLeaderName(user.getRealname());
+                                    }else{
+                                        etlKuaishouVideoInfo.setDesignTeamLeaderId(user.getLeaderId());
+                                        etlKuaishouVideoInfo.setDesignTeamLeaderName(user.getLeaderName());
+                                    }
                                 }
                             }
                             String clipName = ascription.getString("clipName");

+ 8 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/mapper/ETLReportBytedanceVideoMapper.java

@@ -34,4 +34,12 @@ public interface ETLReportBytedanceVideoMapper extends BaseMapper<ETLReportByted
     List<ETLReportBytedanceVideo> getDuplicateVideoInfo(@Param("date")String date);
 
     void deleteDuplicateVideoInfo(@Param("video")ETLReportBytedanceVideo video);
+
+    List<ETLReportBytedanceVideo> getBytedanceVideoClipInfo();
+    List<ETLReportBytedanceVideo> getBytedanceVideoShotInfo();
+    List<ETLReportBytedanceVideo> getBytedanceVideoPlanInfo();
+
+    void updateClipInfo(@Param("clipId")String clipId, @Param("clipName")String clipName, @Param("md5")String md5);
+    void updateShotInfo(@Param("shotId")String shotId, @Param("shotName")String shotName, @Param("md5")String md5);
+    void updatePlanInfo(@Param("planId")String planId, @Param("planName")String planName, @Param("md5")String md5);
 }

+ 55 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/mapper/xml/ETLReportBytedanceVideoMapper.xml

@@ -201,6 +201,16 @@
             )
         </foreach>
     </insert>
+    <update id="updateClipInfo">
+        update etl_report_bytedance_video set clip_id=#{clipId},clip_name=#{clipName} where md5=#{md5}
+    </update>
+    <update id="updateShotInfo">
+        update etl_report_bytedance_video set shot_id=#{shotId},shot_name=#{shotName} where md5=#{md5}
+    </update>
+    <update id="updatePlanInfo">
+        update etl_report_bytedance_video set plan_id=#{planId},plan_name=#{planName} where md5=#{md5}
+    </update>
+
     <delete id="deleteDuplicateVideoInfo">
         delete from
                     etl_report_bytedance_video
@@ -319,4 +329,49 @@
         from etl_report_bytedance_video where stat_datetime =#{date} and cost>0 GROUP BY stat_datetime,material_id,account_id
         HAVING total >1 ORDER BY total DESC;
     </select>
+    <select id="getBytedanceVideoClipInfo"
+            resultType="cn.com.ctop.toutiao.modules.link.entity.ETLReportBytedanceVideo">
+        SELECT DISTINCT
+            video.md5 as 'md5',
+            cmi.clip_id as 'clip_id',
+            u.realname as 'clip_name'
+        FROM
+            etl_report_bytedance_video video
+                LEFT JOIN ctop_material_ascription cmi ON video.md5 = cmi.
+                CODE LEFT JOIN sys_user u ON u.id = cmi.clip_id
+        WHERE
+            cmi.clip_id IS NOT NULL
+          AND video.clip_id = ''
+          AND u.realname IS NOT NULL
+    </select>
+    <select id="getBytedanceVideoShotInfo"
+            resultType="cn.com.ctop.toutiao.modules.link.entity.ETLReportBytedanceVideo">
+        SELECT DISTINCT
+            video.md5 as 'md5',
+            cmi.shot_id as 'shot_id',
+            u.realname as 'shot_name'
+        FROM
+            etl_report_bytedance_video video
+                LEFT JOIN ctop_material_ascription cmi ON video.md5 = cmi.
+                CODE LEFT JOIN sys_user u ON u.id = cmi.shot_id
+        WHERE
+            cmi.shot_id IS NOT NULL
+          AND video.shot_id = ''
+          AND u.realname IS NOT NULL
+    </select>
+    <select id="getBytedanceVideoPlanInfo"
+            resultType="cn.com.ctop.toutiao.modules.link.entity.ETLReportBytedanceVideo">
+        SELECT DISTINCT
+            video.md5 as 'md5',
+            cmi.plan_id as 'plan_id',
+            u.realname as 'plan_name'
+        FROM
+            etl_report_bytedance_video video
+                LEFT JOIN ctop_material_ascription cmi ON video.md5 = cmi.
+                CODE LEFT JOIN sys_user u ON u.id = cmi.plan_id
+        WHERE
+            cmi.plan_id IS NOT NULL
+          AND video.plan_id = ''
+          AND u.realname IS NOT NULL
+    </select>
 </mapper>

+ 6 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/service/IETLReportBytedanceVideoService.java

@@ -16,4 +16,10 @@ public interface IETLReportBytedanceVideoService extends IService<ETLReportByted
     void cleanETL(String statDate);
 
     void cleanData(String data);
+
+    void etlBytedanceVideoClipIdJob();
+
+    void etlBytedanceVideoPlanInfoJob();
+
+    void etlBytedanceVideoShotInfoJob();
 }

+ 30 - 51
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/service/impl/ETLReportBytedanceVideoServiceImpl.java

@@ -35,57 +35,6 @@ public class ETLReportBytedanceVideoServiceImpl extends ServiceImpl<ETLReportByt
         return baseReportBytedanceVideoDailyService.list(queryWrapper);
     }
 
-/*    @Override
-    public void cleanETL(String statDate) {
-        List<BaseReportBytedanceVideoDaily> source = listByStatDate(statDate);
-        if(source.isEmpty()){
-            XxlJobHelper.log("头条数据报表清洗ETL异常,请检查数据源");
-        }
-        List<ETLReportBytedanceVideo> batch = new ArrayList<>();
-        source.forEach(it->{
-            JSONObject videoInfo= etlReportBytedanceVideoMapper.queryVideoInfoByMaterialId(it.getMaterialId());
-            if(null==videoInfo||videoInfo.isEmpty()){
-                XxlJobHelper.log("头条数据报表清洗ETL异常,未关联到video_info,materialId:{}",it.getMaterialId());
-            }else {
-                //TODO 暂时从老数据库查询,新素材库怕同步数据有偏差
-                //JSONObject materialVideo= etlReportBytedanceVideoMapper.queryMaterialVideoByMd5(videoInfo.getString("md5"));
-                JSONObject materialVideo= etlReportBytedanceVideoMapper.queryMaterialVideoByMd5Old(videoInfo.getString("md5"));
-                //JSONObject relatePerson=etlReportBytedanceVideoMapper.queryRelatePersonByMd5(videoInfo.getString("md5"));
-                JSONObject relatePerson=etlReportBytedanceVideoMapper.queryRelatePersonByMd5Old(videoInfo.getString("md5"));
-
-                JSONObject accountDetail=etlReportBytedanceVideoMapper.queryAccountDetailByAccountId(it.getAccountId());
-                ETLReportBytedanceVideo temp = JSONObject.toJavaObject(JSONObject.parseObject(JSON.toJSON(it).toString()),ETLReportBytedanceVideo.class);
-                temp.setMd5(videoInfo.getString("md5"));
-                temp.setToutiaoUrl(videoInfo.getString("url"));
-                if(null!=materialVideo&&!materialVideo.isEmpty()) {
-                    temp.setUrl(materialVideo.getString("url"));
-                    temp.setCoverUrl(materialVideo.getString("coverUrl"));
-                    temp.setMaterialName(materialVideo.getString("materialName"));
-                    temp.setMaterialCreateTime(materialVideo.getDate("createTime").toString());
-                }
-                if(null!=accountDetail&&!accountDetail.isEmpty()){
-                    temp.setAccountName(accountDetail.getString("accountName"));
-                    temp.setProjectId(accountDetail.getLong("projectId"));
-                    temp.setProjectName(accountDetail.getString("projectName"));
-                    temp.setProductId(accountDetail.getLong("productId"));
-                    temp.setProductName(accountDetail.getString("productName"));
-                    temp.setAdvertiserId(accountDetail.getString("advertiserId"));
-                    temp.setAdvertiserName(accountDetail.getString("advertiserName"));
-                }
-                if(null!=relatePerson&&!relatePerson.isEmpty()){
-                    temp.setClipId(relatePerson.getString("clipId"));
-                    temp.setClipName(relatePerson.getString("clipName"));
-                    temp.setShotId(relatePerson.getString("shotId"));
-                    temp.setShotName(relatePerson.getString("shotName"));
-                    temp.setPlanId(relatePerson.getString("planId"));
-                    temp.setPlanName(relatePerson.getString("planName"));
-                }
-                batch.add(temp);
-            }
-            this.saveBatch(batch);
-        });
-    }*/
-
     @Override
     public void cleanETL(String statDate) {
         List<ETLReportBytedanceVideo> etlReportBytedanceVideos = etlReportBytedanceVideoMapper.queryETLData(statDate);
@@ -112,4 +61,34 @@ public class ETLReportBytedanceVideoServiceImpl extends ServiceImpl<ETLReportByt
         }
     }
 
+    @Override
+    public void etlBytedanceVideoClipIdJob() {
+        List <ETLReportBytedanceVideo> clipInfos = etlReportBytedanceVideoMapper.getBytedanceVideoClipInfo();
+        if(null!=clipInfos&&!clipInfos.isEmpty()){
+            clipInfos.forEach(clipInfo->{
+                etlReportBytedanceVideoMapper.updateClipInfo(clipInfo.getClipId(),clipInfo.getClipName(),clipInfo.getMd5());
+            });
+        }
+    }
+
+    @Override
+    public void etlBytedanceVideoPlanInfoJob() {
+        List <ETLReportBytedanceVideo> planInfos = etlReportBytedanceVideoMapper.getBytedanceVideoPlanInfo();
+        if(null!=planInfos&&!planInfos.isEmpty()){
+            planInfos.forEach(planInfo->{
+                etlReportBytedanceVideoMapper.updatePlanInfo(planInfo.getPlanId(),planInfo.getPlanName(),planInfo.getMd5());
+            });
+        }
+    }
+
+    @Override
+    public void etlBytedanceVideoShotInfoJob() {
+        List <ETLReportBytedanceVideo> shotInfos = etlReportBytedanceVideoMapper.getBytedanceVideoShotInfo();
+        if(null!=shotInfos&&!shotInfos.isEmpty()){
+            shotInfos.forEach(shotInfo->{
+                etlReportBytedanceVideoMapper.updateShotInfo(shotInfo.getShotId(),shotInfo.getShotName(),shotInfo.getMd5());
+            });
+        }
+    }
+
 }

+ 3 - 1
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/entity/BytedanceCheckReportTaskInfo.java

@@ -47,6 +47,7 @@ public class BytedanceCheckReportTaskInfo {
 	private Date createTime;
 	/**updateTime*/
 	private Date updateTime;
+	private Integer costStatus;
 
 	public BytedanceCheckReportTaskInfo(){
 
@@ -57,7 +58,8 @@ public class BytedanceCheckReportTaskInfo {
 		this.campaignCost = BigDecimal.ZERO;
 		this.matCost = BigDecimal.ZERO;
 		this.imageCost = BigDecimal.ZERO;
-		this.status = 1;
+		this.status = 0;
+		this.costStatus = 0;
 		this.stateDate = date;
 		this.createTime = new Date();
 		this.updateTime = new Date();

+ 1 - 1
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/IBytedanceCheckReportTaskInfoService.java

@@ -16,7 +16,7 @@ public interface IBytedanceCheckReportTaskInfoService extends IService<Bytedance
 
     BytedanceCheckReportTaskInfo getInfoByParams(Long accountId, String date);
 
-    List<BytedanceCheckReportTaskInfo> getListByParams(Long accountId, String date);
+    List<BytedanceCheckReportTaskInfo> getListByParams(Long accountId, String date,Integer status);
 
     void checkCost(BytedanceCheckReportTaskInfo taskInfo);
 

+ 15 - 7
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/BytedanceCheckReportTaskInfoServiceImpl.java

@@ -70,7 +70,7 @@ public class BytedanceCheckReportTaskInfoServiceImpl extends ServiceImpl<Bytedan
     }
 
     @Override
-    public List<BytedanceCheckReportTaskInfo> getListByParams(Long accountId, String date) {
+    public List<BytedanceCheckReportTaskInfo> getListByParams(Long accountId, String date,Integer status) {
         QueryWrapper<BytedanceCheckReportTaskInfo> queryWrapper = new QueryWrapper<>();
         if(null!=accountId&&accountId!=0){
             queryWrapper.eq("account_id",accountId);
@@ -78,17 +78,20 @@ public class BytedanceCheckReportTaskInfoServiceImpl extends ServiceImpl<Bytedan
         if(null!=date&&!"".equals(date.trim())){
             queryWrapper.eq("state_date",date);
         }
+        if(null!=status){
+            queryWrapper.eq("status",status);
+        }
         queryWrapper.orderByDesc("id");
         return this.list(queryWrapper);
     }
 
     @Override
     public void checkCost(BytedanceCheckReportTaskInfo taskInfo){
-        System.out.println(taskInfo.getAccountId()+"数据获取开始");
         CtopOauthToken token = tokenService.getTokenByAccountId(taskInfo.getAccountId());
         if(null == token){
             taskInfo.setStatus(-1);
             taskInfo.setDescription("token数据不存在!");
+            taskInfo.setCostStatus(-1);
             taskInfo.setUpdateTime(new Date());
             this.saveOrUpdate(taskInfo);
             return;
@@ -115,17 +118,22 @@ public class BytedanceCheckReportTaskInfoServiceImpl extends ServiceImpl<Bytedan
         }
         if(taskInfo.getAccountCost().compareTo(taskInfo.getCampaignCost())!=0){
             taskInfo.setDescription("账户消耗与组消耗不匹配");
-            taskInfo.setStatus(-1);
+            taskInfo.setStatus(50001);
+            taskInfo.setCostStatus(-1);
             this.saveOrUpdate(taskInfo);
+            return;
         }
-        if(taskInfo.getImageCost().add(taskInfo.getMatCost()).subtract(taskInfo.getAccountCost()).compareTo(new BigDecimal("10"))>0){
+        if(taskInfo.getImageCost().add(taskInfo.getMatCost()).subtract(taskInfo.getAccountCost()).abs().compareTo(new BigDecimal("10"))>0){
             taskInfo.setDescription("账户消耗与素材消耗不匹配");
-            taskInfo.setStatus(-2);
+            taskInfo.setStatus(50001);
+            taskInfo.setCostStatus(-2);
             this.saveOrUpdate(taskInfo);
+            return;
         }
 
         taskInfo.setDescription("消耗数据正常");
-        taskInfo.setStatus(0);
+        taskInfo.setStatus(1);
+        taskInfo.setCostStatus(1);
         this.saveOrUpdate(taskInfo);
     }
 
@@ -141,7 +149,7 @@ public class BytedanceCheckReportTaskInfoServiceImpl extends ServiceImpl<Bytedan
             reportService.getAdvertiserReport(token, DateUtils.parseDate(taskInfo.getStateDate(),"yyyy-MM-dd"),DateUtils.parseDate(taskInfo.getStateDate(),"yyyy-MM-dd"), CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
             reportService.getAdvertiserCampaignReport(token, DateUtils.parseDate(taskInfo.getStateDate(),"yyyy-MM-dd"),DateUtils.parseDate(taskInfo.getStateDate(),"yyyy-MM-dd"), CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
         }
-        if(imageCost.add(matCost).subtract(campaignCost).compareTo(new BigDecimal("10"))>0){
+        if(imageCost.add(matCost).subtract(campaignCost).abs().compareTo(BigDecimal.ZERO)>0){
             reportService.getAdvertiserReport(token, DateUtils.parseDate(taskInfo.getStateDate(),"yyyy-MM-dd"),DateUtils.parseDate(taskInfo.getStateDate(),"yyyy-MM-dd"), CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
             reportService.getAdvertiserCampaignReport(token, DateUtils.parseDate(taskInfo.getStateDate(),"yyyy-MM-dd"),DateUtils.parseDate(taskInfo.getStateDate(),"yyyy-MM-dd"), CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
             bytedanceReportService.bytedanceMaterialReport(token, taskInfo.getStateDate(), taskInfo.getStateDate());

+ 4 - 2
pom.xml

@@ -66,11 +66,13 @@
 
         <repository>
             <id>maven-3rdparty</id>
+            <name>maven-3rdparty</name>
             <url>https://packages.atlassian.com/maven-3rdparty/</url>
         </repository>
 
         <repository>
             <id>ctop</id>
+            <name>ctop</name>
             <url>http://118.24.244.213:8081/repository/3rd_part</url>
         </repository>
     </repositories>
@@ -416,7 +418,7 @@
             <artifactId>aliyun-java-sdk-mts</artifactId>
             <version>2.5.2</version>
         </dependency>
-        <!--<dependency>
+        <dependency>
             <groupId>org.bytedeco</groupId>
             <artifactId>javacv</artifactId>
             <version>1.4.2</version>
@@ -425,7 +427,7 @@
             <groupId>org.bytedeco</groupId>
             <artifactId>javacv-platform</artifactId>
             <version>1.4.2</version>
-        </dependency>-->
+        </dependency>
         <!--<dependency>
             <groupId>com.qcloud</groupId>
             <artifactId>cos_api</artifactId>