Browse Source

Merge branch 'video-dev' into test

hcst_sunzhen 5 năm trước cách đây
mục cha
commit
839aa0cf18
37 tập tin đã thay đổi với 388 bổ sung59 xóa
  1. 111 0
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/TestController.java
  2. 3 3
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/UserAllocationController.java
  3. 6 1
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/BytedanceDailyMaterialReportJob.java
  4. 22 2
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/BytedanceDailyMaterialReportRetryJob.java
  5. 1 0
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/CutFrameJob.java
  6. 3 0
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/KuaishouDailyAccountReportLoadJob.java
  7. 1 1
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/KuaishouDailyAccountReportNextDayStayJob.java
  8. 6 2
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/system/controller/SysUserController.java
  9. 1 1
      jeecg-boot-module-system/src/main/resources/application-dev.yml
  10. 0 1
      jeecg-boot-module-system/src/main/resources/application-prod.yml
  11. 20 6
      jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java
  12. 2 0
      module-common/src/main/java/cn/com/ctop/common/module/mapper/MaterialInfoMapper.java
  13. 8 0
      module-common/src/main/java/cn/com/ctop/common/module/mapper/xml/MaterialInfoMapper.xml
  14. 1 0
      module-common/src/main/java/cn/com/ctop/common/module/service/IMaterialInfoService.java
  15. 1 1
      module-common/src/main/java/cn/com/ctop/common/module/service/impl/MaterialCutFrameServiceImpl.java
  16. 8 2
      module-common/src/main/java/cn/com/ctop/common/module/service/impl/MaterialInfoServiceImpl.java
  17. 0 1
      module-common/src/main/java/cn/com/ctop/common/module/utils/LoadFileUtil.java
  18. 0 1
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinAppServiceImpl.java
  19. 0 1
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinReportAppHourServiceImpl.java
  20. 2 2
      module-ctop/src/main/java/cn/com/ctop/manage/modules/material/service/impl/MaterialUploadServiceImpl.java
  21. 4 0
      module-job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/ByteDanceVideoReportJob.java
  22. 8 1
      module-job-kuaishou/src/main/java/cn/com/ctop/job/kuaishou/handler/KuaishouDailyAccountReportLoadJob.java
  23. 24 2
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/controller/BatchController.java
  24. 2 0
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/IKuaiShouVideoGetService.java
  25. 98 0
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/impl/KuaiShouVideoGetServiceImpl.java
  26. 6 0
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/impl/KuaishouInterfaceServiceImpl.java
  27. 3 1
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/xml/KuaishouReportDailyCampaignMapper.xml
  28. 3 1
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/xml/KuaishouReportDailyCreativeMapper.xml
  29. 3 1
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/xml/KuaishouReportDailyGroupMapper.xml
  30. 2 2
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/controller/ByteDanceVideoReportDailyController.java
  31. 3 3
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/controller/BytedanceHomepageReportCtrl.java
  32. 1 2
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/controller/BytedanceReportController.java
  33. 6 6
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/xml/BytedanceHomepageMapper.xml
  34. 1 1
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/IBytedanceHomepageReportService.java
  35. 1 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/ByteDanceVideoReportDailyServiceImpl.java
  36. 17 8
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/BytedanceHomepageReportServiceImpl.java
  37. 10 6
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/ReportServiceImpl.java

+ 111 - 0
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/TestController.java

@@ -20,11 +20,13 @@ import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import lombok.extern.slf4j.Slf4j;
 import lombok.extern.slf4j.Slf4j;
 import org.jeecg.common.api.vo.Result;
 import org.jeecg.common.api.vo.Result;
 import org.jeecg.common.util.DateUtils;
 import org.jeecg.common.util.DateUtils;
+import org.quartz.JobExecutionException;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
 import org.springframework.web.bind.annotation.RestController;
 
 
+import java.io.IOException;
 import java.text.ParseException;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.text.SimpleDateFormat;
 import java.util.*;
 import java.util.*;
@@ -87,6 +89,115 @@ public class TestController {
     @Autowired
     @Autowired
     private ICtopOauthTokenService oauthTokenService;
     private ICtopOauthTokenService oauthTokenService;
 
 
+    @Autowired
+    private IMaterialCutFrameService cutFrameService;
+
+
+    @GetMapping(value = "/getAccountList")
+    public void getAccountList(String startDate, String endDate) throws JobExecutionException {
+        Thread thread = new Thread() {
+            @Override
+            public void run() {
+                SimpleDateFormat sim = new SimpleDateFormat("yyyy-MM-dd");
+                try {
+                    Date start = sim.parse(startDate);
+                    Date end = sim.parse(endDate);
+                    //1:查询当日数据
+                    List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
+                    if (null == tokens || tokens.size() <= 0) {
+                        log.info("定时获取快手数据异常:未获取到可用的token");
+                        return;
+                    }
+
+                    executorService = Executors.newFixedThreadPool(3);
+                    tokens.forEach(token -> {
+                        executorService.submit(new Runnable() {
+                            @Override
+                            public void run() {
+                                //1: 获取广告主信息数据
+                                kuaishouInterfaceService.getAdvertiserReportDaily(token, start, end);
+                            }
+                        });
+                    });
+                } catch (ParseException e) {
+                    e.printStackTrace();
+                }
+
+
+            }
+        };
+        thread.start();
+
+    }
+
+
+    @GetMapping(value = "/getVideo")
+    public String getVideo(String startDate, String endDate) throws IOException {
+
+        try {
+
+
+            Thread thread = new Thread() {
+                @Override
+                public void run() {
+
+
+                    QueryWrapper<CtopOauthToken> tokenQueryWrapper = new QueryWrapper<>();
+                    tokenQueryWrapper.eq("media_id", 2);
+                    List<CtopOauthToken> list = tokenService.list(tokenQueryWrapper);
+
+                    executorService = Executors.newFixedThreadPool(8);
+                    list.forEach(token -> {
+                        executorService.submit(new Runnable() {
+                            @Override
+                            public void run() {
+                                try {
+                                    //获取全量视频素材数据
+                                    kuaishouInterfaceService.getVideoList(token, null, null);
+                                    // 获取图片信息数据
+                                    //   kuaishouInterfaceService.getImageList(token, startDate, endDate);
+                                } catch (Exception e) {
+                                    e.printStackTrace();
+                                } finally {
+                                }
+                            }
+                        });
+                    });
+                    log.info("快手物料数据同步完成");
+                }
+            };
+            thread.start();
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+        return "Success";
+    }
+
+    @GetMapping(value = "/createCutTask")
+    public String createCutTask(String startDate, String endDate) throws IOException {
+
+        List<MaterialInfo> materialInfos = materialInfoService.getListByDate(startDate, endDate);
+        if (!Check.isNull(materialInfos)) {
+            for (MaterialInfo materialInfo : materialInfos) {
+                if (!Check.isNull(materialInfo)) {
+                    String code = materialInfo.getCode();
+                    QueryWrapper<MaterialParameter> parameterQueryWrapper = new QueryWrapper<>();
+                    parameterQueryWrapper.eq("material_id", code);
+                    parameterQueryWrapper.last("limit 1");
+                    MaterialParameter materialParameter = parameterService.getOne(parameterQueryWrapper);
+                    if (!Check.isNull(materialParameter)) {
+                        cutFrameService.getCutFrame(materialInfo.getUrl(), code, materialParameter.getSecond(), materialParameter.getHeight(), materialParameter.getWidth());
+
+                    }
+
+                }
+
+            }
+
+        }
+        return "Success";
+    }
+
 
 
     @GetMapping(value = "/getToken")
     @GetMapping(value = "/getToken")
     public String getToken(Long accountId, String refreshToken, Integer agentType) {
     public String getToken(Long accountId, String refreshToken, Integer agentType) {

+ 3 - 3
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/UserAllocationController.java

@@ -124,10 +124,10 @@ public class UserAllocationController {
             QueryWrapper<UserAllocation> queryWrapper = new QueryWrapper<>();
             QueryWrapper<UserAllocation> queryWrapper = new QueryWrapper<>();
             queryWrapper.eq("user_id", userId);
             queryWrapper.eq("user_id", userId);
             queryWrapper.eq("project_id", projectId);
             queryWrapper.eq("project_id", projectId);
-            Project project = projectService.getById(projectId);
-            if (!Check.isNull(project)) {
+           /*  Project project = projectService.getById(projectId);
+                if (!Check.isNull(project)) {
                 queryWrapper.eq("media_id", project.getMediaId());
                 queryWrapper.eq("media_id", project.getMediaId());
-            }
+            }*/
 
 
             List<UserAllocation> userAllocations = userAllocationMapper.selectList(queryWrapper);
             List<UserAllocation> userAllocations = userAllocationMapper.selectList(queryWrapper);
             result.setSuccess(true);
             result.setSuccess(true);

+ 6 - 1
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/BytedanceDailyMaterialReportJob.java

@@ -29,9 +29,12 @@ public class BytedanceDailyMaterialReportJob implements Job {
 
 
     @Override
     @Override
     public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
     public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
+        Date getDate2 = DateUtils.addDay(new Date(), -2);
+        String date2 = DateUtils.formatDate(getDate2);
+
         Date getDate = DateUtils.addDay(new Date(), -1);
         Date getDate = DateUtils.addDay(new Date(), -1);
         String date = DateUtils.formatDate(getDate);
         String date = DateUtils.formatDate(getDate);
-        log.info("头条获取素材报表数据任务开始,任务时间:" + date);
+        log.info("头条获取素材报表数据任务开始,任务时间:" + date2 + "~" + date);
 
 
         List<CtopOauthToken> tokens = tokenService.selectToutiaoToken();
         List<CtopOauthToken> tokens = tokenService.selectToutiaoToken();
         if (null == tokens || tokens.size() <= 0) {
         if (null == tokens || tokens.size() <= 0) {
@@ -45,6 +48,8 @@ public class BytedanceDailyMaterialReportJob implements Job {
                 @Override
                 @Override
                 public void run() {
                 public void run() {
                     try {
                     try {
+                        //获取头条素材报表两天前的数据
+                        bytedanceReportService.bytedanceMaterialReport(token, date2, date2);
                         //获取头条素材报表数据
                         //获取头条素材报表数据
                         bytedanceReportService.bytedanceMaterialReport(token, date, date);
                         bytedanceReportService.bytedanceMaterialReport(token, date, date);
                     } catch (Exception e) {
                     } catch (Exception e) {

+ 22 - 2
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/BytedanceDailyMaterialReportRetryJob.java

@@ -3,8 +3,10 @@ package org.jeecg.modules.ctop.job;
 import cn.com.ctop.common.module.entity.CtopOauthToken;
 import cn.com.ctop.common.module.entity.CtopOauthToken;
 import cn.com.ctop.common.module.service.ICtopOauthTokenService;
 import cn.com.ctop.common.module.service.ICtopOauthTokenService;
 import cn.com.ctop.toutiao.modules.report.entity.BytedanceReportMaterialRetry;
 import cn.com.ctop.toutiao.modules.report.entity.BytedanceReportMaterialRetry;
+import cn.com.ctop.toutiao.modules.report.service.IByteDanceVideoReportDailyService;
 import cn.com.ctop.toutiao.modules.report.service.IBytedanceReportService;
 import cn.com.ctop.toutiao.modules.report.service.IBytedanceReportService;
 import lombok.extern.slf4j.Slf4j;
 import lombok.extern.slf4j.Slf4j;
+import org.jeecg.common.util.DateUtils;
 import org.quartz.Job;
 import org.quartz.Job;
 import org.quartz.JobExecutionContext;
 import org.quartz.JobExecutionContext;
 import org.quartz.JobExecutionException;
 import org.quartz.JobExecutionException;
@@ -25,6 +27,8 @@ public class BytedanceDailyMaterialReportRetryJob implements Job {
     private ICtopOauthTokenService tokenService;
     private ICtopOauthTokenService tokenService;
     @Autowired
     @Autowired
     private IBytedanceReportService bytedanceReportService;
     private IBytedanceReportService bytedanceReportService;
+    @Autowired
+    private IByteDanceVideoReportDailyService byteDanceVideoReportDailyService;
 
 
     @Override
     @Override
     public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
     public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
@@ -38,9 +42,25 @@ public class BytedanceDailyMaterialReportRetryJob implements Job {
                 @Override
                 @Override
                 public void run() {
                 public void run() {
                     try {
                     try {
-                        CtopOauthToken token = tokenService.getOauthTokenByAccountId(String.valueOf(retry.getAccountId()));
+                        String startDate = retry.getStartDate();
+                        String endDate = retry.getEndDate();
+                        Long accountId = retry.getAccountId();
+                        CtopOauthToken token = tokenService.getOauthTokenByAccountId(String.valueOf(accountId));
                         log.info("头条素材报表重试定时任务,当前accountId为:" + token.getAccountId());
                         log.info("头条素材报表重试定时任务,当前accountId为:" + token.getAccountId());
-                        bytedanceReportService.bytedanceMaterialReportRetry(token, retry.getStartDate(), retry.getEndDate());
+
+                        Long days = DateUtils.getDiscrepantDays(startDate, endDate); //间隔天数
+                        String start = null;
+                        String end = null;
+                        for (int i = 0; i <= days; i++) {
+                            start = DateUtils.addDay(startDate, i);
+                            end = start;
+                            //获取头条素材报表数据
+                            int code = bytedanceReportService.bytedanceMaterialReportRetry(token, start, end);
+                            if(code ==200 || code == 1){
+                                //重试成功清洗数据
+                                byteDanceVideoReportDailyService.videoInfoListByAccountId(start,end,accountId);
+                            }
+                        }
                     } catch (Exception e) {
                     } catch (Exception e) {
                         e.printStackTrace();
                         e.printStackTrace();
                     } finally {
                     } finally {

+ 1 - 0
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/CutFrameJob.java

@@ -32,6 +32,7 @@ public class CutFrameJob implements Job {
             @Override
             @Override
             public void run() {
             public void run() {
 
 
+                log.info("开始抽帧定时任务");
                 QueryWrapper<MaterialCutFrameTask> taskQueryWrapper = new QueryWrapper<>();
                 QueryWrapper<MaterialCutFrameTask> taskQueryWrapper = new QueryWrapper<>();
                 taskQueryWrapper.eq("job_status", 0);
                 taskQueryWrapper.eq("job_status", 0);
                 List<MaterialCutFrameTask> list = taskService.list(taskQueryWrapper);
                 List<MaterialCutFrameTask> list = taskService.list(taskQueryWrapper);

+ 3 - 0
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/KuaishouDailyAccountReportLoadJob.java

@@ -33,6 +33,7 @@ public class KuaishouDailyAccountReportLoadJob implements Job {
         Thread thread = new Thread() {
         Thread thread = new Thread() {
             @Override
             @Override
             public void run() {
             public void run() {
+                Date getDate2 = DateUtils.addDay(new Date(), -2);
                 Date getDate = DateUtils.addDay(new Date(), -1);
                 Date getDate = DateUtils.addDay(new Date(), -1);
                 //1:查询当日数据
                 //1:查询当日数据
                 List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
                 List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
@@ -46,6 +47,8 @@ public class KuaishouDailyAccountReportLoadJob implements Job {
                     executorService.submit(new Runnable() {
                     executorService.submit(new Runnable() {
                         @Override
                         @Override
                         public void run() {
                         public void run() {
+                            //跑两天前的账户报表---次留统计用
+                            kuaishouInterfaceService.getAdvertiserReportDaily(token, getDate2, getDate2);
                             //1: 获取广告主信息数据
                             //1: 获取广告主信息数据
                             kuaishouInterfaceService.getAdvertiserReportDaily(token, getDate, getDate);
                             kuaishouInterfaceService.getAdvertiserReportDaily(token, getDate, getDate);
                         }
                         }

+ 1 - 1
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/KuaishouDailyAccountReportNextDayStayJob.java

@@ -34,7 +34,7 @@ public class KuaishouDailyAccountReportNextDayStayJob implements Job {
         Thread thread = new Thread() {
         Thread thread = new Thread() {
             @Override
             @Override
             public void run() {
             public void run() {
-                Date getDate = DateUtils.addDay(new Date(), -2);
+                Date getDate = new Date();
                 //1:查询当日数据
                 //1:查询当日数据
                 List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
                 List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
                 if (null == tokens || tokens.size() <= 0) {
                 if (null == tokens || tokens.size() <= 0) {

+ 6 - 2
jeecg-boot-module-system/src/main/java/org/jeecg/modules/system/controller/SysUserController.java

@@ -94,9 +94,13 @@ public class SysUserController {
     @RequestMapping(value = "/list", method = RequestMethod.GET)
     @RequestMapping(value = "/list", method = RequestMethod.GET)
     public Result<IPage<SysUser>> queryPageList(SysUser user, @RequestParam(name = "pageNo", defaultValue = "1") Integer pageNo,
     public Result<IPage<SysUser>> queryPageList(SysUser user, @RequestParam(name = "pageNo", defaultValue = "1") Integer pageNo,
                                                 @RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize, HttpServletRequest req) {
                                                 @RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize, HttpServletRequest req) {
-        Result<IPage<SysUser>> result = new Result<IPage<SysUser>>();
+        Result<IPage<SysUser>> result = new Result<>();
+        String realName = user.getRealname();
         QueryWrapper<SysUser> queryWrapper = QueryGenerator.initQueryWrapper(user, req.getParameterMap());
         QueryWrapper<SysUser> queryWrapper = QueryGenerator.initQueryWrapper(user, req.getParameterMap());
-        Page<SysUser> page = new Page<SysUser>(pageNo, pageSize);
+        if(null!=realName&&!"".equals(realName)){
+            queryWrapper.like("realname",realName);
+        }
+        Page<SysUser> page = new Page<>(pageNo, pageSize);
         IPage<SysUser> pageList = sysUserService.page(page, queryWrapper);
         IPage<SysUser> pageList = sysUserService.page(page, queryWrapper);
         result.setSuccess(true);
         result.setSuccess(true);
         result.setResult(pageList);
         result.setResult(pageList);

+ 1 - 1
jeecg-boot-module-system/src/main/resources/application-dev.yml

@@ -94,7 +94,7 @@ spring:
         connectionProperties: druid.stat.mergeSql\=true;druid.stat.slowSqlMillis\=5000
         connectionProperties: druid.stat.mergeSql\=true;druid.stat.slowSqlMillis\=5000
       datasource:
       datasource:
         master:
         master:
-          url:  url: jdbc:mysql://39.106.184.70:3306/jeecg-boot?characterEncoding=UTF-8&useUnicode=true&useSSL=false
+          url: jdbc:mysql://39.106.184.70:3306/jeecg-boot?characterEncoding=UTF-8&useUnicode=true&useSSL=false
           username: hcst
           username: hcst
           password: test@20190531
           password: test@20190531
           driver-class-name: com.mysql.jdbc.Driver
           driver-class-name: com.mysql.jdbc.Driver

+ 0 - 1
jeecg-boot-module-system/src/main/resources/application-prod.yml

@@ -145,7 +145,6 @@ jeecg:
     video-upload: /mnt/upload/video/
     video-upload: /mnt/upload/video/
     image-upload: /mnt/upload/image/
     image-upload: /mnt/upload/image/
     chrome-driver: /usr/bin/chromedriver
     chrome-driver: /usr/bin/chromedriver
-#    chrome-driver: D://chromedriver.exe
     csv-upload: /mnt/upload/csv/
     csv-upload: /mnt/upload/csv/
     bak-database-file: /mnt/data/bak/
     bak-database-file: /mnt/data/bak/
     report-history: /mnt/upload/report/history/
     report-history: /mnt/upload/report/history/

+ 20 - 6
jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java

@@ -1,10 +1,10 @@
 package org.jeecg;
 package org.jeecg;
 
 
 import cn.com.ctop.common.module.entity.BindAccountLogin;
 import cn.com.ctop.common.module.entity.BindAccountLogin;
-import cn.com.ctop.common.module.entity.CtopOauthToken;
 import cn.com.ctop.common.module.service.IBindAccountLoginService;
 import cn.com.ctop.common.module.service.IBindAccountLoginService;
 import cn.com.ctop.common.module.service.ICtopOauthTokenService;
 import cn.com.ctop.common.module.service.ICtopOauthTokenService;
 import cn.com.ctop.common.module.utils.CtopAdConstant;
 import cn.com.ctop.common.module.utils.CtopAdConstant;
+import cn.com.ctop.common.module.utils.HttpUtils;
 import cn.com.ctop.crawler.modules.pangolin.entity.PangolinApp;
 import cn.com.ctop.crawler.modules.pangolin.entity.PangolinApp;
 import cn.com.ctop.crawler.modules.pangolin.service.PangolinAppService;
 import cn.com.ctop.crawler.modules.pangolin.service.PangolinAppService;
 import cn.com.ctop.crawler.modules.pangolin.service.PangolinCrawlerService;
 import cn.com.ctop.crawler.modules.pangolin.service.PangolinCrawlerService;
@@ -13,6 +13,8 @@ import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouWebInterfaceService
 import cn.com.ctop.toutiao.modules.report.service.IByteDanceVideoReportDailyService;
 import cn.com.ctop.toutiao.modules.report.service.IByteDanceVideoReportDailyService;
 import cn.com.ctop.toutiao.modules.report.service.IBytedanceReportService;
 import cn.com.ctop.toutiao.modules.report.service.IBytedanceReportService;
 import cn.com.ctop.toutiao.modules.report.service.IReportService;
 import cn.com.ctop.toutiao.modules.report.service.IReportService;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import lombok.extern.slf4j.Slf4j;
 import lombok.extern.slf4j.Slf4j;
 import org.jeecg.common.util.DateUtils;
 import org.jeecg.common.util.DateUtils;
@@ -22,10 +24,9 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.test.context.junit4.SpringRunner;
 import org.springframework.test.context.junit4.SpringRunner;
 
 
-import java.text.SimpleDateFormat;
-import java.util.Date;
 import java.util.HashMap;
 import java.util.HashMap;
 import java.util.List;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Executors;
@@ -46,14 +47,27 @@ public class SampleTest {
     @Test
     @Test
     public void testOceanEngineJob() {
     public void testOceanEngineJob() {
         try {
         try {
-            CtopOauthToken byId = oauthTokenService.getById(1654059015242756L);
+            String url = "https://ad.e.kuaishou.com/rest/openapi/v1/tool/key_frame";
+            JSONArray ptotoArr = new JSONArray();
+            ptotoArr.add("5188991225263567823");
+
+            Map<String, String> headers = new HashMap<>();
+            headers.put("Access-Token", "e251db4aa139eb36623818581003bcec");
+            headers.put("Content-Type", "application/json");
+            JSONObject requestJson = new JSONObject();
+            requestJson.put("advertiser_id", 161468);
+            requestJson.put("photo_ids", ptotoArr);
+            //  requestJson.put("type", type);
+            String result = HttpUtils.kuaiShouhttpPostRequest(url, requestJson.toJSONString(), headers);
+            System.err.println(result);
+            //     CtopOauthToken byId = oauthTokenService.getById(1654059015242756L);
 
 
           /*  SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
           /*  SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
             Date start = simpleDateFormat.parse("2020-01-01");
             Date start = simpleDateFormat.parse("2020-01-01");
             Date end = simpleDateFormat.parse("2020-06-03");*/
             Date end = simpleDateFormat.parse("2020-06-03");*/
 
 
 
 
-            String nowDate = "2020-06-03";
+            /*String nowDate = "2020-06-03";
             String endDate = "2020-01-01";
             String endDate = "2020-01-01";
             SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
             SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
             Date start = simpleDateFormat.parse(nowDate);
             Date start = simpleDateFormat.parse(nowDate);
@@ -63,7 +77,7 @@ public class SampleTest {
                 String formatDate = simpleDateFormat.format(dates.get(i));
                 String formatDate = simpleDateFormat.format(dates.get(i));
                 Date parse = simpleDateFormat.parse(formatDate);
                 Date parse = simpleDateFormat.parse(formatDate);
                 reportService.getAdvertiserReport(byId, parse, parse, CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
                 reportService.getAdvertiserReport(byId, parse, parse, CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
-            }
+            }*/
         } catch (Exception e) {
         } catch (Exception e) {
             e.printStackTrace();
             e.printStackTrace();
         }
         }

+ 2 - 0
module-common/src/main/java/cn/com/ctop/common/module/mapper/MaterialInfoMapper.java

@@ -34,4 +34,6 @@ public interface MaterialInfoMapper extends BaseMapper<MaterialInfo> {
     MaterialInfo getMaterialInfoByCode(@Param("code") String code);
     MaterialInfo getMaterialInfoByCode(@Param("code") String code);
 
 
     Long getProjectIdByAccountId(@Param("accountId") Long accountId);
     Long getProjectIdByAccountId(@Param("accountId") Long accountId);
+
+    List<MaterialInfo> getListByDate(@Param("startDate") String startDate,@Param("endDate") String endDate);
 }
 }

+ 8 - 0
module-common/src/main/java/cn/com/ctop/common/module/mapper/xml/MaterialInfoMapper.xml

@@ -130,5 +130,13 @@
         limit 1
         limit 1
     </select>
     </select>
 
 
+    <select id="getListByDate" resultType="cn.com.ctop.common.module.entity.MaterialInfo">
+        select  *  from
+        ctop_material_info
+        where  1 = 1
+        and DATE_FORMAT(create_time,'%Y-%m-%d') &gt;= #{startDate}
+        and DATE_FORMAT(create_time,'%Y-%m-%d') &lt;= #{endDate}
+    </select>
+
 
 
 </mapper>
 </mapper>

+ 1 - 0
module-common/src/main/java/cn/com/ctop/common/module/service/IMaterialInfoService.java

@@ -64,4 +64,5 @@ public interface IMaterialInfoService extends IService<MaterialInfo> {
 
 
     List<String> getEffiCode();
     List<String> getEffiCode();
 
 
+    List<MaterialInfo> getListByDate(String startDate, String endDate);
 }
 }

+ 1 - 1
module-common/src/main/java/cn/com/ctop/common/module/service/impl/MaterialCutFrameServiceImpl.java

@@ -97,7 +97,7 @@ public class MaterialCutFrameServiceImpl extends ServiceImpl<MaterialCutFrameMap
             JSONObject input = new JSONObject();
             JSONObject input = new JSONObject();
             input.put("Location", ossLocation);
             input.put("Location", ossLocation);
             input.put("Bucket", ossBucket);
             input.put("Bucket", ossBucket);
-
+         //   https://ctop-media.oss-cn-beijing.aliyuncs.com/video/2020-06-03/bp%E8%95%89%E4%B8%8B%E7%9B%B4%E6%92%AD2-1591188835505.mp4
             String replaceUrl = url.replace("https://ctop-media.oss-cn-beijing.aliyuncs.com/", "");
             String replaceUrl = url.replace("https://ctop-media.oss-cn-beijing.aliyuncs.com/", "");
             input.put("Object", replaceUrl);
             input.put("Object", replaceUrl);
 
 

+ 8 - 2
module-common/src/main/java/cn/com/ctop/common/module/service/impl/MaterialInfoServiceImpl.java

@@ -336,7 +336,8 @@ public class MaterialInfoServiceImpl extends ServiceImpl<MaterialInfoMapper, Mat
                 log.info("获取素材基本信息,code:{}", materialInfo.getCode());
                 log.info("获取素材基本信息,code:{}", materialInfo.getCode());
                 if (!Check.isNull(materialInfo)) {
                 if (!Check.isNull(materialInfo)) {
                     long l = System.currentTimeMillis();
                     long l = System.currentTimeMillis();
-                    String url = materialInfo.getUrl().replace(replaceOldValue, replaceValue);
+                    String videoUrl = materialInfo.getUrl();
+                    String url = videoUrl.replace(replaceOldValue, replaceValue);
                     log.info("replaceUrl:{}", url);
                     log.info("replaceUrl:{}", url);
                     String localUrl = null;
                     String localUrl = null;
                     it.sauronsoftware.jave.MultimediaInfo m = null;
                     it.sauronsoftware.jave.MultimediaInfo m = null;
@@ -383,7 +384,7 @@ public class MaterialInfoServiceImpl extends ServiceImpl<MaterialInfoMapper, Mat
                             public void run() {
                             public void run() {
                                 try {
                                 try {
                                     log.info("开始抽帧");
                                     log.info("开始抽帧");
-                                    materialCutFrameService.getCutFrame(url, materialInfo.getCode(), secondDuration, height, width);
+                                    materialCutFrameService.getCutFrame(videoUrl, materialInfo.getCode(), secondDuration, height, width);
                                 } catch (Exception e) {
                                 } catch (Exception e) {
                                     e.printStackTrace();
                                     e.printStackTrace();
                                 }
                                 }
@@ -603,6 +604,11 @@ public class MaterialInfoServiceImpl extends ServiceImpl<MaterialInfoMapper, Mat
 
 
         return materialInfoList;
         return materialInfoList;
     }
     }
+
+    @Override
+    public List<MaterialInfo> getListByDate(String startDate, String endDate) {
+        return materialInfoMapper.getListByDate(startDate, endDate);
+    }
 }
 }
 
 
 
 

+ 0 - 1
module-common/src/main/java/cn/com/ctop/common/module/utils/LoadFileUtil.java

@@ -37,7 +37,6 @@ public class LoadFileUtil {
             //文件保存位置
             //文件保存位置
             File saveDir = new File(savePath);
             File saveDir = new File(savePath);
             if (!saveDir.exists()) {
             if (!saveDir.exists()) {
-//            saveDir.mkdir();
                 saveDir.mkdirs();
                 saveDir.mkdirs();
             }
             }
             String localPath = saveDir + File.separator + fileName;
             String localPath = saveDir + File.separator + fileName;

+ 0 - 1
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinAppServiceImpl.java

@@ -28,7 +28,6 @@ public class PangolinAppServiceImpl extends ServiceImpl<PangolinAppMapper, Pango
     @Override
     @Override
     public Map<String, Object> getListByUserId(String userId, JSONObject data) {
     public Map<String, Object> getListByUserId(String userId, JSONObject data) {
         Map<String,Object>result = new HashMap<>();
         Map<String,Object>result = new HashMap<>();
-        System.out.println(data.toJSONString());
         List<Long>projectIds = null;
         List<Long>projectIds = null;
         List<String>accountNames = null;
         List<String>accountNames = null;
         JSONArray accountNameArray = data.getJSONArray("accountName");
         JSONArray accountNameArray = data.getJSONArray("accountName");

+ 0 - 1
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinReportAppHourServiceImpl.java

@@ -64,7 +64,6 @@ public class PangolinReportAppHourServiceImpl extends ServiceImpl<PangolinReport
 
 
     private JSONObject getComparedDate(Integer appId,Integer hour,String userId,String type,String today, String yesterday, String sevenDaysAgo, String compareTime1, String compareTime2,List<Long>projectIds,List<String>  accountNames) {
     private JSONObject getComparedDate(Integer appId,Integer hour,String userId,String type,String today, String yesterday, String sevenDaysAgo, String compareTime1, String compareTime2,List<Long>projectIds,List<String>  accountNames) {
         JSONObject data = pangolinReportAppHourMapper.getComparedDate(appId,hour,userId,type,today, yesterday,sevenDaysAgo,compareTime1,compareTime2,projectIds,accountNames);
         JSONObject data = pangolinReportAppHourMapper.getComparedDate(appId,hour,userId,type,today, yesterday,sevenDaysAgo,compareTime1,compareTime2,projectIds,accountNames);
-        System.out.println(data.toJSONString());
         if(null==data||data.toJSONString().equals("null")){
         if(null==data||data.toJSONString().equals("null")){
             data = new JSONObject();
             data = new JSONObject();
             data.put("today",0);
             data.put("today",0);

+ 2 - 2
module-ctop/src/main/java/cn/com/ctop/manage/modules/material/service/impl/MaterialUploadServiceImpl.java

@@ -110,8 +110,8 @@ public class MaterialUploadServiceImpl implements IMaterialUploadService {
                         continue;
                         continue;
                     }
                     }
                     touTiaoExecutorService.submit(new Runnable() {
                     touTiaoExecutorService.submit(new Runnable() {
-                        @Override
-                        public void run() {
+                                @Override
+                                public void run() {
                             try {
                             try {
                                 log.info("头条文件多线程上传:{},accountId:{}", Thread.currentThread().getName(), accountId);
                                 log.info("头条文件多线程上传:{},accountId:{}", Thread.currentThread().getName(), accountId);
                                 if ("VIDEO".equals(materialInfo.getType())) {
                                 if ("VIDEO".equals(materialInfo.getType())) {

+ 4 - 0
module-job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/ByteDanceVideoReportJob.java

@@ -22,6 +22,10 @@ public class ByteDanceVideoReportJob {
 
 
     @XxlJob("byteDanceVideoReport")
     @XxlJob("byteDanceVideoReport")
     public ReturnT<String> execute(String param) throws Exception {
     public ReturnT<String> execute(String param) throws Exception {
+        String getDate2 = DateUtils.formatDate(DateUtils.addDay(new Date(), -2));
+        XxlJobLogger.log("头条视频报表清洗数据定时任务开始, 执行时间为:" + getDate2);
+        byteDanceVideoReportDailyService.videoInfoList(getDate2, getDate2);
+
         String getDate = DateUtils.formatDate(DateUtils.addDay(new Date(), -1));
         String getDate = DateUtils.formatDate(DateUtils.addDay(new Date(), -1));
         XxlJobLogger.log("头条视频报表清洗数据定时任务开始, 执行时间为:" + getDate);
         XxlJobLogger.log("头条视频报表清洗数据定时任务开始, 执行时间为:" + getDate);
         byteDanceVideoReportDailyService.videoInfoList(getDate, getDate);
         byteDanceVideoReportDailyService.videoInfoList(getDate, getDate);

+ 8 - 1
module-job-kuaishou/src/main/java/cn/com/ctop/job/kuaishou/handler/KuaishouDailyAccountReportLoadJob.java

@@ -31,6 +31,7 @@ public class KuaishouDailyAccountReportLoadJob {
     @XxlJob("kuaishouDailyAccountReport")
     @XxlJob("kuaishouDailyAccountReport")
     public ReturnT<String> execute(String param) throws Exception {
     public ReturnT<String> execute(String param) throws Exception {
         Date getDate = DateUtils.addDay(new Date(), -1);
         Date getDate = DateUtils.addDay(new Date(), -1);
+        Date getDate2 = DateUtils.addDay(new Date(), -2);
         //1:查询当日数据
         //1:查询当日数据
         List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
         List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
         if (null == tokens || tokens.size() <= 0) {
         if (null == tokens || tokens.size() <= 0) {
@@ -39,7 +40,13 @@ public class KuaishouDailyAccountReportLoadJob {
         }
         }
 
 
         executorService = Executors.newFixedThreadPool(3);
         executorService = Executors.newFixedThreadPool(3);
-        tokens.forEach(token -> executorService.submit(() -> kuaishouInterfaceService.getAdvertiserReportDaily(token, getDate, getDate)));
+        tokens.forEach(token ->
+                executorService.submit(() ->{
+                    kuaishouInterfaceService.getAdvertiserReportDaily(token, getDate2, getDate2);
+                    kuaishouInterfaceService.getAdvertiserReportDaily(token, getDate, getDate);
+                }
+                )
+        );
         return ReturnT.SUCCESS;
         return ReturnT.SUCCESS;
     }
     }
 }
 }

+ 24 - 2
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/controller/BatchController.java

@@ -114,16 +114,38 @@ public class BatchController {
             spendVo.setBudget(budget);
             spendVo.setBudget(budget);
             result.setSuccess(true);
             result.setSuccess(true);
             result.setResult(spendVo);
             result.setResult(spendVo);
-
             Thread thread = new Thread() {
             Thread thread = new Thread() {
                 @Override
                 @Override
                 public void run() {
                 public void run() {
                     iKuaishouInterfaceService.getAppList(oauthToken.getAccountId(), oauthToken.getAccessToken());
                     iKuaishouInterfaceService.getAppList(oauthToken.getAccountId(), oauthToken.getAccessToken());
+                }
+            };
+            thread.start();
+
+            Thread thread1 = new Thread() {
+                @Override
+                public void run() {
                     iKuaishouInterfaceService.getPopulationList(accountId, oauthToken.getAccessToken());
                     iKuaishouInterfaceService.getPopulationList(accountId, oauthToken.getAccessToken());
+                }
+            };
+            thread1.start();
+
+            Thread thread2 = new Thread() {
+                @Override
+                public void run() {
                     iKuaishouInterfaceService.getVideoList(oauthToken, null, null);
                     iKuaishouInterfaceService.getVideoList(oauthToken, null, null);
                 }
                 }
             };
             };
-            thread.start();
+            thread2.start();
+
+            Thread thread3 = new Thread() {
+                @Override
+                public void run() {
+                    iKuaishouInterfaceService.getSuZaoList(oauthToken.getAccessToken(), oauthToken.getAccountId(), 1);
+                    videoGetService.getKeyFrame(oauthToken);
+                }
+            };
+            thread3.start();
 
 
         } catch (Exception e) {
         } catch (Exception e) {
             e.printStackTrace();
             e.printStackTrace();

+ 2 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/IKuaiShouVideoGetService.java

@@ -1,5 +1,6 @@
 package cn.com.ctop.kuaishou.modules.batch.service;
 package cn.com.ctop.kuaishou.modules.batch.service;
 
 
+import cn.com.ctop.common.module.entity.CtopOauthToken;
 import cn.com.ctop.kuaishou.modules.batch.entity.KuaiShouImageGet;
 import cn.com.ctop.kuaishou.modules.batch.entity.KuaiShouImageGet;
 import cn.com.ctop.kuaishou.modules.batch.entity.KuaiShouVideoGet;
 import cn.com.ctop.kuaishou.modules.batch.entity.KuaiShouVideoGet;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
@@ -16,4 +17,5 @@ import java.util.List;
 public interface IKuaiShouVideoGetService extends IService<KuaiShouVideoGet> {
 public interface IKuaiShouVideoGetService extends IService<KuaiShouVideoGet> {
     void replaceBatch(List<KuaiShouVideoGet> videoGets);
     void replaceBatch(List<KuaiShouVideoGet> videoGets);
 
 
+    void getKeyFrame(CtopOauthToken token);
 }
 }

+ 98 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/impl/KuaiShouVideoGetServiceImpl.java

@@ -1,13 +1,28 @@
 package cn.com.ctop.kuaishou.modules.batch.service.impl;
 package cn.com.ctop.kuaishou.modules.batch.service.impl;
 
 
+import cn.com.ctop.common.module.entity.CtopOauthToken;
+import cn.com.ctop.common.module.entity.MaterialCutFrame;
+import cn.com.ctop.common.module.service.IMaterialCutFrameService;
+import cn.com.ctop.common.module.utils.Check;
+import cn.com.ctop.common.module.utils.HttpUtils;
+import cn.com.ctop.common.module.utils.LoadFileUtil;
 import cn.com.ctop.kuaishou.modules.batch.entity.KuaiShouVideoGet;
 import cn.com.ctop.kuaishou.modules.batch.entity.KuaiShouVideoGet;
 import cn.com.ctop.kuaishou.modules.batch.mapper.KuaiShouVideoGetMapper;
 import cn.com.ctop.kuaishou.modules.batch.mapper.KuaiShouVideoGetMapper;
 import cn.com.ctop.kuaishou.modules.batch.service.IKuaiShouVideoGetService;
 import cn.com.ctop.kuaishou.modules.batch.service.IKuaiShouVideoGetService;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 import org.springframework.stereotype.Service;
 
 
+import java.io.IOException;
+import java.util.HashMap;
 import java.util.List;
 import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 
 
 /**
 /**
  * @Description: 快手-获取视频接口
  * @Description: 快手-获取视频接口
@@ -19,9 +34,92 @@ import java.util.List;
 public class KuaiShouVideoGetServiceImpl extends ServiceImpl<KuaiShouVideoGetMapper, KuaiShouVideoGet> implements IKuaiShouVideoGetService {
 public class KuaiShouVideoGetServiceImpl extends ServiceImpl<KuaiShouVideoGetMapper, KuaiShouVideoGet> implements IKuaiShouVideoGetService {
     @Autowired
     @Autowired
     private KuaiShouVideoGetMapper videoGetMapper;
     private KuaiShouVideoGetMapper videoGetMapper;
+    @Autowired
+    private IMaterialCutFrameService materialCutFrameService;
+    static ExecutorService executorService = Executors.newFixedThreadPool(15);
+
+
+    @Value("${oss.replace.download}")
+    private String downloadUrl;
 
 
     @Override
     @Override
     public void replaceBatch(List<KuaiShouVideoGet> videoGets) {
     public void replaceBatch(List<KuaiShouVideoGet> videoGets) {
         videoGetMapper.replaceBatch(videoGets);
         videoGetMapper.replaceBatch(videoGets);
     }
     }
+
+    /**
+     * 获取关键帧
+     *
+     * @param
+     */
+    @Override
+    public void getKeyFrame(CtopOauthToken token) {
+        try {
+            QueryWrapper<KuaiShouVideoGet> videoGetQueryWrapper = new QueryWrapper<>();
+            videoGetQueryWrapper.eq("account_id", token.getAccountId());
+            videoGetQueryWrapper.eq("channel_type", 1);
+            List<KuaiShouVideoGet> kuaiShouVideoGets = videoGetMapper.selectList(videoGetQueryWrapper);
+            if (Check.isNull(kuaiShouVideoGets)) {
+                return;
+            }
+            for (KuaiShouVideoGet videoGet : kuaiShouVideoGets) {
+                if (!Check.isNull(videoGet.getSignature())) {
+                    QueryWrapper<MaterialCutFrame> cutFrameQueryWrapper = new QueryWrapper<>();
+                    cutFrameQueryWrapper.eq("video_signature", videoGet.getSignature());
+                    cutFrameQueryWrapper.last("limit 1");
+                    MaterialCutFrame one = materialCutFrameService.getOne(cutFrameQueryWrapper);
+                    if (!Check.isNull(one)) {
+                        continue;
+                    }
+                    executorService.submit(new Runnable() {
+                        @Override
+                        public void run() {
+                            String url = "https://ad.e.kuaishou.com/rest/openapi/v1/tool/key_frame";
+                            JSONArray photoArr = new JSONArray();
+                            photoArr.add(videoGet.getPhotoId());
+                            Map<String, String> headers = new HashMap<>();
+                            headers.put("Access-Token", token.getAccessToken());
+                            headers.put("Content-Type", "application/json");
+                            JSONObject requestJson = new JSONObject();
+                            requestJson.put("advertiser_id", token.getAccountId());
+                            requestJson.put("photo_ids", photoArr);
+                            String result = HttpUtils.kuaiShouhttpPostRequest(url, requestJson.toJSONString(), headers);
+                            JSONObject resultJson = JSONObject.parseObject(result);
+                            if (!Check.isNull(resultJson)) {
+                                Integer code = resultJson.getInteger("code");
+                                if (code == 0) {
+                                    JSONArray dataArr = resultJson.getJSONArray("data");
+                                    if (!Check.isNull(dataArr)) {
+                                        for (int i = 0; i < dataArr.size(); i++) {
+                                            String imageUrl = dataArr.getString(i);
+                                            String localPath = LoadFileUtil.downLoadFromUrl(imageUrl, downloadUrl);
+                                            String md5 = null;
+                                            try {
+                                                md5 = LoadFileUtil.getMD5(localPath);
+                                            } catch (IOException e) {
+                                                e.printStackTrace();
+                                                continue;
+                                            }
+                                            MaterialCutFrame materialCutFrame = new MaterialCutFrame();
+                                            materialCutFrame.setImageIndex(i);
+                                            materialCutFrame.setVideoSignature(videoGet.getSignature());
+                                            materialCutFrame.setUrl(imageUrl);
+                                            materialCutFrame.setSignature(md5);
+                                            materialCutFrameService.save(materialCutFrame);
+                                            LoadFileUtil.delFile(localPath);
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    });
+                }
+
+            }
+
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+
+    }
 }
 }

+ 6 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/impl/KuaishouInterfaceServiceImpl.java

@@ -291,6 +291,8 @@ public class KuaishouInterfaceServiceImpl implements IKuaishouInterfaceService {
                 detailJson.remove("show");
                 detailJson.remove("show");
                 detailJson.remove("adScene");
                 detailJson.remove("adScene");
                 detailJson.remove("stat_hour");
                 detailJson.remove("stat_hour");
+                detailJson.remove("event_valid_clues");
+                detailJson.remove("event_valid_clues_cost");
                 KuaishouReportDailyAccount kuaishouReportDailyAccount = JsonUtil.jsonToObj(detailJson, KuaishouReportDailyAccount.class);
                 KuaishouReportDailyAccount kuaishouReportDailyAccount = JsonUtil.jsonToObj(detailJson, KuaishouReportDailyAccount.class);
                 kuaishouReportDailyAccount.setAccountId(token.getAccountId());
                 kuaishouReportDailyAccount.setAccountId(token.getAccountId());
                 dailyAccountMapper.insertSelective(kuaishouReportDailyAccount);
                 dailyAccountMapper.insertSelective(kuaishouReportDailyAccount);
@@ -3011,6 +3013,10 @@ public class KuaishouInterfaceServiceImpl implements IKuaishouInterfaceService {
         }
         }
         JSONObject dataJson = resultJson.getJSONObject("data");
         JSONObject dataJson = resultJson.getJSONObject("data");
         if (!Check.isNull(dataJson)) {
         if (!Check.isNull(dataJson)) {
+            Integer total_count = dataJson.getInteger("total_count");
+            if (total_count > 2000) {
+                return;
+            }
             JSONArray details = dataJson.getJSONArray("details");
             JSONArray details = dataJson.getJSONArray("details");
             if (Check.isNull(details)) {
             if (Check.isNull(details)) {
                 return;
                 return;

+ 3 - 1
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/xml/KuaishouReportDailyCampaignMapper.xml

@@ -64,7 +64,9 @@
         event_next_day_stay_cost,
         event_next_day_stay_cost,
         event_next_day_stay_ratio,
         event_next_day_stay_ratio,
         event_next_day_stay,
         event_next_day_stay,
-        play_3s_ratio
+        play_3s_ratio,
+        @event_valid_clues,
+        @event_valid_clues_cost
         )
         )
           set account_id = #{accountId}
           set account_id = #{accountId}
     </insert>
     </insert>

+ 3 - 1
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/xml/KuaishouReportDailyCreativeMapper.xml

@@ -69,7 +69,9 @@
         event_next_day_stay_cost,
         event_next_day_stay_cost,
         event_next_day_stay_ratio,
         event_next_day_stay_ratio,
         event_next_day_stay,
         event_next_day_stay,
-        play_3s_ratio
+        play_3s_ratio,
+        @event_valid_clues,
+        @event_valid_clues_cost
         )
         )
           set account_id = #{accountId}
           set account_id = #{accountId}
     </insert>
     </insert>

+ 3 - 1
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/xml/KuaishouReportDailyGroupMapper.xml

@@ -66,7 +66,9 @@
         event_next_day_stay_cost,
         event_next_day_stay_cost,
         event_next_day_stay_ratio,
         event_next_day_stay_ratio,
         event_next_day_stay,
         event_next_day_stay,
-        play_3s_ratio
+        play_3s_ratio,
+        @event_valid_clues,
+        @event_valid_clues_cost
         )
         )
           set account_id = #{accountId}
           set account_id = #{accountId}
     </insert>
     </insert>

+ 2 - 2
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/controller/ByteDanceVideoReportDailyController.java

@@ -56,15 +56,15 @@ public class ByteDanceVideoReportDailyController {
 			 dateList.add(endStr);
 			 dateList.add(endStr);
 		 }
 		 }
 
 
-
 		 final ExecutorService executorService = Executors.newFixedThreadPool(8);
 		 final ExecutorService executorService = Executors.newFixedThreadPool(8);
 		 dateList.forEach(date -> {
 		 dateList.forEach(date -> {
 			 executorService.submit(new Runnable() {
 			 executorService.submit(new Runnable() {
 				 @Override
 				 @Override
 				 public void run() {
 				 public void run() {
 					 try {
 					 try {
-						 System.out.println("头条视频按天汇总接口现在在跑的时间为:" + date + "~" + date);
+						 log.info("头条视频按天汇总接口现在在跑的时间为:" + date + "~" + date);
 						 videoReportDailyService.videoInfoList(date, date);
 						 videoReportDailyService.videoInfoList(date, date);
+						 log.info("头条视频按天汇总接口运行结束,时间为" + date + "~" + date);
 					 } catch (Exception e) {
 					 } catch (Exception e) {
 						 e.printStackTrace();
 						 e.printStackTrace();
 					 } finally {
 					 } finally {

+ 3 - 3
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/controller/BytedanceHomepageReportCtrl.java

@@ -25,10 +25,10 @@ public class BytedanceHomepageReportCtrl {
     IBytedanceHomepageReportService bytedanceHomepageReportService;
     IBytedanceHomepageReportService bytedanceHomepageReportService;
 
 
     @PostMapping("/report/week")
     @PostMapping("/report/week")
-    public Result<Map<String,JSONObject>> getWeekReport(@RequestBody JSONObject requestBody) {
-        Result<Map<String,JSONObject>> result = new Result<>();
+    public Result<Map<String,Object>> getWeekReport(@RequestBody JSONObject requestBody) {
+        Result<Map<String,Object>> result = new Result<>();
         try {
         try {
-            Map<String, JSONObject> weekReport = bytedanceHomepageReportService.getWeekReport(requestBody);
+            Map<String, Object> weekReport = bytedanceHomepageReportService.getWeekReport(requestBody);
             result.setResult(weekReport);
             result.setResult(weekReport);
             result.setSuccess(true);
             result.setSuccess(true);
         } catch (Exception e) {
         } catch (Exception e) {

+ 1 - 2
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/controller/BytedanceReportController.java

@@ -203,7 +203,7 @@ public class BytedanceReportController {
                                 start = DateUtils.addDay(startDate, i);
                                 start = DateUtils.addDay(startDate, i);
                                 end = start;
                                 end = start;
                                 //获取头条素材报表数据
                                 //获取头条素材报表数据
-                                int code = bytedanceReportService.bytedanceMaterialReport(token, start, end);
+                                int code = bytedanceReportService.bytedanceMaterialReportRetry(token, start, end);
                                 if(code ==200 || code == 1){
                                 if(code ==200 || code == 1){
                                     //重试成功清洗数据
                                     //重试成功清洗数据
                                     byteDanceVideoReportDailyService.videoInfoListByAccountId(start,end,accountId);
                                     byteDanceVideoReportDailyService.videoInfoListByAccountId(start,end,accountId);
@@ -212,7 +212,6 @@ public class BytedanceReportController {
 
 
                         } catch (Exception e) {
                         } catch (Exception e) {
                             e.printStackTrace();
                             e.printStackTrace();
-                        } finally {
                         }
                         }
                     }
                     }
                 });
                 });

+ 6 - 6
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/xml/BytedanceHomepageMapper.xml

@@ -79,7 +79,7 @@
        FROM
        FROM
          ctop_bytedance_report_advertiser_daily
          ctop_bytedance_report_advertiser_daily
        WHERE
        WHERE
-         stat_datetime &lt;= #{endDate}
+         stat_datetime &lt;= #{endDate} '00:00:00'
        AND
        AND
          stat_datetime &gt;= #{startDate}
          stat_datetime &gt;= #{startDate}
     </select>
     </select>
@@ -120,7 +120,7 @@
        FROM
        FROM
          ctop_bytedance_report_advertiser_daily
          ctop_bytedance_report_advertiser_daily
        WHERE
        WHERE
-            stat_datetime &lt;= #{endDate}
+            stat_datetime &lt;= #{endDate} '00:00:00'
        AND  stat_datetime &gt;= #{startDate}
        AND  stat_datetime &gt;= #{startDate}
        group by stat_datetime
        group by stat_datetime
        order by stat_datetime asc
        order by stat_datetime asc
@@ -162,7 +162,7 @@
        FROM
        FROM
          ctop_bytedance_report_advertiser_daily
          ctop_bytedance_report_advertiser_daily
        WHERE
        WHERE
-         stat_datetime &lt;= #{endDate}
+         stat_datetime &lt;= #{endDate} '00:00:00'
        AND
        AND
          stat_datetime &gt;= #{startDate}
          stat_datetime &gt;= #{startDate}
       group by DATE_FORMAT(stat_datetime,'%Y-%m')
       group by DATE_FORMAT(stat_datetime,'%Y-%m')
@@ -220,7 +220,7 @@
        LEFT JOIN
        LEFT JOIN
          ctop_user_allocation t2 ON t1.advertiser_id=t2.account_id
          ctop_user_allocation t2 ON t1.advertiser_id=t2.account_id
        WHERE
        WHERE
-         stat_datetime &lt;= #{endDate}
+         stat_datetime &lt;= #{endDate} '00:00:00'
        AND
        AND
          stat_datetime &gt;= #{startDate}
          stat_datetime &gt;= #{startDate}
        GROUP BY t1.advertiser_id
        GROUP BY t1.advertiser_id
@@ -237,7 +237,7 @@
        LEFT JOIN
        LEFT JOIN
          ctop_user_allocation t2 ON t1.advertiser_id=t2.account_id
          ctop_user_allocation t2 ON t1.advertiser_id=t2.account_id
        WHERE
        WHERE
-         stat_datetime &lt;= #{endDate}
+         stat_datetime &lt;= #{endDate} '00:00:00'
        AND
        AND
          stat_datetime &gt;= #{startDate}
          stat_datetime &gt;= #{startDate}
        GROUP BY t1.advertiser_id
        GROUP BY t1.advertiser_id
@@ -254,7 +254,7 @@
        LEFT JOIN
        LEFT JOIN
          ctop_user_allocation t2 ON t1.advertiser_id=t2.account_id
          ctop_user_allocation t2 ON t1.advertiser_id=t2.account_id
        WHERE
        WHERE
-         stat_datetime &lt;= #{endDate}
+         stat_datetime &lt;= #{endDate} '00:00:00'
        AND
        AND
          stat_datetime &gt;= #{startDate}
          stat_datetime &gt;= #{startDate}
        GROUP BY t1.advertiser_id
        GROUP BY t1.advertiser_id

+ 1 - 1
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/IBytedanceHomepageReportService.java

@@ -7,7 +7,7 @@ import java.util.Map;
 
 
 public interface IBytedanceHomepageReportService {
 public interface IBytedanceHomepageReportService {
 
 
-    Map<String,JSONObject> getWeekReport(JSONObject requestBody);
+    Map<String,Object> getWeekReport(JSONObject requestBody);
 
 
     Map<String,List<JSONObject>> getChartByDate(JSONObject requestBody);
     Map<String,List<JSONObject>> getChartByDate(JSONObject requestBody);
 
 

+ 1 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/ByteDanceVideoReportDailyServiceImpl.java

@@ -43,6 +43,7 @@ public class ByteDanceVideoReportDailyServiceImpl extends ServiceImpl<ByteDanceV
         videoInfoListByCompanyId(startDate, endDate, "d57fecdcf7a94d009736d9c850731582");
         videoInfoListByCompanyId(startDate, endDate, "d57fecdcf7a94d009736d9c850731582");
         //华东
         //华东
         videoInfoListByCompanyId(startDate, endDate, "4b10089775c040119e139087517aed88");
         videoInfoListByCompanyId(startDate, endDate, "4b10089775c040119e139087517aed88");
+
     }
     }
 
 
     private void videoInfoListByCompanyId(String startDate, String endDate, String companyId){
     private void videoInfoListByCompanyId(String startDate, String endDate, String companyId){

+ 17 - 8
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/BytedanceHomepageReportServiceImpl.java

@@ -25,8 +25,8 @@ public class BytedanceHomepageReportServiceImpl implements IBytedanceHomepageRep
     BytedanceHomepageMapper bytedanceHomepageMapper;
     BytedanceHomepageMapper bytedanceHomepageMapper;
 
 
     @Override
     @Override
-    public Map<String, JSONObject> getWeekReport(JSONObject requestBody) {
-        Map<String, JSONObject> result = new HashMap<>();
+    public Map<String, Object> getWeekReport(JSONObject requestBody) {
+        Map<String, Object> result = new HashMap<>();
         //当天日期
         //当天日期
         String nowDate = DateUtils.getNowDate(SystemDateConstant.yyyy_MM_dd);
         String nowDate = DateUtils.getNowDate(SystemDateConstant.yyyy_MM_dd);
         //本周汇总数据
         //本周汇总数据
@@ -43,16 +43,25 @@ public class BytedanceHomepageReportServiceImpl implements IBytedanceHomepageRep
         JSONObject top = new JSONObject();
         JSONObject top = new JSONObject();
         top.put("top7", costTop);
         top.put("top7", costTop);
         //周明细环比数据拼接
         //周明细环比数据拼接
-        JSONObject costWeekData = new JSONObject();
+        List<Map<String,Object>> costWeekData = new ArrayList<>();
         JSONObject costWeekLink = new JSONObject();
         JSONObject costWeekLink = new JSONObject();
-        JSONObject clickWeekData = new JSONObject();
+        List<Map<String,Object>> clickWeekData = new ArrayList<>();
         JSONObject clickWeekLink = new JSONObject();
         JSONObject clickWeekLink = new JSONObject();
-        JSONObject showNumWeekData = new JSONObject();
+        List<Map<String,Object>> showNumWeekData = new ArrayList<>();
         JSONObject showNumWeekLink = new JSONObject();
         JSONObject showNumWeekLink = new JSONObject();
         weekDetailData.forEach(day -> {
         weekDetailData.forEach(day -> {
-            costWeekData.put((String) day.get("statDate"), day.get("cost"));
-            clickWeekData.put((String) day.get("statDate"), day.get("click"));
-            showNumWeekData.put((String) day.get("statDate"), day.get("showNum"));
+            Map<String,Object> cost= new HashMap<>();
+            Map<String,Object> click= new HashMap<>();
+            Map<String,Object> showNum= new HashMap<>();
+            cost.put("statDate",day.get("statDate"));
+            cost.put("cost",day.get("cost"));
+            click.put("statDate",day.get("statDate"));
+            click.put("click",day.get("click"));
+            showNum.put("statDate",day.get("statDate"));
+            showNum.put("showNum",day.get("showNum"));
+            costWeekData.add(cost);
+            clickWeekData.add(click);
+            showNumWeekData.add(showNum);
         });
         });
         costWeekLink.put("weekCostSum", weekSumData.get("cost"));
         costWeekLink.put("weekCostSum", weekSumData.get("cost"));
         costWeekLink.put("weekCostSumLink", countLink(weekSumData.getBigDecimal("cost"), weekBeforeSumData.getBigDecimal("cost")));
         costWeekLink.put("weekCostSumLink", countLink(weekSumData.getBigDecimal("cost"), weekBeforeSumData.getBigDecimal("cost")));

+ 10 - 6
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/ReportServiceImpl.java

@@ -7,6 +7,7 @@ import cn.com.ctop.common.module.utils.CtopAdConstant;
 import cn.com.ctop.common.module.utils.HttpUtils;
 import cn.com.ctop.common.module.utils.HttpUtils;
 import cn.com.ctop.common.module.utils.PropertiesUtils;
 import cn.com.ctop.common.module.utils.PropertiesUtils;
 import cn.com.ctop.common.module.vo.ByteDanceAdvertiserReportDTO;
 import cn.com.ctop.common.module.vo.ByteDanceAdvertiserReportDTO;
+import cn.com.ctop.toutiao.modules.material.service.IByteDanceAdvertiserDataService;
 import cn.com.ctop.toutiao.modules.report.entity.*;
 import cn.com.ctop.toutiao.modules.report.entity.*;
 import cn.com.ctop.toutiao.modules.report.mapper.BytedanceReportMaterialDailyMapper;
 import cn.com.ctop.toutiao.modules.report.mapper.BytedanceReportMaterialDailyMapper;
 import cn.com.ctop.toutiao.modules.report.service.*;
 import cn.com.ctop.toutiao.modules.report.service.*;
@@ -57,6 +58,14 @@ public class ReportServiceImpl implements IReportService {
     private IBytedanceCreativeDailyReportService creativeDailyReportService;
     private IBytedanceCreativeDailyReportService creativeDailyReportService;
     @Autowired
     @Autowired
     private IBytedanceCreativeHourlyReportService creativeHourlyReportService;
     private IBytedanceCreativeHourlyReportService creativeHourlyReportService;
+    @Autowired
+    private BytedanceReportMaterialDailyMapper materialDailyMapper;
+    @Autowired
+    private IBytedanceReportService bytedanceReportService;
+    @Autowired
+    private IByteDanceVideoReportDailyService byteDanceVideoReportDailyService;
+    @Autowired
+    private IByteDanceAdvertiserDataService advertiserDataService;
 
 
     @Override
     @Override
     public void getAdvertiserReport(CtopOauthToken token, Date startDate, Date endDate, String bytedanceReportTypePl) {
     public void getAdvertiserReport(CtopOauthToken token, Date startDate, Date endDate, String bytedanceReportTypePl) {
@@ -246,12 +255,6 @@ public class ReportServiceImpl implements IReportService {
         }
         }
     }
     }
 
 
-    @Autowired
-    private BytedanceReportMaterialDailyMapper materialDailyMapper;
-    @Autowired
-    private IBytedanceReportService bytedanceReportService;
-    @Autowired
-    private IByteDanceVideoReportDailyService byteDanceVideoReportDailyService;
 
 
     @Override
     @Override
     public void loadBytedanceHistoryData(CtopOauthToken token) {
     public void loadBytedanceHistoryData(CtopOauthToken token) {
@@ -263,6 +266,7 @@ public class ReportServiceImpl implements IReportService {
         }
         }
 
 
         try {
         try {
+            advertiserDataService.getMaterialList(token);
             log.info("获取历史素材报表:accountId:{}", token.getAccountId());
             log.info("获取历史素材报表:accountId:{}", token.getAccountId());
             String nowDate = DateUtils.getNowDate("yyyy-MM-dd");
             String nowDate = DateUtils.getNowDate("yyyy-MM-dd");
             String endDate = DateUtils.addDay(nowDate, -180);
             String endDate = DateUtils.addDay(nowDate, -180);