Ver Fonte

Merge branch 'master' into V1.1.5

yumeng há 4 anos atrás
pai
commit
3858028ece
16 ficheiros alterados com 156 adições e 334 exclusões
  1. 6 233
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/TestController.java
  2. 0 1
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/MergeVideoJob.java
  3. 10 10
      jeecg-boot-module-system/src/main/resources/application-wps.yml
  4. 8 2
      jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java
  5. 8 5
      module-common/src/main/java/cn/com/ctop/common/module/service/impl/CtopOauthTokenServiceImpl.java
  6. 8 0
      module-job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedanceMaterialsLoadJob.java
  7. 3 6
      module-job-kuaishou/src/main/java/cn/com/ctop/job/kuaishou/handler/KuaishouDailyAccountReportLoadJob.java
  8. 27 38
      module-job-kuaishou/src/main/java/cn/com/ctop/job/kuaishou/handler/KuaishouDailyAudienceReportJob.java
  9. 0 1
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/IKuaishouInterfaceService.java
  10. 16 9
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/impl/KuaishouInterfaceServiceImpl.java
  11. 19 1
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/entity/KuaishouReportHourlyAccount.java
  12. 16 0
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/xml/KuaishouReportHourlyAccountMapper.xml
  13. 9 0
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/service/impl/AccountReportServiceImpl.java
  14. 24 24
      module-kuaishou/src/main/resources/kuaishou_config.properties
  15. 2 2
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/material/mapper/xml/BytedanceImageInfoMapper.xml
  16. 0 2
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/material/service/impl/ByteDanceAdvertiserDataServiceImpl.java

+ 6 - 233
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/TestController.java

@@ -9,7 +9,6 @@ import cn.com.ctop.common.module.vo.ResFileDTO;
 import cn.com.ctop.crawler.modules.core.entity.CrawlerDouyinMusicTask;
 import cn.com.ctop.crawler.modules.core.service.CrawlerDouyinMusicTaskService;
 import cn.com.ctop.crawler.modules.douyin.service.DouyinMusicService;
-import cn.com.ctop.kuaishou.modules.batch.entity.KuaiShouGroup;
 import cn.com.ctop.kuaishou.modules.batch.entity.KuaishouTemplate;
 import cn.com.ctop.kuaishou.modules.batch.service.*;
 import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouWebInterfaceService;
@@ -24,7 +23,6 @@ import cn.com.ctop.toutiao.modules.report.service.*;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
-import com.xxl.job.core.context.XxlJobHelper;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.ibatis.annotations.Param;
@@ -93,9 +91,6 @@ public class TestController {
     private MusicInfoService musicInfoService;
     @Autowired
     private IKuaiShouHistoryReportTaskService kuaiShouHistoryReportTaskService;
-    @Autowired
-    private IKuaishouReportDailyGroupService reportDailyGroupService;
-
 
     static ExecutorService executorService = Executors.newFixedThreadPool(15);
     static ExecutorService videoService = Executors.newFixedThreadPool(5);
@@ -133,10 +128,12 @@ public class TestController {
 
     @Autowired
     private IEtlKuaiShouNewlyService newlyService;
+    @Autowired
+    private IKuaishouReportDailyGroupService reportDailyGroupService;
 
 
     @GetMapping("/cleanNewlyData")
-    public void cleanNewlyData(String date) {
+    public void cleanNewlyData() {
         Date endDate = new Date();
         List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
         for (int i = 0; i < 7; i++) {
@@ -155,80 +152,7 @@ public class TestController {
                     }
                 });
             }
-
-
-
-            //1: 获取广告主信息数据
-            //   getAdvertiserReportHourly(token, getStartDate, getStartDate);
-            //  getAdvertiserReportDaily(token, getStartDate, getStartDate, "adScene");
-            //2:获取广告计划信息数据
-            // getAdvertiserCampaignReportHourly(token, getStartDate, getStartDate);
-            //3:获取广告组信息数据
-            // getAdvertiserGroupReportHourly(token, getStartDate, getStartDate);
-            //4: 获取广告创意信息数据
-            // getAdvertiserCreativeReportHourly(token, getStartDate, getStartDate);
-        }
-
-
-        String nowDate = DateUtils.getNowDate("yyyy-MM-dd");
-        newlyService.cleanNewlyData(date);
-        XxlJobHelper.log("快手清洗上新计划数据完成");
-    }
-
-
-    @GetMapping("/cleanGroupReport")
-    public void cleanGroupReport() throws Exception {
-        List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
-        if (null == tokens || tokens.isEmpty()) {
-            XxlJobHelper.log("定时获取快手数据异常:未获取到可用的token");
-            XxlJobHelper.handleFail();
-            return;
-        }
-        String nowDate = DateUtils.getNowDate("yyyy-MM-dd");
-        for (CtopOauthToken token : tokens) {
-            executorService.submit(new Runnable() {
-                @Override
-                public void run() {
-                    etlKuaiShouGroupDailyReportService.getGroupReport(token.getAccountId(), token.getAccessToken(), nowDate, nowDate, 1);
-                }
-            });
-        }
-    }
-
-
-    @GetMapping("/cleanAccountMaterialOverview")
-    public void cleanAccountMaterialOverview() throws Exception {
-        String nowDate = DateUtils.getNowDate("yyyy-MM-dd");
-        String yesterdayDate = DateUtils.getAnotherDay("yyyy-MM-dd", nowDate, -1);
-        List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
-        if (null == tokens || tokens.isEmpty()) {
-            XxlJobHelper.log("定时获取快手数据异常:未获取到可用的token");
-            XxlJobHelper.handleFail();
-            return;
-        }
-        for (CtopOauthToken token : tokens) {
-            materialExecutorService.submit(new Runnable() {
-                @Override
-                public void run() {
-                    accountMaterialOverviewService.cleanAccountMaterialOverview(token.getAccountId(), yesterdayDate);
-                }
-            });
         }
-
-
-    }
-
-
-    @GetMapping("/cleanDate")
-    public String cleanDate(String statDate) throws ParseException {
-        //  kuaiShouNewlyService.cleanNewlyData(statDate);
-        // CtopOauthToken
-        CtopOauthToken tokenByAccountId = tokenService.getTokenByAccountId(9901975L);
-        SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
-        Date parse = simpleDateFormat.parse("2021-03-09");
-        reportDailyGroupService.getAdvertiserGroupReportDaily(tokenByAccountId, parse, parse);
-
-        return "success";
     }
 
     @GetMapping("/getVideoByProjectId")
@@ -759,43 +683,6 @@ public class TestController {
         return "success";
     }
 
-    @GetMapping(value = "/getAccountList")
-    public void getAccountList(String startDate, String endDate) throws JobExecutionException {
-        Thread thread = new Thread() {
-            @Override
-            public void run() {
-                SimpleDateFormat sim = new SimpleDateFormat("yyyy-MM-dd");
-                try {
-                    Date start = sim.parse(startDate);
-                    Date end = sim.parse(endDate);
-                    //1:查询当日数据
-                    List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
-                    if (null == tokens || tokens.size() <= 0) {
-                        log.info("定时获取快手数据异常:未获取到可用的token");
-                        return;
-                    }
-
-                    executorService = Executors.newFixedThreadPool(3);
-                    tokens.forEach(token -> {
-                        executorService.submit(new Runnable() {
-                            @Override
-                            public void run() {
-                                //1: 获取广告主信息数据
-                                kuaishouInterfaceService.getAdvertiserReportDaily(token, start, end, "adScene");
-                            }
-                        });
-                    });
-                } catch (ParseException e) {
-                    e.printStackTrace();
-                }
-
-
-            }
-        };
-        thread.start();
-
-    }
-
 
     @GetMapping(value = "/createCutTask")
     public String createCutTask(String startDate, String endDate) throws IOException {
@@ -819,13 +706,6 @@ public class TestController {
     }
 
 
-   /* @GetMapping(value = "/getToken")
-    public String getToken(Long accountId, String token, String refreshToken, Integer agentType) {
-        oauthTokenService.getKuaiShouRefreshToken(accountId, token, refreshToken, agentType);
-        return "Success";
-    }*/
-
-
     @GetMapping(value = "/getAccountReport")
     public String getAccountReport(Long accountId, String startDate, String endDate) throws ParseException {
         QueryWrapper<CtopOauthToken> oauthTokenQueryWrapper = new QueryWrapper<>();
@@ -836,34 +716,11 @@ public class TestController {
         SimpleDateFormat sim = new SimpleDateFormat("yyyy-MM-dd");
         Date date1 = sim.parse(startDate);
         Date date2 = sim.parse(endDate);
-        kuaishouInterfaceService.getAdvertiserReportDaily(token, date1, date2, "adScene");
+        //    kuaishouInterfaceService.getAdvertiserReportDaily(token, date1, date2);
         return "Success";
     }
 
 
-    @GetMapping(value = "/getAccountReportList")
-    public String getAccountReportList(String startDate, String endDate) {
-        Result<String> result = new Result<>();
-        try {
-            SimpleDateFormat sim = new SimpleDateFormat("yyyy-MM-dd");
-            Date date1 = sim.parse(startDate);
-            Date date2 = sim.parse(endDate);
-            QueryWrapper<CtopOauthToken> tokenQueryWrapper = new QueryWrapper<>();
-            tokenQueryWrapper.eq("media_id", 2);
-            List<CtopOauthToken> ctopOauthTokens = tokenMapper.selectList(tokenQueryWrapper);
-            if (!Check.isNull(ctopOauthTokens)) {
-                for (CtopOauthToken token : ctopOauthTokens) {
-                    kuaishouInterfaceService.getAdvertiserReportDaily(token, date1, date2, "adScene");
-                }
-            }
-        } catch (Exception e) {
-            e.printStackTrace();
-            result.setSuccess(false);
-        }
-        return "success";
-    }
-
-
     @GetMapping(value = "/updateCode")
     public String updateCode() {
         List<MaterialInfo> list = materialInfoService.list();
@@ -1000,6 +857,7 @@ public class TestController {
         return "success";
     }
 
+
     @GetMapping(value = "/gerCreative")
     public void gerCreative() {
         QueryWrapper<CtopOauthToken> oauthTokenQueryWrapper = new QueryWrapper<>();
@@ -1089,7 +947,6 @@ public class TestController {
         }
     }
 
-
     @Autowired
     private IByteDanceAdvertiserDataService advertiserDataService;
 
@@ -1118,38 +975,6 @@ public class TestController {
     private IBytedancePlanHourlyReportService planHourlyReportService;
 
 
-    @GetMapping(value = "/AccountReportByAccount")
-    public void AccountReportByAccount(Long accountId, String startDateStr, String endDateStr) {
-        QueryWrapper<CtopOauthToken> oauthTokenQueryWrapper = new QueryWrapper<>();
-        oauthTokenQueryWrapper.eq("media_id", 2);
-        oauthTokenQueryWrapper.eq("account_id", accountId);
-        oauthTokenQueryWrapper.last("limit 1");
-        CtopOauthToken token = tokenMapper.selectOne(oauthTokenQueryWrapper);
-        executorService.submit(new Runnable() {
-            @Override
-            public void run() {
-                try {
-                    SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
-                    Date startDate = null;
-                    Date endDate = null;
-                    try {
-                        startDate = simpleDateFormat.parse(startDateStr);
-                        endDate = simpleDateFormat.parse(endDateStr);
-                    } catch (ParseException e) {
-                        e.printStackTrace();
-                    }
-                    kuaishouInterfaceService.getAdvertiserReportDaily(token, startDate, endDate, "adScene");
-
-                    Thread.sleep(1 * 200);
-                    //  kuaishouInterfaceService.getAppList(token.getAccountId(), token.getAccessToken());
-                } catch (Exception e) {
-                    e.printStackTrace();
-                }
-            }
-        });
-    }
-
-
     @GetMapping(value = "/getAllCreativeList")
     public void getAllCreativeList() {
 
@@ -1208,27 +1033,7 @@ public class TestController {
 
     }
 
-    @GetMapping(value = "/KuaishouDailyAccountReportLoadJob")
-    public void KuaishouDailyAccountReportLoadJob() {
-        Date getDate = DateUtils.addDay(new Date(), -2);
-        //1:查询当日数据
-        List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
-        if (null == tokens || tokens.size() <= 0) {
-            log.info("定时获取快手数据异常:未获取到可用的token");
-            return;
-        }
 
-        executorService = Executors.newFixedThreadPool(3);
-        tokens.forEach(token -> {
-            executorService.submit(new Runnable() {
-                @Override
-                public void run() {
-                    //1: 获取广告主信息数据
-                    kuaishouInterfaceService.getAdvertiserReportDaily(token, getDate, getDate, "adScene");
-                }
-            });
-        });
-    }
 
     @Autowired
     private IBytedanceEffectVideoInfoService effectVideoInfoService;
@@ -1426,37 +1231,5 @@ public class TestController {
     @Autowired
     IETLReportBytedanceVideoService etlReportBytedanceVideoService;
 
-//    @GetMapping("execute")
-//    public Map<String, Object> execute(String date) throws Exception {
-//        Map<String, Object> result = new HashMap<>();
-////        List<CtopOauthToken> tokens = oauthTokenService.selectToutiaoToken();
-////        countDownLatch = new CountDownLatch(tokens.size());
-////        tokens.forEach(token -> executor.submit(() -> {
-////            try {
-////                bytedanceVideoReportService.pullVideoReport(token,date,date,1);
-////            } catch (Exception e) {
-////                XxlJobHelper.log("头条视频报表基础数据拉取异常:{}",e.getMessage());
-////            } finally {
-////                countDownLatch.countDown();
-////            }
-////        }));
-////        try {
-////            countDownLatch.await();
-////        } catch (InterruptedException e) {
-////            e.printStackTrace();
-////        }
-//        Map<String, Object> params = new HashMap<>();
-//        params.put("stat_datetime", date);
-//        etlReportBytedanceVideoService.removeByMap(params);
-//        etlReportBytedanceVideoService.cleanETL(date);
-//        ResultMapUtils.setResultMap(result, StatusCode.COMMON_SUCCESS);
-//        public Map<String, Object> execute (Long accountId) throws Exception {
-//            Map<String, Object> result = new HashMap<>();
-//            CtopOauthToken token = tokenService.getTokenByAccountId(accountId);
-//            kuaishouInterfaceService.getGroupList(token, new Date(), new Date());
-//            kuaishouInterfaceService.getAdvertiserGroupReportDaily(token, new Date(), new Date());
-//            ResultMapUtils.setResultMap(result, StatusCode.COMMON_SUCCESS);
-//            return result;
-//        }
-//    }
+
 }

+ 0 - 1
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/MergeVideoJob.java

@@ -36,7 +36,6 @@ public class MergeVideoJob implements Job {
 
                     try {
                         String localPath = LoadFileUtil.downLoadFromUrl(url, PropertiesUtils.getValue("kuaishou_config", "video_sava_path"));
-//                        String localPath = LoadFileUtil.downLoadFromUrl(url,"D:/video/");
                         String md5 = LoadFileUtil.getMD5(localPath);
                         LoadFileUtil.delFile(localPath);
                         MaterialInfo materialInfo = new MaterialInfo();

+ 10 - 10
jeecg-boot-module-system/src/main/resources/application-wps.yml

@@ -95,21 +95,21 @@ spring:
         # 通过connectProperties属性来打开mergeSql功能;慢SQL记录
         connectionProperties: druid.stat.mergeSql\=true;druid.stat.slowSqlMillis\=5000
       datasource:
-#        master:
-#          url: jdbc:mysql://139.186.27.96:4000/jiaoyang?characterEncoding=UTF-8&useUnicode=true&useSSL=false
-#          username: data
-#          password: hcst@2021
-#          driver-class-name: com.mysql.jdbc.Driver
+        master:
+          url: jdbc:mysql://139.186.27.96:4000/jiaoyang?characterEncoding=UTF-8&useUnicode=true&useSSL=false
+          username: data
+          password: hcst@2021
+          driver-class-name: com.mysql.jdbc.Driver
 #        master:
 #          url: jdbc:mysql://139.186.151.174:3306/jeecg-boot?characterEncoding=UTF-8&useUnicode=true
 #          username: hcst
 #          password: hcst@2020
 #          driver-class-name: com.mysql.jdbc.Driver
-        master:
-          url: jdbc:mysql://139.186.27.96:4000/jeecg-boot?characterEncoding=UTF-8&useUnicode=true&useSSL=false
-          username: data
-          password: hcst@2021
-          driver-class-name: com.mysql.jdbc.Driver
+#        master:
+#          url: jdbc:mysql://139.186.27.96:4000/jeecg-boot?characterEncoding=UTF-8&useUnicode=true&useSSL=false
+#          username: data
+#          password: hcst@2021
+#          driver-class-name: com.mysql.jdbc.Driver
   #redis 配置
   redis:
     database: 0

+ 8 - 2
jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java

@@ -75,10 +75,16 @@ public class SampleTest {
 
     @Test
     public void loadBytedanceCreativeData() throws ParseException {
+        Date getDate = DateUtils.addDay(new Date(), -1);
+        Date getDate2 = DateUtils.addDay(new Date(), -2);
 
+        CtopOauthToken token = new CtopOauthToken();
+        token.setAccountId(9901975L);
+        token.setAccessToken("1c7cebebc3174c39fe941d136df65b1d");
+        kuaishouInterfaceService.getAdvertiserReportDaily(token, getDate2, getDate,"adScene");
 
-        JSONObject materialOverview = kuaiShouAccountMaterialOverviewService.getMaterialOverview(5473041L, "2021-03-09");
-        System.err.println(materialOverview);
+     /*   JSONObject materialOverview = kuaiShouAccountMaterialOverviewService.getMaterialOverview(5473041L, "2021-03-09");
+        System.err.println(materialOverview);*/
 
       /*  CtopOauthToken token = new CtopOauthToken();
         token.setAccountId(9901975L);

+ 8 - 5
module-common/src/main/java/cn/com/ctop/common/module/service/impl/CtopOauthTokenServiceImpl.java

@@ -13,10 +13,10 @@ import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.xxl.job.core.context.XxlJobHelper;
 import lombok.extern.slf4j.Slf4j;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
+import javax.annotation.Resource;
 import java.util.*;
 
 /**
@@ -36,9 +36,12 @@ public class CtopOauthTokenServiceImpl extends ServiceImpl<CtopOauthTokenMapper,
     private String bytedanceUrlApiPrefix;
     @Value("${bytedance.url.refresh-token}")
     private String bytedanceUrlRefreshToken;
-    @Autowired
+    @Value("${kuaishou.api-url}")
+    private String kuaishouApiUrl;
+
+    @Resource
     private CtopOauthTokenMapper ctopOauthTokenMapper;
-    @Autowired
+    @Resource
     private OauthAgentTokenMapper agentTokenMapper;
 
     @Override
@@ -123,7 +126,7 @@ public class CtopOauthTokenServiceImpl extends ServiceImpl<CtopOauthTokenMapper,
     @Override
     public void getKuaiShouRefreshToken(Long accountId, String token, String refreshToken, Long appId, String secret) {
         log.info("快手刷新token入参,accountId:{},token:{},refreshToken:{}", accountId, token, refreshToken);
-        String url = PropertiesUtils.getConfig("kuaishou_api_url") + "/rest/openapi/oauth2/authorize/refresh_token";
+        String url = kuaishouApiUrl + "/rest/openapi/oauth2/authorize/refresh_token";
         Map<String, String> headers = new HashMap<>();
         headers.put("Content-Type", " application/json");
         Map<String, Object> param = new HashMap<>();
@@ -245,7 +248,7 @@ public class CtopOauthTokenServiceImpl extends ServiceImpl<CtopOauthTokenMapper,
     @Override
     public void getKuaiShouAgentRefreshToken(Long agentId, String accessToken, String refreshToken) {
         log.info("快手刷新token入参,token:{},refreshToken:{}", accessToken, refreshToken);
-        String url = PropertiesUtils.getConfig("kuaishou_api_url") + "/rest/openapi/oauth2/authorize/refresh_token";
+        String url = kuaishouApiUrl + "/rest/openapi/oauth2/authorize/refresh_token";
         Map<String, String> headers = new HashMap<>();
         headers.put("Content-Type", " application/json");
         Map<String, Object> param = new HashMap<>();

+ 8 - 0
module-job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedanceMaterialsLoadJob.java

@@ -3,6 +3,8 @@ package cn.com.ctop.job.bytedance.handler;
 import cn.com.ctop.common.module.entity.CtopOauthToken;
 import cn.com.ctop.common.module.service.ICtopOauthTokenService;
 import cn.com.ctop.toutiao.modules.material.service.IByteDanceAdvertiserDataService;
+import cn.com.ctop.toutiao.modules.material.service.IByteDanceVideoInfoService;
+import cn.com.ctop.toutiao.modules.material.service.IBytedanceImageInfoService;
 import com.xxl.job.core.context.XxlJobHelper;
 import com.xxl.job.core.handler.annotation.XxlJob;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -22,6 +24,10 @@ public class BytedanceMaterialsLoadJob {
     static CountDownLatch countDownLatch = null;
     @Autowired
     private IByteDanceAdvertiserDataService advertiserDataService;
+    @Autowired
+    private IByteDanceVideoInfoService videoInfoService;
+    @Autowired
+    private IBytedanceImageInfoService imageInfoService;
 
     @XxlJob("bytedanceMaterialsLoadJob")
     public void execute() throws Exception{
@@ -47,6 +53,8 @@ public class BytedanceMaterialsLoadJob {
         } catch (InterruptedException e) {
             e.printStackTrace();
         }
+        videoInfoService.updateMaterialType();
+        imageInfoService.updateType();
         XxlJobHelper.log("物料数据同步完成");
     }
 }

+ 3 - 6
module-job-kuaishou/src/main/java/cn/com/ctop/job/kuaishou/handler/KuaishouDailyAccountReportLoadJob.java

@@ -37,13 +37,10 @@ public class KuaishouDailyAccountReportLoadJob {
         if (null == tokens || tokens.isEmpty()) {
             XxlJobHelper.log("定时获取快手数据异常:未获取到可用的token");
             XxlJobHelper.handleFail();
-            return ;
+            return;
         }
-        tokens.forEach(token -> executorService.submit(() ->{
-                    kuaishouInterfaceService.getAdvertiserReportDaily(token, getDate2, getDate2,"adScene");
-                    kuaishouInterfaceService.getAdvertiserReportDaily(token, getDate, getDate,"adScene");
-
-
+        tokens.forEach(token -> executorService.submit(() -> {
+            kuaishouInterfaceService.getAdvertiserReportDaily(token, getDate2, getDate, "adScene");
         }));
     }
 }

+ 27 - 38
module-job-kuaishou/src/main/java/cn/com/ctop/job/kuaishou/handler/KuaishouDailyAudienceReportJob.java

@@ -2,7 +2,6 @@ package cn.com.ctop.job.kuaishou.handler;
 
 import cn.com.ctop.common.module.entity.CtopOauthToken;
 import cn.com.ctop.common.module.service.ICtopOauthTokenService;
-import cn.com.ctop.common.module.utils.Check;
 import cn.com.ctop.kuaishou.modules.batch.entity.KuaiShouGroup;
 import cn.com.ctop.kuaishou.modules.batch.service.IKuaiShouGroupService;
 import cn.com.ctop.kuaishou.modules.batch.service.IKuaishouStrategyService;
@@ -34,53 +33,43 @@ public class KuaishouDailyAudienceReportJob {
     @Autowired
     private IKuaishouStrategyService kuaishouStrategyService;
 
-    static ExecutorService executorService = Executors.newFixedThreadPool(6);
+    static ExecutorService executorService = null;
 
     @XxlJob("kuaishouDailyAudienceReportJob")
     public void execute() throws Exception {
-        List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
-        if (null == tokens || tokens.isEmpty()) {
-            XxlJobHelper.log("定时获取人群分析报表数据异常:未获取到可用的token");
-            XxlJobHelper.handleFail();
-            return;
-        }
-        //查询手淘有效账户数据
-        List<Long> accountIds = kuaishouStrategyService.queryShouTaoAccounts();
-        if (Check.isNull(accountIds) || accountIds.size() == 0) {
-            XxlJobHelper.log("定时获取人群分析报表数据异常:未获取到有效策略账户信息");
-            XxlJobHelper.handleFail();
-            return;
-        }
-        //筛选出有效账户,获取该账户的组级人群报表数据
-        List<CtopOauthToken> activeTokens = new ArrayList<>();
-        for (Long accountId : accountIds) {
-            for (CtopOauthToken token : tokens) {
-                if (token.getAccountId() - accountId == 0) {
-                    activeTokens.add(token);
+        Thread thread = new Thread() {
+            @Override
+            public void run() {
+                List<String> dates = new ArrayList<>();
+                for (int i = 5; i < 36; i++) {
+                    String dat = DateUtils.getLastDay(DateUtils.date2Str(), i);
+                    dates.add(dat);
+                }
+                XxlJobHelper.log("-----------清洗日期:" + dates.toString());
+                //  9556344、9743727、9743738、9812668、9901966、9767034
+                CtopOauthToken token = tokenService.getTokenByAccountId(9812668L);
+                int page = 1;
+                int pageSize = 2000;
+                List<KuaiShouGroup> kuaiShouGroups = kuaiShouGroupService.queryUnitInfoByAccountId(token.getAccountId());
+                executorService = Executors.newFixedThreadPool(8);
+                for (String date : dates) {
+                    executorService.submit(new Runnable() {
+                        @Override
+                        public void run() {
+                            getAudienceReportData(kuaiShouGroups, token, date, "ageSegment", page, pageSize);
+                        }
+                    });
                 }
             }
-        }
-        if (!Check.isNull(activeTokens) && activeTokens.size() > 0) {
-            int page = 1;
-            int pageSize = 2000;
-            //查询两天前日期
-            String date = DateUtils.getLastDay(DateUtils.date2Str(), 2);
-            activeTokens.forEach(token -> executorService.submit(() -> {
-                getAudienceReportData(token, date, "province", page, pageSize);
-                getAudienceReportData(token, date, "city", page, pageSize);
-                getAudienceReportData(token, date, "gender", page, pageSize);
-                getAudienceReportData(token, date, "ageSegment", page, pageSize);
-                getAudienceReportData(token, date, "clientId", page, pageSize);
-                getAudienceReportData(token, date, "businessInterestTags", page, pageSize);
-            }));
-        }
+        };
+        thread.start();
     }
 
-    private void getAudienceReportData(CtopOauthToken token, String date, String type, Integer page, Integer pageSize) {
+    private void getAudienceReportData(List<KuaiShouGroup> kuaiShouGroups, CtopOauthToken token, String date, String type, Integer page, Integer pageSize) {
         try {
-            List<KuaiShouGroup> kuaiShouGroups = kuaiShouGroupService.queryUnitInfoByAccountId(token.getAccountId());
             for (KuaiShouGroup kuaiShouGroup : kuaiShouGroups) {
                 kuaishouAudienceReportDailyService.crowdAnalysisReport(token, date, type, kuaiShouGroup.getCampaignId(), kuaiShouGroup.getUnitId(), page, pageSize);
+                Thread.sleep(600);
             }
         } catch (Exception e) {
             XxlJobHelper.log("定时获取人群分析报表数据异常", e);

+ 0 - 1
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/IKuaishouInterfaceService.java

@@ -284,7 +284,6 @@ public interface IKuaishouInterfaceService {
     void getAdvertiserCampaignReportDaily(CtopOauthToken token, Date startDate, Date endDate);
 
 
-
     /**
      * 获取创意分天报表 暂停使用
      *

+ 16 - 9
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/impl/KuaishouInterfaceServiceImpl.java

@@ -132,7 +132,13 @@ public class KuaishouInterfaceServiceImpl implements IKuaishouInterfaceService {
 
     @Override
     public void getAdvertiserReportHourly(CtopOauthToken token, Date startDate, Date endDate) {
-        getAdvertiserReportHourlyByPage(token, startDate, endDate, 1);
+        try {
+            Thread.sleep(200L);
+            getAdvertiserReportHourlyByPage(token, startDate, endDate, 1);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+
     }
 
 
@@ -173,7 +179,6 @@ public class KuaishouInterfaceServiceImpl implements IKuaishouInterfaceService {
             addList.add(kuaishouReportHourlyAccount);
         }
         hourlyAccountService.replaceBatch(addList);
-
     }
 
 
@@ -274,7 +279,13 @@ public class KuaishouInterfaceServiceImpl implements IKuaishouInterfaceService {
 
     @Override
     public void getAdvertiserReportDaily(CtopOauthToken token, Date startDate, Date endDate, String reportDims) {
-        getAccountDailyReportByPage(token, startDate, endDate, 1, reportDims);
+        try {
+            Thread.sleep(200L);
+            getAccountDailyReportByPage(token, startDate, endDate, 1, reportDims);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+
     }
 
 
@@ -301,12 +312,10 @@ public class KuaishouInterfaceServiceImpl implements IKuaishouInterfaceService {
         param.put("temporal_granularity", "DAILY");
         param.put("page_size", 2000);
         param.put("page", page);
-        System.out.println(param);
         String result = HttpUtils.httpPostRequest(url, param, headers);
         JSONObject resultJson = JSONObject.parseObject(result);
         Integer code = resultJson.getInteger("code");
         String message = resultJson.getString("message");
-        System.out.println(resultJson.toJSONString());
         if (null == code || code != 0) {
             log.error("获取快手广告主报表异常:{},accountId:{}", message, token.getAccountId());
             return;
@@ -467,7 +476,7 @@ public class KuaishouInterfaceServiceImpl implements IKuaishouInterfaceService {
             campaigns.add(hourlyCampaign);
         }
         hourlyCampaignService.replaceBatch(campaigns);
-        getAdvertiserReportHourlyByPage(token, startDate, endDate, page + 1);
+        getCampaignReportHourlyByPage(token, startDate, endDate, page + 1);
     }
 
 
@@ -504,8 +513,6 @@ public class KuaishouInterfaceServiceImpl implements IKuaishouInterfaceService {
     }
 
 
-
-
     @Override
     public void getAdvertiserGroupReportHourly(CtopOauthToken token, Date startDate, Date endDate) {
         getGroupHourlyReportByPage(token, startDate, endDate, 1);
@@ -2548,7 +2555,7 @@ public class KuaishouInterfaceServiceImpl implements IKuaishouInterfaceService {
                     }
 
                 } else {
-                    log.error("获取账户余额失败,"+resultJson.getString("message"));
+                    log.error("获取账户余额失败," + resultJson.getString("message"));
                 }
             } else {
                 log.error("获取账户余额返回为空");

+ 19 - 1
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/entity/KuaishouReportHourlyAccount.java

@@ -104,8 +104,26 @@ public class KuaishouReportHourlyAccount implements Serializable {
     private BigDecimal eventCreditGrantLandingPageCost;
     @JsonProperty
     private BigDecimal eventNextDayStay;
+    @JsonProperty("play_3s_ratio")
+    @TableField("play_3s_ratio")
+    private BigDecimal play3sRatio;
+
+    @JsonProperty
+    private Long share;
+    @JsonProperty
+    private Long comment;
+    @JsonProperty
+    private Long photoLike;
+    @JsonProperty
+    private Long follow;
+    @JsonProperty("cancel_follow")
+    private Long cancelFollow;
+    @JsonProperty
+    private Long report;
+    @JsonProperty
+    private Long block;
     @JsonProperty
-    private Double play3sRatio;
+    private Long negative;
 
     @JsonProperty
     private Date createTime;

+ 16 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/xml/KuaishouReportHourlyAccountMapper.xml

@@ -9,6 +9,14 @@
         photo_show,
         aclick,
         bclick,
+        share,
+        comment,
+        photo_like,
+        follow,
+        cancel_follow,
+        report,
+        block,
+        negative,
         download_started,
         download_completed,
         activation,
@@ -56,6 +64,14 @@
             #{add.photoShow},
             #{add.aclick},
             #{add.bclick},
+            #{add.share},
+            #{add.comment},
+            #{add.photoLike},
+            #{add.follow},
+            #{add.cancelFollow},
+            #{add.report},
+            #{add.block},
+            #{add.negative},
             #{add.downloadStarted},
             #{add.downloadCompleted},
             #{add.activation},

+ 9 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/service/impl/AccountReportServiceImpl.java

@@ -50,6 +50,9 @@ public class AccountReportServiceImpl implements IAccountReportService {
                 // 今日汇总
                 Calendar calendar = Calendar.getInstance();
                 int maxHour = calendar.get(Calendar.HOUR_OF_DAY);
+                if(maxHour == 0){
+                    return null;
+                }
                 String maxHourStr = String.valueOf(maxHour - 1);
                 // 今日 小时汇总数据
                 accountJson = accountReportMapper.selectSumDayHourlyByHourAndDate(nowDate, maxHourStr, accountIds);
@@ -682,4 +685,10 @@ public class AccountReportServiceImpl implements IAccountReportService {
         }
         return returnJson;
     }
+
+    public static void main(String[] args) {
+        Calendar calendar = Calendar.getInstance();
+        int maxHour = calendar.get(Calendar.HOUR_OF_DAY);
+        System.out.println(maxHour);
+    }
 }

+ 24 - 24
module-kuaishou/src/main/resources/kuaishou_config.properties

@@ -1,30 +1,12 @@
-#kuaishou_api_url=https://ad.e.kuaishou.com
-#kuaishou_auth_url=https://ad.e.kuaishou.com
-#kuaishou_appid=27
-#kuaishou_third_appid=328
-#oauth_type=agent
-#kuaishou_secret=QovprB1tNhXptgDc
-#kuaishou_third_secret=1iFTBiMdohkyEQQs
-#kuaishou_callback_url=http://adsp.c-top.com.cn:8080/jeecg-boot/kuaishou
-#kuaishou_third_callback_url=http://callback.shyouteng.com.cn/kuaishou
-#apk_sava_path=/mnt/file/apk
-#image_sava_path=/mnt/file/image
-#video_sava_path=/mnt/file/video
-### 模板视频存放路径
-#model_video_src_path=/mnt/data/src/
-### 片段合成视频存放路径
-#model_video_fill_path=/mnt/data/fill/
-###
-#model_video_final_path=/mnt/data/final/
-
-
-# jiaoyang
 kuaishou_api_url=https://ad.e.kuaishou.com
 kuaishou_auth_url=https://ad.e.kuaishou.com
-kuaishou_appid=328
+kuaishou_appid=27
+kuaishou_third_appid=328
 oauth_type=agent
-kuaishou_secret=1iFTBiMdohkyEQQs
-kuaishou_callback_url=http://callback.shyouteng.com.cn/kuaishou
+kuaishou_secret=QovprB1tNhXptgDc
+kuaishou_third_secret=1iFTBiMdohkyEQQs
+kuaishou_callback_url=http://adsp.c-top.com.cn:8080/jeecg-boot/kuaishou
+kuaishou_third_callback_url=http://callback.shyouteng.com.cn/kuaishou
 apk_sava_path=/mnt/file/apk
 image_sava_path=/mnt/file/image
 video_sava_path=/mnt/file/video
@@ -34,3 +16,21 @@ model_video_src_path=/mnt/data/src/
 model_video_fill_path=/mnt/data/fill/
 ##
 model_video_final_path=/mnt/data/final/
+
+
+# jiaoyang
+#kuaishou_api_url=https://ad.e.kuaishou.com
+#kuaishou_auth_url=https://ad.e.kuaishou.com
+#kuaishou_appid=328
+#oauth_type=agent
+#kuaishou_secret=1iFTBiMdohkyEQQs
+#kuaishou_callback_url=http://callback.shyouteng.com.cn/kuaishou
+#apk_sava_path=/mnt/file/apk
+#image_sava_path=/mnt/file/image
+#video_sava_path=/mnt/file/video
+## 模板视频存放路径
+#model_video_src_path=/mnt/data/src/
+## 片段合成视频存放路径
+#model_video_fill_path=/mnt/data/fill/
+##
+#model_video_final_path=/mnt/data/final/

+ 2 - 2
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/material/mapper/xml/BytedanceImageInfoMapper.xml

@@ -37,13 +37,13 @@
         update
             ctop_bytedance_image_info
         set type = 2
-        where height &gt; width
+        where height &gt; width and type is null
         and (height between 1280 and 2560)  and (width between 720 and 1440);
     </update>
     <update id="updateImageHorizonType">
         update ctop_bytedance_image_info
             set type = 1
-        where height &lt; width
+        where height &lt; width and type is null
         and (width between 1280 and 2560)  and (height between 720 and 1440)
     </update>
 </mapper>

+ 0 - 2
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/material/service/impl/ByteDanceAdvertiserDataServiceImpl.java

@@ -639,8 +639,6 @@ public class ByteDanceAdvertiserDataServiceImpl implements IByteDanceAdvertiserD
     public void getMaterialList(CtopOauthToken token) {
         getVideoByPage(token, "", 1);
         getImageByPage(token, "", 1);
-        videoInfoService.updateMaterialType();
-        imageInfoService.updateType();
     }
 
     @Autowired