|
@@ -9,12 +9,13 @@ import cn.com.ctop.common.module.vo.ResFileDTO;
|
|
import cn.com.ctop.crawler.modules.core.entity.CrawlerDouyinMusicTask;
|
|
import cn.com.ctop.crawler.modules.core.entity.CrawlerDouyinMusicTask;
|
|
import cn.com.ctop.crawler.modules.core.service.CrawlerDouyinMusicTaskService;
|
|
import cn.com.ctop.crawler.modules.core.service.CrawlerDouyinMusicTaskService;
|
|
import cn.com.ctop.crawler.modules.douyin.service.DouyinMusicService;
|
|
import cn.com.ctop.crawler.modules.douyin.service.DouyinMusicService;
|
|
|
|
+import cn.com.ctop.kuaishou.modules.batch.entity.KuaiShouGroup;
|
|
import cn.com.ctop.kuaishou.modules.batch.entity.KuaishouTemplate;
|
|
import cn.com.ctop.kuaishou.modules.batch.entity.KuaishouTemplate;
|
|
import cn.com.ctop.kuaishou.modules.batch.service.*;
|
|
import cn.com.ctop.kuaishou.modules.batch.service.*;
|
|
import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouWebInterfaceService;
|
|
import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouWebInterfaceService;
|
|
import cn.com.ctop.kuaishou.modules.material.entity.KuaishouAccessToken;
|
|
import cn.com.ctop.kuaishou.modules.material.entity.KuaishouAccessToken;
|
|
import cn.com.ctop.kuaishou.modules.material.entity.KuaishouResultToken;
|
|
import cn.com.ctop.kuaishou.modules.material.entity.KuaishouResultToken;
|
|
-import cn.com.ctop.kuaishou.modules.report.service.IKuaiShouDailyAgentService;
|
|
|
|
|
|
+import cn.com.ctop.kuaishou.modules.report.service.*;
|
|
import cn.com.ctop.toutiao.modules.link.service.IETLReportBytedanceVideoService;
|
|
import cn.com.ctop.toutiao.modules.link.service.IETLReportBytedanceVideoService;
|
|
import cn.com.ctop.toutiao.modules.material.service.IByteDanceAdvertiserDataService;
|
|
import cn.com.ctop.toutiao.modules.material.service.IByteDanceAdvertiserDataService;
|
|
import cn.com.ctop.toutiao.modules.material.service.IBytedanceEffectVideoInfoService;
|
|
import cn.com.ctop.toutiao.modules.material.service.IBytedanceEffectVideoInfoService;
|
|
@@ -91,6 +92,9 @@ public class TestController {
|
|
private MusicInfoService musicInfoService;
|
|
private MusicInfoService musicInfoService;
|
|
@Autowired
|
|
@Autowired
|
|
private IKuaiShouHistoryReportTaskService kuaiShouHistoryReportTaskService;
|
|
private IKuaiShouHistoryReportTaskService kuaiShouHistoryReportTaskService;
|
|
|
|
+ @Autowired
|
|
|
|
+ private IKuaishouReportDailyGroupService reportDailyGroupService;
|
|
|
|
+
|
|
|
|
|
|
static ExecutorService executorService = Executors.newFixedThreadPool(15);
|
|
static ExecutorService executorService = Executors.newFixedThreadPool(15);
|
|
static ExecutorService videoService = Executors.newFixedThreadPool(5);
|
|
static ExecutorService videoService = Executors.newFixedThreadPool(5);
|
|
@@ -115,8 +119,84 @@ public class TestController {
|
|
|
|
|
|
@Autowired
|
|
@Autowired
|
|
private IUserAllocationService allocationService;
|
|
private IUserAllocationService allocationService;
|
|
|
|
+ @Autowired
|
|
|
|
+ private IEtlKuaiShouNewlyService kuaiShouNewlyService;
|
|
|
|
+ static ExecutorService materialExecutorService = Executors.newFixedThreadPool(8);
|
|
|
|
+
|
|
|
|
+ @Autowired
|
|
|
|
+ private ICtopOauthTokenService tokenService;
|
|
|
|
+ @Autowired
|
|
|
|
+ private IEtlKuaiShouGroupDailyReportService etlKuaiShouGroupDailyReportService;
|
|
|
|
+ @Autowired
|
|
|
|
+ private IEtlKuaiShouAccountMaterialOverviewService accountMaterialOverviewService;
|
|
|
|
+
|
|
|
|
+ @Autowired
|
|
|
|
+ private IEtlKuaiShouNewlyService newlyService;
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+ @GetMapping("/cleanNewlyData")
|
|
|
|
+ public void cleanNewlyData() {
|
|
|
|
+ String nowDate = DateUtils.getNowDate("yyyy-MM-dd");
|
|
|
|
+ newlyService.cleanNewlyData(nowDate);
|
|
|
|
+ XxlJobHelper.log("快手清洗上新计划数据完成");
|
|
|
|
+ }
|
|
|
|
|
|
|
|
|
|
|
|
+ @GetMapping("/cleanGroupReport")
|
|
|
|
+ public void cleanGroupReport() throws Exception {
|
|
|
|
+ List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
|
|
|
|
+ if (null == tokens || tokens.isEmpty()) {
|
|
|
|
+ XxlJobHelper.log("定时获取快手数据异常:未获取到可用的token");
|
|
|
|
+ XxlJobHelper.handleFail();
|
|
|
|
+ return;
|
|
|
|
+ }
|
|
|
|
+ String nowDate = DateUtils.getNowDate("yyyy-MM-dd");
|
|
|
|
+ for (CtopOauthToken token : tokens) {
|
|
|
|
+ executorService.submit(new Runnable() {
|
|
|
|
+ @Override
|
|
|
|
+ public void run() {
|
|
|
|
+ etlKuaiShouGroupDailyReportService.getGroupReport(token.getAccountId(), token.getAccessToken(), nowDate, nowDate, 1);
|
|
|
|
+ }
|
|
|
|
+ });
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+ @GetMapping("/cleanAccountMaterialOverview")
|
|
|
|
+ public void cleanAccountMaterialOverview() throws Exception {
|
|
|
|
+ String nowDate = DateUtils.getNowDate("yyyy-MM-dd");
|
|
|
|
+ String yesterdayDate = DateUtils.getAnotherDay("yyyy-MM-dd", nowDate, -1);
|
|
|
|
+ List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
|
|
|
|
+ if (null == tokens || tokens.isEmpty()) {
|
|
|
|
+ XxlJobHelper.log("定时获取快手数据异常:未获取到可用的token");
|
|
|
|
+ XxlJobHelper.handleFail();
|
|
|
|
+ return;
|
|
|
|
+ }
|
|
|
|
+ for (CtopOauthToken token : tokens) {
|
|
|
|
+ materialExecutorService.submit(new Runnable() {
|
|
|
|
+ @Override
|
|
|
|
+ public void run() {
|
|
|
|
+ accountMaterialOverviewService.cleanAccountMaterialOverview(token.getAccountId(), yesterdayDate);
|
|
|
|
+ }
|
|
|
|
+ });
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+ @GetMapping("/cleanDate")
|
|
|
|
+ public String cleanDate(String statDate) throws ParseException {
|
|
|
|
+ // kuaiShouNewlyService.cleanNewlyData(statDate);
|
|
|
|
+ // CtopOauthToken
|
|
|
|
+ CtopOauthToken tokenByAccountId = tokenService.getTokenByAccountId(9901975L);
|
|
|
|
+ SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
|
|
|
|
+ Date parse = simpleDateFormat.parse("2021-03-09");
|
|
|
|
+ reportDailyGroupService.getAdvertiserGroupReportDaily(tokenByAccountId, parse, parse);
|
|
|
|
+
|
|
|
|
+ return "success";
|
|
|
|
+ }
|
|
|
|
+
|
|
@GetMapping("/getVideoByProjectId")
|
|
@GetMapping("/getVideoByProjectId")
|
|
public String getVideoByProjectId(Long projectId) {
|
|
public String getVideoByProjectId(Long projectId) {
|
|
QueryWrapper<UserAllocation> queryWrapper = new QueryWrapper<>();
|
|
QueryWrapper<UserAllocation> queryWrapper = new QueryWrapper<>();
|
|
@@ -416,26 +496,6 @@ public class TestController {
|
|
|
|
|
|
}
|
|
}
|
|
|
|
|
|
- @GetMapping(value = "/getGroupReport")
|
|
|
|
- public void getGroupReport() throws ParseException {
|
|
|
|
- List<CtopOauthToken> tokens = oauthTokenService.selectKuaiShouToken();
|
|
|
|
- SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
|
|
|
|
- Date parse1 = simpleDateFormat.parse("2020-09-12");
|
|
|
|
- Date parse2 = simpleDateFormat.parse("2020-09-17");
|
|
|
|
- for (CtopOauthToken token : tokens) {
|
|
|
|
-
|
|
|
|
- suzhaoService.submit(new Runnable() {
|
|
|
|
- @Override
|
|
|
|
- public void run() {
|
|
|
|
- //1: 获取广告主信息数据
|
|
|
|
- kuaishouInterfaceService.getAdvertiserGroupReportDaily(token, parse1, parse2);
|
|
|
|
- }
|
|
|
|
- });
|
|
|
|
-
|
|
|
|
-
|
|
|
|
- }
|
|
|
|
- }
|
|
|
|
-
|
|
|
|
|
|
|
|
@GetMapping(value = "/syncDouyinMusicAsc")
|
|
@GetMapping(value = "/syncDouyinMusicAsc")
|
|
public String syncDouyinMusicAsc() {
|
|
public String syncDouyinMusicAsc() {
|
|
@@ -687,7 +747,7 @@ public class TestController {
|
|
@Override
|
|
@Override
|
|
public void run() {
|
|
public void run() {
|
|
//1: 获取广告主信息数据
|
|
//1: 获取广告主信息数据
|
|
- kuaishouInterfaceService.getAdvertiserReportDaily(token, start, end,"adScene");
|
|
|
|
|
|
+ kuaishouInterfaceService.getAdvertiserReportDaily(token, start, end, "adScene");
|
|
}
|
|
}
|
|
});
|
|
});
|
|
});
|
|
});
|
|
@@ -742,7 +802,7 @@ public class TestController {
|
|
SimpleDateFormat sim = new SimpleDateFormat("yyyy-MM-dd");
|
|
SimpleDateFormat sim = new SimpleDateFormat("yyyy-MM-dd");
|
|
Date date1 = sim.parse(startDate);
|
|
Date date1 = sim.parse(startDate);
|
|
Date date2 = sim.parse(endDate);
|
|
Date date2 = sim.parse(endDate);
|
|
- kuaishouInterfaceService.getAdvertiserReportDaily(token, date1, date2,"adScene");
|
|
|
|
|
|
+ kuaishouInterfaceService.getAdvertiserReportDaily(token, date1, date2, "adScene");
|
|
return "Success";
|
|
return "Success";
|
|
}
|
|
}
|
|
|
|
|
|
@@ -759,7 +819,7 @@ public class TestController {
|
|
List<CtopOauthToken> ctopOauthTokens = tokenMapper.selectList(tokenQueryWrapper);
|
|
List<CtopOauthToken> ctopOauthTokens = tokenMapper.selectList(tokenQueryWrapper);
|
|
if (!Check.isNull(ctopOauthTokens)) {
|
|
if (!Check.isNull(ctopOauthTokens)) {
|
|
for (CtopOauthToken token : ctopOauthTokens) {
|
|
for (CtopOauthToken token : ctopOauthTokens) {
|
|
- kuaishouInterfaceService.getAdvertiserReportDaily(token, date1, date2,"adScene");
|
|
|
|
|
|
+ kuaishouInterfaceService.getAdvertiserReportDaily(token, date1, date2, "adScene");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} catch (Exception e) {
|
|
} catch (Exception e) {
|
|
@@ -906,39 +966,6 @@ public class TestController {
|
|
return "success";
|
|
return "success";
|
|
}
|
|
}
|
|
|
|
|
|
- @GetMapping(value = "/getGroupData")
|
|
|
|
- public String getGroupData() {
|
|
|
|
-
|
|
|
|
-
|
|
|
|
- Date endDate = new Date();
|
|
|
|
- QueryWrapper<CtopOauthToken> tokenQueryWrapper = new QueryWrapper<>();
|
|
|
|
- tokenQueryWrapper.eq("media_id", 2);
|
|
|
|
- List<CtopOauthToken> ctopOauthTokens = tokenMapper.selectList(tokenQueryWrapper);
|
|
|
|
- for (CtopOauthToken token : ctopOauthTokens) {
|
|
|
|
-
|
|
|
|
- for (int i = 0; i < 7; i++) {
|
|
|
|
- Date getStartDate = DateUtils.addDay(endDate, -i);
|
|
|
|
- //3:获取广告组信息数据
|
|
|
|
- kuaishouInterfaceService.getAdvertiserGroupReportHourly(token, getStartDate, getStartDate);
|
|
|
|
- }
|
|
|
|
- for (int i = 0; i < 6; i++) {
|
|
|
|
- Date getStartDate = new Date();
|
|
|
|
- Date getEndDate = new Date();
|
|
|
|
- if (i == 0) {
|
|
|
|
- getStartDate = DateUtils.addDay(endDate, -175 + i * 30);
|
|
|
|
- getEndDate = DateUtils.addDay(endDate, -150 + i * 30);
|
|
|
|
- } else {
|
|
|
|
- getStartDate = DateUtils.addDay(endDate, -180 + i * 30);
|
|
|
|
- getEndDate = DateUtils.addDay(endDate, -150 + i * 30);
|
|
|
|
- }
|
|
|
|
- //3:获取广告组信息数据
|
|
|
|
- kuaishouInterfaceService.getAdvertiserGroupReportDaily(token, getStartDate, getEndDate);
|
|
|
|
- }
|
|
|
|
- }
|
|
|
|
- return "Success";
|
|
|
|
- }
|
|
|
|
-
|
|
|
|
-
|
|
|
|
@GetMapping(value = "/gerCreative")
|
|
@GetMapping(value = "/gerCreative")
|
|
public void gerCreative() {
|
|
public void gerCreative() {
|
|
QueryWrapper<CtopOauthToken> oauthTokenQueryWrapper = new QueryWrapper<>();
|
|
QueryWrapper<CtopOauthToken> oauthTokenQueryWrapper = new QueryWrapper<>();
|
|
@@ -1029,10 +1056,7 @@ public class TestController {
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
- @Autowired
|
|
|
|
- private IBytedancePlanDailyReportService planDailyReportService;
|
|
|
|
- @Autowired
|
|
|
|
- private ICtopOauthTokenService tokenService;
|
|
|
|
|
|
+
|
|
|
|
|
|
|
|
|
|
@Autowired
|
|
@Autowired
|
|
@@ -1083,7 +1107,7 @@ public class TestController {
|
|
} catch (ParseException e) {
|
|
} catch (ParseException e) {
|
|
e.printStackTrace();
|
|
e.printStackTrace();
|
|
}
|
|
}
|
|
- kuaishouInterfaceService.getAdvertiserReportDaily(token, startDate, endDate,"adScene");
|
|
|
|
|
|
+ kuaishouInterfaceService.getAdvertiserReportDaily(token, startDate, endDate, "adScene");
|
|
|
|
|
|
Thread.sleep(1 * 200);
|
|
Thread.sleep(1 * 200);
|
|
// kuaishouInterfaceService.getAppList(token.getAccountId(), token.getAccessToken());
|
|
// kuaishouInterfaceService.getAppList(token.getAccountId(), token.getAccessToken());
|
|
@@ -1169,7 +1193,7 @@ public class TestController {
|
|
@Override
|
|
@Override
|
|
public void run() {
|
|
public void run() {
|
|
//1: 获取广告主信息数据
|
|
//1: 获取广告主信息数据
|
|
- kuaishouInterfaceService.getAdvertiserReportDaily(token, getDate, getDate,"adScene");
|
|
|
|
|
|
+ kuaishouInterfaceService.getAdvertiserReportDaily(token, getDate, getDate, "adScene");
|
|
}
|
|
}
|
|
});
|
|
});
|
|
});
|
|
});
|
|
@@ -1370,7 +1394,31 @@ public class TestController {
|
|
IBytedanceVideoReportService bytedanceVideoReportService;
|
|
IBytedanceVideoReportService bytedanceVideoReportService;
|
|
@Autowired
|
|
@Autowired
|
|
IETLReportBytedanceVideoService etlReportBytedanceVideoService;
|
|
IETLReportBytedanceVideoService etlReportBytedanceVideoService;
|
|
|
|
+
|
|
@GetMapping("execute")
|
|
@GetMapping("execute")
|
|
|
|
+ public Map<String, Object> execute(String date) throws Exception {
|
|
|
|
+ Map<String, Object> result = new HashMap<>();
|
|
|
|
+// List<CtopOauthToken> tokens = oauthTokenService.selectToutiaoToken();
|
|
|
|
+// countDownLatch = new CountDownLatch(tokens.size());
|
|
|
|
+// tokens.forEach(token -> executor.submit(() -> {
|
|
|
|
+// try {
|
|
|
|
+// bytedanceVideoReportService.pullVideoReport(token,date,date,1);
|
|
|
|
+// } catch (Exception e) {
|
|
|
|
+// XxlJobHelper.log("头条视频报表基础数据拉取异常:{}",e.getMessage());
|
|
|
|
+// } finally {
|
|
|
|
+// countDownLatch.countDown();
|
|
|
|
+// }
|
|
|
|
+// }));
|
|
|
|
+// try {
|
|
|
|
+// countDownLatch.await();
|
|
|
|
+// } catch (InterruptedException e) {
|
|
|
|
+// e.printStackTrace();
|
|
|
|
+// }
|
|
|
|
+ Map<String, Object> params = new HashMap<>();
|
|
|
|
+ params.put("stat_datetime", date);
|
|
|
|
+ etlReportBytedanceVideoService.removeByMap(params);
|
|
|
|
+ etlReportBytedanceVideoService.cleanETL(date);
|
|
|
|
+ ResultMapUtils.setResultMap(result, StatusCode.COMMON_SUCCESS);
|
|
public Map<String,Object> execute(Long accountId) throws Exception {
|
|
public Map<String,Object> execute(Long accountId) throws Exception {
|
|
Map<String,Object>result = new HashMap<>();
|
|
Map<String,Object>result = new HashMap<>();
|
|
CtopOauthToken token = tokenService.getTokenByAccountId(accountId);
|
|
CtopOauthToken token = tokenService.getTokenByAccountId(accountId);
|
|
@@ -1379,4 +1427,61 @@ public class TestController {
|
|
ResultMapUtils.setResultMap(result,StatusCode.COMMON_SUCCESS);
|
|
ResultMapUtils.setResultMap(result,StatusCode.COMMON_SUCCESS);
|
|
return result;
|
|
return result;
|
|
}
|
|
}
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+ @Autowired
|
|
|
|
+ private IKuaishouAudienceReportDailyService kuaishouAudienceReportDailyService;
|
|
|
|
+ @Autowired
|
|
|
|
+ private IKuaiShouGroupService kuaiShouGroupService;
|
|
|
|
+ @Autowired
|
|
|
|
+ private IKuaishouStrategyService kuaishouStrategyService;
|
|
|
|
+
|
|
|
|
+ @GetMapping("reportDaily")
|
|
|
|
+ public void ReportDaily() throws Exception {
|
|
|
|
+ CtopOauthToken token1 = tokenService.getTokenByAccountId(9743738L);
|
|
|
|
+ CtopOauthToken token2 = tokenService.getTokenByAccountId(9743727L);
|
|
|
|
+ CtopOauthToken token3 = tokenService.getTokenByAccountId(9556344L);
|
|
|
|
+ List<CtopOauthToken> tokens = new ArrayList<>();
|
|
|
|
+ tokens.add(token1);
|
|
|
|
+ tokens.add(token2);
|
|
|
|
+ tokens.add(token3);
|
|
|
|
+ //查询两天前数据 策略
|
|
|
|
+ List<String> dates = new ArrayList<>();
|
|
|
|
+ for (int i = 2; i < 33; i++) {
|
|
|
|
+ String dat = DateUtils.getLastDay(DateUtils.date2Str(), i);
|
|
|
|
+ dates.add(dat);
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ ExecutorService executorServic1e = Executors.newFixedThreadPool(30);
|
|
|
|
+ CtopOauthToken token = tokenService.getTokenByAccountId(9743738L);
|
|
|
|
+ dates.forEach(date -> executorServic1e.submit(() -> {
|
|
|
|
+ if (null == tokens || tokens.isEmpty()) {
|
|
|
|
+ XxlJobHelper.log("定时获取人群分析报表数据异常:未获取到可用的token");
|
|
|
|
+ XxlJobHelper.handleFail();
|
|
|
|
+ return;
|
|
|
|
+ }
|
|
|
|
+ int page = 1;
|
|
|
|
+ int pageSize = 2000;
|
|
|
|
+ getAudienceReportData(token, date, "ageSegment", page, pageSize);
|
|
|
|
+ getAudienceReportData(token, date, "gender", page, pageSize);
|
|
|
|
+ getAudienceReportData(token, date, "businessInterestTags", page, pageSize);
|
|
|
|
+ getAudienceReportData(token, date, "city", page, pageSize);
|
|
|
|
+ getAudienceReportData(token, date, "clientId", page, pageSize);
|
|
|
|
+ getAudienceReportData(token, date, "province", page, pageSize);
|
|
|
|
+ }));
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ private void getAudienceReportData(CtopOauthToken token, String date, String type, Integer page, Integer pageSize) {
|
|
|
|
+ System.out.println(date + "————:" + token.getAccountId());
|
|
|
|
+ try {
|
|
|
|
+ List<KuaiShouGroup> kuaiShouGroups = kuaiShouGroupService.queryUnitInfoByAccountId(token.getAccountId());
|
|
|
|
+ for (KuaiShouGroup kuaiShouGroup : kuaiShouGroups) {
|
|
|
|
+ kuaishouAudienceReportDailyService.crowdAnalysisReport(token, date, type, kuaiShouGroup.getCampaignId(), kuaiShouGroup.getUnitId(), page, pageSize);
|
|
|
|
+ Thread.sleep(410);
|
|
|
|
+ }
|
|
|
|
+ } catch (Exception e) {
|
|
|
|
+ XxlJobHelper.log("定时获取人群分析报表数据异常", e);
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+
|
|
}
|
|
}
|