|
@@ -9,12 +9,13 @@ import cn.com.ctop.common.module.vo.ResFileDTO;
|
|
|
import cn.com.ctop.crawler.modules.core.entity.CrawlerDouyinMusicTask;
|
|
|
import cn.com.ctop.crawler.modules.core.service.CrawlerDouyinMusicTaskService;
|
|
|
import cn.com.ctop.crawler.modules.douyin.service.DouyinMusicService;
|
|
|
+import cn.com.ctop.kuaishou.modules.batch.entity.KuaiShouGroup;
|
|
|
import cn.com.ctop.kuaishou.modules.batch.entity.KuaishouTemplate;
|
|
|
import cn.com.ctop.kuaishou.modules.batch.service.*;
|
|
|
import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouWebInterfaceService;
|
|
|
import cn.com.ctop.kuaishou.modules.material.entity.KuaishouAccessToken;
|
|
|
import cn.com.ctop.kuaishou.modules.material.entity.KuaishouResultToken;
|
|
|
-import cn.com.ctop.kuaishou.modules.report.service.IKuaiShouDailyAgentService;
|
|
|
+import cn.com.ctop.kuaishou.modules.report.service.*;
|
|
|
import cn.com.ctop.toutiao.modules.link.service.IETLReportBytedanceVideoService;
|
|
|
import cn.com.ctop.toutiao.modules.material.service.IByteDanceAdvertiserDataService;
|
|
|
import cn.com.ctop.toutiao.modules.material.service.IBytedanceEffectVideoInfoService;
|
|
@@ -23,6 +24,7 @@ import cn.com.ctop.toutiao.modules.report.service.*;
|
|
|
import com.alibaba.fastjson.JSONArray;
|
|
|
import com.alibaba.fastjson.JSONObject;
|
|
|
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
|
|
|
+import com.xxl.job.core.context.XxlJobHelper;
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
import org.apache.commons.lang3.StringUtils;
|
|
|
import org.apache.ibatis.annotations.Param;
|
|
@@ -91,6 +93,9 @@ public class TestController {
|
|
|
private MusicInfoService musicInfoService;
|
|
|
@Autowired
|
|
|
private IKuaiShouHistoryReportTaskService kuaiShouHistoryReportTaskService;
|
|
|
+ @Autowired
|
|
|
+ private IKuaishouReportDailyGroupService reportDailyGroupService;
|
|
|
+
|
|
|
|
|
|
static ExecutorService executorService = Executors.newFixedThreadPool(15);
|
|
|
static ExecutorService videoService = Executors.newFixedThreadPool(5);
|
|
@@ -115,8 +120,117 @@ public class TestController {
|
|
|
|
|
|
@Autowired
|
|
|
private IUserAllocationService allocationService;
|
|
|
+ @Autowired
|
|
|
+ private IEtlKuaiShouNewlyService kuaiShouNewlyService;
|
|
|
+ static ExecutorService materialExecutorService = Executors.newFixedThreadPool(8);
|
|
|
+
|
|
|
+ @Autowired
|
|
|
+ private ICtopOauthTokenService tokenService;
|
|
|
+ @Autowired
|
|
|
+ private IEtlKuaiShouGroupDailyReportService etlKuaiShouGroupDailyReportService;
|
|
|
+ @Autowired
|
|
|
+ private IEtlKuaiShouAccountMaterialOverviewService accountMaterialOverviewService;
|
|
|
+
|
|
|
+ @Autowired
|
|
|
+ private IEtlKuaiShouNewlyService newlyService;
|
|
|
+
|
|
|
+
|
|
|
+ @GetMapping("/cleanNewlyData")
|
|
|
+ public void cleanNewlyData(String date) {
|
|
|
+ Date endDate = new Date();
|
|
|
+ List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
|
|
|
+ for (int i = 0; i < 7; i++) {
|
|
|
+ Date getStartDate = DateUtils.addDay(endDate, -i);
|
|
|
+ String s = DateUtils.formatDate(getStartDate);
|
|
|
+ newlyService.cleanNewlyData(s);
|
|
|
+
|
|
|
+
|
|
|
+ for (CtopOauthToken token : tokens) {
|
|
|
+ executorService.submit(new Runnable() {
|
|
|
+ @Override
|
|
|
+ public void run() {
|
|
|
+ accountMaterialOverviewService.cleanAccountMaterialOverview(token.getAccountId(), s);
|
|
|
+ reportDailyGroupService.getAdvertiserGroupReportDaily(token, getStartDate, getStartDate);
|
|
|
+ etlKuaiShouGroupDailyReportService.getGroupReport(token.getAccountId(), token.getAccessToken(), s, s, 1);
|
|
|
+ }
|
|
|
+ });
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+ //1: 获取广告主信息数据
|
|
|
+ // getAdvertiserReportHourly(token, getStartDate, getStartDate);
|
|
|
+ // getAdvertiserReportDaily(token, getStartDate, getStartDate, "adScene");
|
|
|
+ //2:获取广告计划信息数据
|
|
|
+ // getAdvertiserCampaignReportHourly(token, getStartDate, getStartDate);
|
|
|
+ //3:获取广告组信息数据
|
|
|
+ // getAdvertiserGroupReportHourly(token, getStartDate, getStartDate);
|
|
|
+ //4: 获取广告创意信息数据
|
|
|
+ // getAdvertiserCreativeReportHourly(token, getStartDate, getStartDate);
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+ String nowDate = DateUtils.getNowDate("yyyy-MM-dd");
|
|
|
+ newlyService.cleanNewlyData(date);
|
|
|
+ XxlJobHelper.log("快手清洗上新计划数据完成");
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+ @GetMapping("/cleanGroupReport")
|
|
|
+ public void cleanGroupReport() throws Exception {
|
|
|
+ List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
|
|
|
+ if (null == tokens || tokens.isEmpty()) {
|
|
|
+ XxlJobHelper.log("定时获取快手数据异常:未获取到可用的token");
|
|
|
+ XxlJobHelper.handleFail();
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ String nowDate = DateUtils.getNowDate("yyyy-MM-dd");
|
|
|
+ for (CtopOauthToken token : tokens) {
|
|
|
+ executorService.submit(new Runnable() {
|
|
|
+ @Override
|
|
|
+ public void run() {
|
|
|
+ etlKuaiShouGroupDailyReportService.getGroupReport(token.getAccountId(), token.getAccessToken(), nowDate, nowDate, 1);
|
|
|
+ }
|
|
|
+ });
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+ @GetMapping("/cleanAccountMaterialOverview")
|
|
|
+ public void cleanAccountMaterialOverview() throws Exception {
|
|
|
+ String nowDate = DateUtils.getNowDate("yyyy-MM-dd");
|
|
|
+ String yesterdayDate = DateUtils.getAnotherDay("yyyy-MM-dd", nowDate, -1);
|
|
|
+ List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
|
|
|
+ if (null == tokens || tokens.isEmpty()) {
|
|
|
+ XxlJobHelper.log("定时获取快手数据异常:未获取到可用的token");
|
|
|
+ XxlJobHelper.handleFail();
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ for (CtopOauthToken token : tokens) {
|
|
|
+ materialExecutorService.submit(new Runnable() {
|
|
|
+ @Override
|
|
|
+ public void run() {
|
|
|
+ accountMaterialOverviewService.cleanAccountMaterialOverview(token.getAccountId(), yesterdayDate);
|
|
|
+ }
|
|
|
+ });
|
|
|
+ }
|
|
|
|
|
|
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+ @GetMapping("/cleanDate")
|
|
|
+ public String cleanDate(String statDate) throws ParseException {
|
|
|
+ // kuaiShouNewlyService.cleanNewlyData(statDate);
|
|
|
+ // CtopOauthToken
|
|
|
+ CtopOauthToken tokenByAccountId = tokenService.getTokenByAccountId(9901975L);
|
|
|
+ SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
|
|
|
+ Date parse = simpleDateFormat.parse("2021-03-09");
|
|
|
+ reportDailyGroupService.getAdvertiserGroupReportDaily(tokenByAccountId, parse, parse);
|
|
|
+
|
|
|
+ return "success";
|
|
|
+ }
|
|
|
+
|
|
|
@GetMapping("/getVideoByProjectId")
|
|
|
public String getVideoByProjectId(Long projectId) {
|
|
|
QueryWrapper<UserAllocation> queryWrapper = new QueryWrapper<>();
|
|
@@ -416,26 +530,6 @@ public class TestController {
|
|
|
|
|
|
}
|
|
|
|
|
|
- @GetMapping(value = "/getGroupReport")
|
|
|
- public void getGroupReport() throws ParseException {
|
|
|
- List<CtopOauthToken> tokens = oauthTokenService.selectKuaiShouToken();
|
|
|
- SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
|
|
|
- Date parse1 = simpleDateFormat.parse("2020-09-12");
|
|
|
- Date parse2 = simpleDateFormat.parse("2020-09-17");
|
|
|
- for (CtopOauthToken token : tokens) {
|
|
|
-
|
|
|
- suzhaoService.submit(new Runnable() {
|
|
|
- @Override
|
|
|
- public void run() {
|
|
|
- //1: 获取广告主信息数据
|
|
|
- kuaishouInterfaceService.getAdvertiserGroupReportDaily(token, parse1, parse2);
|
|
|
- }
|
|
|
- });
|
|
|
-
|
|
|
-
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
|
|
|
@GetMapping(value = "/syncDouyinMusicAsc")
|
|
|
public String syncDouyinMusicAsc() {
|
|
@@ -687,7 +781,7 @@ public class TestController {
|
|
|
@Override
|
|
|
public void run() {
|
|
|
//1: 获取广告主信息数据
|
|
|
- kuaishouInterfaceService.getAdvertiserReportDaily(token, start, end);
|
|
|
+ kuaishouInterfaceService.getAdvertiserReportDaily(token, start, end, "adScene");
|
|
|
}
|
|
|
});
|
|
|
});
|
|
@@ -742,7 +836,7 @@ public class TestController {
|
|
|
SimpleDateFormat sim = new SimpleDateFormat("yyyy-MM-dd");
|
|
|
Date date1 = sim.parse(startDate);
|
|
|
Date date2 = sim.parse(endDate);
|
|
|
- kuaishouInterfaceService.getAdvertiserReportDaily(token, date1, date2);
|
|
|
+ kuaishouInterfaceService.getAdvertiserReportDaily(token, date1, date2, "adScene");
|
|
|
return "Success";
|
|
|
}
|
|
|
|
|
@@ -759,7 +853,7 @@ public class TestController {
|
|
|
List<CtopOauthToken> ctopOauthTokens = tokenMapper.selectList(tokenQueryWrapper);
|
|
|
if (!Check.isNull(ctopOauthTokens)) {
|
|
|
for (CtopOauthToken token : ctopOauthTokens) {
|
|
|
- kuaishouInterfaceService.getAdvertiserReportDaily(token, date1, date2);
|
|
|
+ kuaishouInterfaceService.getAdvertiserReportDaily(token, date1, date2, "adScene");
|
|
|
}
|
|
|
}
|
|
|
} catch (Exception e) {
|
|
@@ -906,39 +1000,6 @@ public class TestController {
|
|
|
return "success";
|
|
|
}
|
|
|
|
|
|
- @GetMapping(value = "/getGroupData")
|
|
|
- public String getGroupData() {
|
|
|
-
|
|
|
-
|
|
|
- Date endDate = new Date();
|
|
|
- QueryWrapper<CtopOauthToken> tokenQueryWrapper = new QueryWrapper<>();
|
|
|
- tokenQueryWrapper.eq("media_id", 2);
|
|
|
- List<CtopOauthToken> ctopOauthTokens = tokenMapper.selectList(tokenQueryWrapper);
|
|
|
- for (CtopOauthToken token : ctopOauthTokens) {
|
|
|
-
|
|
|
- for (int i = 0; i < 7; i++) {
|
|
|
- Date getStartDate = DateUtils.addDay(endDate, -i);
|
|
|
- //3:获取广告组信息数据
|
|
|
- kuaishouInterfaceService.getAdvertiserGroupReportHourly(token, getStartDate, getStartDate);
|
|
|
- }
|
|
|
- for (int i = 0; i < 6; i++) {
|
|
|
- Date getStartDate = new Date();
|
|
|
- Date getEndDate = new Date();
|
|
|
- if (i == 0) {
|
|
|
- getStartDate = DateUtils.addDay(endDate, -175 + i * 30);
|
|
|
- getEndDate = DateUtils.addDay(endDate, -150 + i * 30);
|
|
|
- } else {
|
|
|
- getStartDate = DateUtils.addDay(endDate, -180 + i * 30);
|
|
|
- getEndDate = DateUtils.addDay(endDate, -150 + i * 30);
|
|
|
- }
|
|
|
- //3:获取广告组信息数据
|
|
|
- kuaishouInterfaceService.getAdvertiserGroupReportDaily(token, getStartDate, getEndDate);
|
|
|
- }
|
|
|
- }
|
|
|
- return "Success";
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
@GetMapping(value = "/gerCreative")
|
|
|
public void gerCreative() {
|
|
|
QueryWrapper<CtopOauthToken> oauthTokenQueryWrapper = new QueryWrapper<>();
|
|
@@ -1030,12 +1091,6 @@ public class TestController {
|
|
|
|
|
|
|
|
|
@Autowired
|
|
|
- private IBytedancePlanDailyReportService planDailyReportService;
|
|
|
- @Autowired
|
|
|
- private ICtopOauthTokenService tokenService;
|
|
|
-
|
|
|
-
|
|
|
- @Autowired
|
|
|
private IByteDanceAdvertiserDataService advertiserDataService;
|
|
|
|
|
|
@GetMapping("getBytedanceData")
|
|
@@ -1083,7 +1138,7 @@ public class TestController {
|
|
|
} catch (ParseException e) {
|
|
|
e.printStackTrace();
|
|
|
}
|
|
|
- kuaishouInterfaceService.getAdvertiserReportDaily(token, startDate, endDate);
|
|
|
+ kuaishouInterfaceService.getAdvertiserReportDaily(token, startDate, endDate, "adScene");
|
|
|
|
|
|
Thread.sleep(1 * 200);
|
|
|
// kuaishouInterfaceService.getAppList(token.getAccountId(), token.getAccessToken());
|
|
@@ -1169,7 +1224,7 @@ public class TestController {
|
|
|
@Override
|
|
|
public void run() {
|
|
|
//1: 获取广告主信息数据
|
|
|
- kuaishouInterfaceService.getAdvertiserReportDaily(token, getDate, getDate);
|
|
|
+ kuaishouInterfaceService.getAdvertiserReportDaily(token, getDate, getDate, "adScene");
|
|
|
}
|
|
|
});
|
|
|
});
|
|
@@ -1370,13 +1425,38 @@ public class TestController {
|
|
|
IBytedanceVideoReportService bytedanceVideoReportService;
|
|
|
@Autowired
|
|
|
IETLReportBytedanceVideoService etlReportBytedanceVideoService;
|
|
|
- @GetMapping("execute")
|
|
|
- public Map<String,Object> execute(Long accountId) throws Exception {
|
|
|
- Map<String,Object>result = new HashMap<>();
|
|
|
- CtopOauthToken token = tokenService.getTokenByAccountId(accountId);
|
|
|
- kuaishouInterfaceService.getGroupList(token,new Date(),new Date());
|
|
|
- kuaishouInterfaceService.getAdvertiserGroupReportDaily(token,new Date(),new Date());
|
|
|
- ResultMapUtils.setResultMap(result,StatusCode.COMMON_SUCCESS);
|
|
|
- return result;
|
|
|
- }
|
|
|
+
|
|
|
+// @GetMapping("execute")
|
|
|
+// public Map<String, Object> execute(String date) throws Exception {
|
|
|
+// Map<String, Object> result = new HashMap<>();
|
|
|
+//// List<CtopOauthToken> tokens = oauthTokenService.selectToutiaoToken();
|
|
|
+//// countDownLatch = new CountDownLatch(tokens.size());
|
|
|
+//// tokens.forEach(token -> executor.submit(() -> {
|
|
|
+//// try {
|
|
|
+//// bytedanceVideoReportService.pullVideoReport(token,date,date,1);
|
|
|
+//// } catch (Exception e) {
|
|
|
+//// XxlJobHelper.log("头条视频报表基础数据拉取异常:{}",e.getMessage());
|
|
|
+//// } finally {
|
|
|
+//// countDownLatch.countDown();
|
|
|
+//// }
|
|
|
+//// }));
|
|
|
+//// try {
|
|
|
+//// countDownLatch.await();
|
|
|
+//// } catch (InterruptedException e) {
|
|
|
+//// e.printStackTrace();
|
|
|
+//// }
|
|
|
+// Map<String, Object> params = new HashMap<>();
|
|
|
+// params.put("stat_datetime", date);
|
|
|
+// etlReportBytedanceVideoService.removeByMap(params);
|
|
|
+// etlReportBytedanceVideoService.cleanETL(date);
|
|
|
+// ResultMapUtils.setResultMap(result, StatusCode.COMMON_SUCCESS);
|
|
|
+// public Map<String, Object> execute (Long accountId) throws Exception {
|
|
|
+// Map<String, Object> result = new HashMap<>();
|
|
|
+// CtopOauthToken token = tokenService.getTokenByAccountId(accountId);
|
|
|
+// kuaishouInterfaceService.getGroupList(token, new Date(), new Date());
|
|
|
+// kuaishouInterfaceService.getAdvertiserGroupReportDaily(token, new Date(), new Date());
|
|
|
+// ResultMapUtils.setResultMap(result, StatusCode.COMMON_SUCCESS);
|
|
|
+// return result;
|
|
|
+// }
|
|
|
+// }
|
|
|
}
|