|
@@ -9,7 +9,6 @@ import cn.com.ctop.common.module.vo.ResFileDTO;
|
|
|
import cn.com.ctop.crawler.modules.core.entity.CrawlerDouyinMusicTask;
|
|
|
import cn.com.ctop.crawler.modules.core.service.CrawlerDouyinMusicTaskService;
|
|
|
import cn.com.ctop.crawler.modules.douyin.service.DouyinMusicService;
|
|
|
-import cn.com.ctop.kuaishou.modules.batch.entity.KuaiShouGroup;
|
|
|
import cn.com.ctop.kuaishou.modules.batch.entity.KuaishouTemplate;
|
|
|
import cn.com.ctop.kuaishou.modules.batch.service.*;
|
|
|
import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouWebInterfaceService;
|
|
@@ -24,7 +23,6 @@ import cn.com.ctop.toutiao.modules.report.service.*;
|
|
|
import com.alibaba.fastjson.JSONArray;
|
|
|
import com.alibaba.fastjson.JSONObject;
|
|
|
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
|
|
|
-import com.xxl.job.core.context.XxlJobHelper;
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
import org.apache.commons.lang3.StringUtils;
|
|
|
import org.apache.ibatis.annotations.Param;
|
|
@@ -93,9 +91,6 @@ public class TestController {
|
|
|
private MusicInfoService musicInfoService;
|
|
|
@Autowired
|
|
|
private IKuaiShouHistoryReportTaskService kuaiShouHistoryReportTaskService;
|
|
|
- @Autowired
|
|
|
- private IKuaishouReportDailyGroupService reportDailyGroupService;
|
|
|
-
|
|
|
|
|
|
static ExecutorService executorService = Executors.newFixedThreadPool(15);
|
|
|
static ExecutorService videoService = Executors.newFixedThreadPool(5);
|
|
@@ -133,10 +128,12 @@ public class TestController {
|
|
|
|
|
|
@Autowired
|
|
|
private IEtlKuaiShouNewlyService newlyService;
|
|
|
+ @Autowired
|
|
|
+ private IKuaishouReportDailyGroupService reportDailyGroupService;
|
|
|
|
|
|
|
|
|
@GetMapping("/cleanNewlyData")
|
|
|
- public void cleanNewlyData(String date) {
|
|
|
+ public void cleanNewlyData() {
|
|
|
Date endDate = new Date();
|
|
|
List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
|
|
|
for (int i = 0; i < 7; i++) {
|
|
@@ -155,80 +152,7 @@ public class TestController {
|
|
|
}
|
|
|
});
|
|
|
}
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
- //1: 获取广告主信息数据
|
|
|
- // getAdvertiserReportHourly(token, getStartDate, getStartDate);
|
|
|
- // getAdvertiserReportDaily(token, getStartDate, getStartDate, "adScene");
|
|
|
- //2:获取广告计划信息数据
|
|
|
- // getAdvertiserCampaignReportHourly(token, getStartDate, getStartDate);
|
|
|
- //3:获取广告组信息数据
|
|
|
- // getAdvertiserGroupReportHourly(token, getStartDate, getStartDate);
|
|
|
- //4: 获取广告创意信息数据
|
|
|
- // getAdvertiserCreativeReportHourly(token, getStartDate, getStartDate);
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- String nowDate = DateUtils.getNowDate("yyyy-MM-dd");
|
|
|
- newlyService.cleanNewlyData(date);
|
|
|
- XxlJobHelper.log("快手清洗上新计划数据完成");
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- @GetMapping("/cleanGroupReport")
|
|
|
- public void cleanGroupReport() throws Exception {
|
|
|
- List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
|
|
|
- if (null == tokens || tokens.isEmpty()) {
|
|
|
- XxlJobHelper.log("定时获取快手数据异常:未获取到可用的token");
|
|
|
- XxlJobHelper.handleFail();
|
|
|
- return;
|
|
|
- }
|
|
|
- String nowDate = DateUtils.getNowDate("yyyy-MM-dd");
|
|
|
- for (CtopOauthToken token : tokens) {
|
|
|
- executorService.submit(new Runnable() {
|
|
|
- @Override
|
|
|
- public void run() {
|
|
|
- etlKuaiShouGroupDailyReportService.getGroupReport(token.getAccountId(), token.getAccessToken(), nowDate, nowDate, 1);
|
|
|
- }
|
|
|
- });
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- @GetMapping("/cleanAccountMaterialOverview")
|
|
|
- public void cleanAccountMaterialOverview() throws Exception {
|
|
|
- String nowDate = DateUtils.getNowDate("yyyy-MM-dd");
|
|
|
- String yesterdayDate = DateUtils.getAnotherDay("yyyy-MM-dd", nowDate, -1);
|
|
|
- List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
|
|
|
- if (null == tokens || tokens.isEmpty()) {
|
|
|
- XxlJobHelper.log("定时获取快手数据异常:未获取到可用的token");
|
|
|
- XxlJobHelper.handleFail();
|
|
|
- return;
|
|
|
- }
|
|
|
- for (CtopOauthToken token : tokens) {
|
|
|
- materialExecutorService.submit(new Runnable() {
|
|
|
- @Override
|
|
|
- public void run() {
|
|
|
- accountMaterialOverviewService.cleanAccountMaterialOverview(token.getAccountId(), yesterdayDate);
|
|
|
- }
|
|
|
- });
|
|
|
}
|
|
|
-
|
|
|
-
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- @GetMapping("/cleanDate")
|
|
|
- public String cleanDate(String statDate) throws ParseException {
|
|
|
- // kuaiShouNewlyService.cleanNewlyData(statDate);
|
|
|
- // CtopOauthToken
|
|
|
- CtopOauthToken tokenByAccountId = tokenService.getTokenByAccountId(9901975L);
|
|
|
- SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
|
|
|
- Date parse = simpleDateFormat.parse("2021-03-09");
|
|
|
- reportDailyGroupService.getAdvertiserGroupReportDaily(tokenByAccountId, parse, parse);
|
|
|
-
|
|
|
- return "success";
|
|
|
}
|
|
|
|
|
|
@GetMapping("/getVideoByProjectId")
|
|
@@ -759,43 +683,6 @@ public class TestController {
|
|
|
return "success";
|
|
|
}
|
|
|
|
|
|
- @GetMapping(value = "/getAccountList")
|
|
|
- public void getAccountList(String startDate, String endDate) throws JobExecutionException {
|
|
|
- Thread thread = new Thread() {
|
|
|
- @Override
|
|
|
- public void run() {
|
|
|
- SimpleDateFormat sim = new SimpleDateFormat("yyyy-MM-dd");
|
|
|
- try {
|
|
|
- Date start = sim.parse(startDate);
|
|
|
- Date end = sim.parse(endDate);
|
|
|
- //1:查询当日数据
|
|
|
- List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
|
|
|
- if (null == tokens || tokens.size() <= 0) {
|
|
|
- log.info("定时获取快手数据异常:未获取到可用的token");
|
|
|
- return;
|
|
|
- }
|
|
|
-
|
|
|
- executorService = Executors.newFixedThreadPool(3);
|
|
|
- tokens.forEach(token -> {
|
|
|
- executorService.submit(new Runnable() {
|
|
|
- @Override
|
|
|
- public void run() {
|
|
|
- //1: 获取广告主信息数据
|
|
|
- kuaishouInterfaceService.getAdvertiserReportDaily(token, start, end, "adScene");
|
|
|
- }
|
|
|
- });
|
|
|
- });
|
|
|
- } catch (ParseException e) {
|
|
|
- e.printStackTrace();
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- }
|
|
|
- };
|
|
|
- thread.start();
|
|
|
-
|
|
|
- }
|
|
|
-
|
|
|
|
|
|
@GetMapping(value = "/createCutTask")
|
|
|
public String createCutTask(String startDate, String endDate) throws IOException {
|
|
@@ -819,13 +706,6 @@ public class TestController {
|
|
|
}
|
|
|
|
|
|
|
|
|
- /* @GetMapping(value = "/getToken")
|
|
|
- public String getToken(Long accountId, String token, String refreshToken, Integer agentType) {
|
|
|
- oauthTokenService.getKuaiShouRefreshToken(accountId, token, refreshToken, agentType);
|
|
|
- return "Success";
|
|
|
- }*/
|
|
|
-
|
|
|
-
|
|
|
@GetMapping(value = "/getAccountReport")
|
|
|
public String getAccountReport(Long accountId, String startDate, String endDate) throws ParseException {
|
|
|
QueryWrapper<CtopOauthToken> oauthTokenQueryWrapper = new QueryWrapper<>();
|
|
@@ -836,34 +716,11 @@ public class TestController {
|
|
|
SimpleDateFormat sim = new SimpleDateFormat("yyyy-MM-dd");
|
|
|
Date date1 = sim.parse(startDate);
|
|
|
Date date2 = sim.parse(endDate);
|
|
|
- kuaishouInterfaceService.getAdvertiserReportDaily(token, date1, date2, "adScene");
|
|
|
+ // kuaishouInterfaceService.getAdvertiserReportDaily(token, date1, date2);
|
|
|
return "Success";
|
|
|
}
|
|
|
|
|
|
|
|
|
- @GetMapping(value = "/getAccountReportList")
|
|
|
- public String getAccountReportList(String startDate, String endDate) {
|
|
|
- Result<String> result = new Result<>();
|
|
|
- try {
|
|
|
- SimpleDateFormat sim = new SimpleDateFormat("yyyy-MM-dd");
|
|
|
- Date date1 = sim.parse(startDate);
|
|
|
- Date date2 = sim.parse(endDate);
|
|
|
- QueryWrapper<CtopOauthToken> tokenQueryWrapper = new QueryWrapper<>();
|
|
|
- tokenQueryWrapper.eq("media_id", 2);
|
|
|
- List<CtopOauthToken> ctopOauthTokens = tokenMapper.selectList(tokenQueryWrapper);
|
|
|
- if (!Check.isNull(ctopOauthTokens)) {
|
|
|
- for (CtopOauthToken token : ctopOauthTokens) {
|
|
|
- kuaishouInterfaceService.getAdvertiserReportDaily(token, date1, date2, "adScene");
|
|
|
- }
|
|
|
- }
|
|
|
- } catch (Exception e) {
|
|
|
- e.printStackTrace();
|
|
|
- result.setSuccess(false);
|
|
|
- }
|
|
|
- return "success";
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
@GetMapping(value = "/updateCode")
|
|
|
public String updateCode() {
|
|
|
List<MaterialInfo> list = materialInfoService.list();
|
|
@@ -1000,6 +857,7 @@ public class TestController {
|
|
|
return "success";
|
|
|
}
|
|
|
|
|
|
+
|
|
|
@GetMapping(value = "/gerCreative")
|
|
|
public void gerCreative() {
|
|
|
QueryWrapper<CtopOauthToken> oauthTokenQueryWrapper = new QueryWrapper<>();
|
|
@@ -1089,7 +947,6 @@ public class TestController {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
-
|
|
|
@Autowired
|
|
|
private IByteDanceAdvertiserDataService advertiserDataService;
|
|
|
|
|
@@ -1118,38 +975,6 @@ public class TestController {
|
|
|
private IBytedancePlanHourlyReportService planHourlyReportService;
|
|
|
|
|
|
|
|
|
- @GetMapping(value = "/AccountReportByAccount")
|
|
|
- public void AccountReportByAccount(Long accountId, String startDateStr, String endDateStr) {
|
|
|
- QueryWrapper<CtopOauthToken> oauthTokenQueryWrapper = new QueryWrapper<>();
|
|
|
- oauthTokenQueryWrapper.eq("media_id", 2);
|
|
|
- oauthTokenQueryWrapper.eq("account_id", accountId);
|
|
|
- oauthTokenQueryWrapper.last("limit 1");
|
|
|
- CtopOauthToken token = tokenMapper.selectOne(oauthTokenQueryWrapper);
|
|
|
- executorService.submit(new Runnable() {
|
|
|
- @Override
|
|
|
- public void run() {
|
|
|
- try {
|
|
|
- SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
|
|
|
- Date startDate = null;
|
|
|
- Date endDate = null;
|
|
|
- try {
|
|
|
- startDate = simpleDateFormat.parse(startDateStr);
|
|
|
- endDate = simpleDateFormat.parse(endDateStr);
|
|
|
- } catch (ParseException e) {
|
|
|
- e.printStackTrace();
|
|
|
- }
|
|
|
- kuaishouInterfaceService.getAdvertiserReportDaily(token, startDate, endDate, "adScene");
|
|
|
-
|
|
|
- Thread.sleep(1 * 200);
|
|
|
- // kuaishouInterfaceService.getAppList(token.getAccountId(), token.getAccessToken());
|
|
|
- } catch (Exception e) {
|
|
|
- e.printStackTrace();
|
|
|
- }
|
|
|
- }
|
|
|
- });
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
@GetMapping(value = "/getAllCreativeList")
|
|
|
public void getAllCreativeList() {
|
|
|
|
|
@@ -1208,27 +1033,7 @@ public class TestController {
|
|
|
|
|
|
}
|
|
|
|
|
|
- @GetMapping(value = "/KuaishouDailyAccountReportLoadJob")
|
|
|
- public void KuaishouDailyAccountReportLoadJob() {
|
|
|
- Date getDate = DateUtils.addDay(new Date(), -2);
|
|
|
- //1:查询当日数据
|
|
|
- List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
|
|
|
- if (null == tokens || tokens.size() <= 0) {
|
|
|
- log.info("定时获取快手数据异常:未获取到可用的token");
|
|
|
- return;
|
|
|
- }
|
|
|
|
|
|
- executorService = Executors.newFixedThreadPool(3);
|
|
|
- tokens.forEach(token -> {
|
|
|
- executorService.submit(new Runnable() {
|
|
|
- @Override
|
|
|
- public void run() {
|
|
|
- //1: 获取广告主信息数据
|
|
|
- kuaishouInterfaceService.getAdvertiserReportDaily(token, getDate, getDate, "adScene");
|
|
|
- }
|
|
|
- });
|
|
|
- });
|
|
|
- }
|
|
|
|
|
|
@Autowired
|
|
|
private IBytedanceEffectVideoInfoService effectVideoInfoService;
|
|
@@ -1426,37 +1231,5 @@ public class TestController {
|
|
|
@Autowired
|
|
|
IETLReportBytedanceVideoService etlReportBytedanceVideoService;
|
|
|
|
|
|
-// @GetMapping("execute")
|
|
|
-// public Map<String, Object> execute(String date) throws Exception {
|
|
|
-// Map<String, Object> result = new HashMap<>();
|
|
|
-//// List<CtopOauthToken> tokens = oauthTokenService.selectToutiaoToken();
|
|
|
-//// countDownLatch = new CountDownLatch(tokens.size());
|
|
|
-//// tokens.forEach(token -> executor.submit(() -> {
|
|
|
-//// try {
|
|
|
-//// bytedanceVideoReportService.pullVideoReport(token,date,date,1);
|
|
|
-//// } catch (Exception e) {
|
|
|
-//// XxlJobHelper.log("头条视频报表基础数据拉取异常:{}",e.getMessage());
|
|
|
-//// } finally {
|
|
|
-//// countDownLatch.countDown();
|
|
|
-//// }
|
|
|
-//// }));
|
|
|
-//// try {
|
|
|
-//// countDownLatch.await();
|
|
|
-//// } catch (InterruptedException e) {
|
|
|
-//// e.printStackTrace();
|
|
|
-//// }
|
|
|
-// Map<String, Object> params = new HashMap<>();
|
|
|
-// params.put("stat_datetime", date);
|
|
|
-// etlReportBytedanceVideoService.removeByMap(params);
|
|
|
-// etlReportBytedanceVideoService.cleanETL(date);
|
|
|
-// ResultMapUtils.setResultMap(result, StatusCode.COMMON_SUCCESS);
|
|
|
-// public Map<String, Object> execute (Long accountId) throws Exception {
|
|
|
-// Map<String, Object> result = new HashMap<>();
|
|
|
-// CtopOauthToken token = tokenService.getTokenByAccountId(accountId);
|
|
|
-// kuaishouInterfaceService.getGroupList(token, new Date(), new Date());
|
|
|
-// kuaishouInterfaceService.getAdvertiserGroupReportDaily(token, new Date(), new Date());
|
|
|
-// ResultMapUtils.setResultMap(result, StatusCode.COMMON_SUCCESS);
|
|
|
-// return result;
|
|
|
-// }
|
|
|
-// }
|
|
|
+
|
|
|
}
|