package org.jeecg; import cn.com.ctop.common.module.entity.BindAccountLogin; import cn.com.ctop.common.module.entity.CtopOauthToken; import cn.com.ctop.common.module.entity.OauthAgentToken; import cn.com.ctop.common.module.service.IBindAccountLoginService; import cn.com.ctop.common.module.service.ICtopOauthTokenService; import cn.com.ctop.common.module.service.IOauthAgentTokenService; import cn.com.ctop.common.module.service.ISendMessageService; import cn.com.ctop.common.module.utils.Check; import cn.com.ctop.common.module.utils.CtopAdConstant; import cn.com.ctop.kuaishou.modules.ai.entity.AiKuaishouUnitLevelOperationRecord; import cn.com.ctop.kuaishou.modules.ai.mapper.AiKuaishouUnitLevelOperationRecordMapper; import cn.com.ctop.kuaishou.modules.batch.entity.KuaishouOperationRecord; import cn.com.ctop.kuaishou.modules.batch.mapper.KuaishouOperationRecordMapper; import cn.com.ctop.kuaishou.modules.batch.service.IKuaiShouReportDailyMaterialService; import cn.com.ctop.kuaishou.modules.batch.service.IKuaishouInterfaceService; import cn.com.ctop.kuaishou.modules.batch.service.IKuaishouInterfaceService2; import cn.com.ctop.kuaishou.modules.batch.service.IKuaishouProgramCreativeService; import cn.com.ctop.kuaishou.modules.dimension.service.IKuaiShouDimensionStrategyService; import cn.com.ctop.kuaishou.modules.dimension.service.IKuaiShouDimensionVariableService; import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouWebInterfaceService; import cn.com.ctop.kuaishou.modules.material.service.IEtlKuaishouVideoInfoService; import cn.com.ctop.kuaishou.modules.report.mapper.EtlKuaishouAccountMaterialReportDailyMapper; import cn.com.ctop.kuaishou.modules.report.mapper.KuaiShouDailyAgentMapper; import cn.com.ctop.kuaishou.modules.report.service.IKuaishouAudienceReportDailyService; import cn.com.ctop.kuaishou.modules.report.service.IKuaishouReportDailyImageService; import cn.com.ctop.kuaishou.modules.report.service.IRuleKuaiShouPlanService; import cn.com.ctop.oa.modules.service.IWechatNoListService; import cn.com.ctop.toutiao.modules.link.service.IETLReportBytedanceVideoService; import cn.com.ctop.toutiao.modules.material.service.IByteDanceAdvertiserDataService; import cn.com.ctop.toutiao.modules.report.entity.BytedanceCheckReportTaskInfo; import cn.com.ctop.toutiao.modules.report.service.*; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import com.xxl.job.core.context.XxlJobHelper; import lombok.extern.slf4j.Slf4j; import org.jeecg.common.util.DateUtils; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.junit4.SpringRunner; import javax.annotation.Resource; import java.text.ParseException; import java.util.*; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @RunWith(SpringRunner.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles("wps") @Slf4j public class SampleTest { @Autowired private IBytedanceVideoEtlInfoService bytedanceVideoEtlInfoService; @Autowired IBytedanceVideoReportService bytedanceVideoReportService; @Autowired IETLReportBytedanceVideoService ietlReportBytedanceVideoService; @Autowired IKuaishouReportDailyImageService kuaishouReportDailyImageService; @Autowired IRuleByteDanceAccountService ruleByteDanceAccountService; @Autowired private ICtopOauthTokenService tokenService; @Autowired private IBindAccountLoginService bindAccountLoginService; @Autowired private IKuaishouWebInterfaceService kuaishouWebInterfaceService; @Autowired private IByteDanceAdvertiserDataService advertiserDataService; @Autowired private IWechatNoListService wechatNoListService; @Autowired private IETLReportBytedanceVideoService bytedanceVideoService; @Autowired private IByteDanceVideoReportDailyService videoReportDailyService; @Resource private EtlKuaishouAccountMaterialReportDailyMapper etlKuaishouAccountMaterialReportDailyMapper; @Autowired private IReportService reportService; @Resource private IBytedanceCheckReportTaskInfoService checkReportTaskInfoService; @Autowired private IEtlBytedanceReportVideoDailyService etlBytedanceReportVideoDailyService; static ExecutorService executorService = null; static CountDownLatch countDownLatch = null; @Autowired private IBytedanceReportService bytedanceReportService; @Autowired private IETLReportBytedanceVideoService etlReportBytedanceVideoService; @Autowired private IKuaishouInterfaceService iKuaishouInterfaceService; @Autowired private IKuaiShouDimensionVariableService kuaiShouDimensionVariableService; @Autowired private IKuaiShouDimensionStrategyService dimensionStrategyService; @Autowired private IKuaiShouDimensionVariableService dimensionVariableService; @Autowired IRuleKuaiShouPlanService ruleKuaiShouPlanService; @Autowired private ISendMessageService sendMessageService; @Autowired private IKuaishouProgramCreativeService kuaishouProgramCreativeService; @Autowired private IOauthAgentTokenService agentTokenService; @Autowired private KuaiShouDailyAgentMapper agentMapper; @Test public void loadKuaishouCookie() throws ParseException { List appList = new ArrayList<>(); appList.add(27L); appList.add(165893351L); for (int i = 0; i < appList.size(); i++) { Long appId = appList.get(i); OauthAgentToken agentToken = agentTokenService.getTokenByAppId(appId); System.err.println(agentToken); // getAgentReport(agentToken.getAccessToken(), agentToken.getAgentId(), startDate, endDate, 1); } } @Test public void deleteKuaishouComment() { Long start = System.currentTimeMillis(); List loginList = bindAccountLoginService.getListByParams(CtopAdConstant.PLATFORM_TYPE_KUAISHOU_PY, 1); if (loginList != null && !loginList.isEmpty()) { executorService = Executors.newFixedThreadPool(5); countDownLatch = new CountDownLatch(loginList.size()); loginList.forEach(login -> executorService.submit(() -> { try { kuaishouWebInterfaceService.deleteAllComment(new HashMap<>(), login); } catch (Exception e) { log.error(e.getMessage(), e); } finally { countDownLatch.countDown(); } })); try { countDownLatch.await(); } catch (InterruptedException e) { e.printStackTrace(); } Long end = System.currentTimeMillis(); log.info("快手删评论所用时长:{}毫秒", end - start); } } @Test public void loadBytedanceVideoReportData() { CtopOauthToken token = tokenService.getTokenByAccountId(1696831659659278L); advertiserDataService.getMaterialList(token); } @Test public void etlBytedanceVideoInfo() { String startDate = "2021-02-01"; for (int i = 0; i < 100; i++) { bytedanceVideoService.cleanData(DateUtils.addDay(startDate, i)); } } @Test public void getData() { //汇创 人员 // wechatUserInfoService.getUserList(); // //考勤记录 // wechatCheckinDataService.getCheckinData("2021-03-01 00:00:00", "2021-03-31 23:59:59"); //异常记录 wechatNoListService.getNoDateByNo("2021-03-01 00:00:00", "2021-03-31 23:59:59", "0"); //优容 // wechatUserInfoService.getYRUserList(); // wechatCheckinDataService.getYRCheckinData("2021-03-01 00:00:00", "2021-03-31 23:59:59"); wechatNoListService.getNoDateByNo("2021-03-01 00:00:00", "2021-03-31 23:59:59", "1"); } @Test public void formatVideoReportData() { for (int i = 1; i < 45; i++) { Date getDate = DateUtils.addDay(new Date(), -i); String date = DateUtils.formatDate(getDate); videoReportDailyService.videoInfoList(date, date); } } @Test public void loadBDAccoutData() { Date startDate = DateUtils.addDay(new Date(), -2000); for (int i = 0; i <= 2000; i++) { Date getDate = DateUtils.addDay(startDate, i); bytedanceVideoEtlInfoService.etlBytedanceVideoInfo(getDate); } } @Test public void etlKuaishouVideoInfo() { etlKuaishouAccountMaterialReportDailyMapper.etlKuaishouAccountMaterialReportDailyData(DateUtils.formatDate()); } @Test public void testLoadBytedanceCampaign() { CtopOauthToken token = tokenService.getTokenByAccountId(1681669009084429L); bytedanceReportService.bytedanceMaterialReport(token, "2021-05-01", "2021-05-06"); } /* @Test public void etlBytedanceVideoReportInfo() throws Exception { etlBytedanceReportVideoDailyService.etlBytedanceVideo(); } */ @Test public void loadBytedanceAccountReport() { CtopOauthToken token1 = tokenService.getTokenByAccountId(1693648201498637L); reportService.getAdvertiserReport(token1, DateUtils.parseDate("2021-04-01", "yyyy-MM-dd"), DateUtils.parseDate("2021-04-12", "yyyy-MM-dd"), CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY); } @Test public void checkCostData() { String date = "2021-06-11"; for (int i = 0; i < 7; i++) { String getDate = DateUtils.addDay(date, i); //1:初始化 checkReportTaskInfoService.initTask(getDate); //2: checkCost List taskInfos = checkReportTaskInfoService.getListByParams(null, getDate, 0); if (null != taskInfos && !taskInfos.isEmpty()) { for (BytedanceCheckReportTaskInfo task : taskInfos) { checkReportTaskInfoService.checkCost(task); } log.info("{}:数据获取完成", getDate); } } } @Test public void refreshReportData() throws Exception { //1:查询异常任务数据 List taskInfos = checkReportTaskInfoService.getListByParams(null, null, 50001); if (null != taskInfos && !taskInfos.isEmpty()) { countDownLatch = new CountDownLatch(taskInfos.size()); taskInfos.forEach(taskInfo -> { try { checkReportTaskInfoService.reloadData(taskInfo); checkReportTaskInfoService.checkCost(taskInfo); } catch (Exception e) { } finally { countDownLatch.countDown(); } }); countDownLatch.await(); XxlJobHelper.log("{}数据更新完成"); } } /* @Test public void loadBkData(){ Date startDate = DateUtils.addDay(new Date(),-60); for(int i=0;i<=60;i++){ Date getDate = DateUtils.addDay(startDate,i); bytedanceVideoEtlInfoService.etlBytedanceVideoInfo(getDate); } }*/ @Autowired private IEtlKuaishouVideoInfoService etlKuaishouVideoInfo; @Test public void loadkuaishouVideoData(){ String startDate = "2021-05-07"; for(int i=0;i<=60;i++){ String getDate = DateUtils.formatDate(DateUtils.addDay(startDate,i)); etlKuaishouVideoInfo.etlKuaishouVideoInfo(getDate); } } @Autowired private IKuaiShouReportDailyMaterialService kuaiShouReportDailyMaterialService; @Test public void loadKuaishouMatData() throws InterruptedException { String startDate = "2021-06-18"; String endDate = "2021-06-24"; //1:查询当日数据 List tokens = tokenService.selectKuaiShouToken(); countDownLatch = new CountDownLatch(tokens.size()); tokens.forEach(token -> executorService.submit(() -> { try { kuaiShouReportDailyMaterialService.getMaterialReportByAccountIdAndStatDate(token.getAccountId(), token.getAccessToken(), startDate, endDate, "DAILY", 1); }catch (Exception e){ }finally { countDownLatch.countDown(); } })); countDownLatch.await(); System.out.println("数据获取完成"); } @Test public void loadKuaiShouMatHourlyData() throws InterruptedException{ // kuaiShouReportDailyMaterialService.getMaterialReportByAccountIdAndStatDate(7411077l, tokenService.getOauthTokenByAccountId("7411077").getAccessToken(), "2021-07-02", "2021-07-02", "HOURLY", 1); String startDate = "2021-07-19"; String endDate = "2021-07-19"; System.out.println(DateUtils.now()); List tokens = tokenService.selectKuaiShouToken(); System.out.println(tokens.size()); countDownLatch = new CountDownLatch(tokens.size()); tokens.forEach(token -> executorService.submit(() -> { try { kuaiShouReportDailyMaterialService.getMaterialReportByAccountIdAndStatDate(token.getAccountId(), token.getAccessToken(), startDate, endDate, "HOURLY", 1); }catch (Exception e){ }finally { countDownLatch.countDown(); } })); countDownLatch.await(); System.out.println("数据获取完成"); System.out.println(DateUtils.now()); } @Autowired IKuaishouInterfaceService2 kuaishouInterfaceService2; @Resource private KuaishouOperationRecordMapper kuaishouOperationRecordMapper; @Test public void dealOperationRecordData() throws InterruptedException { Date beginDate = DateUtils.parseDate("2020-04-28", "yyyy-MM-dd"); //chuangyi Date endDate = DateUtils.parseDate("2021-07-19", "yyyy-MM-dd"); // Date beginDate = DateUtils.parseDate("2020-04-21", "yyyy-MM-dd"); // Date endDate = DateUtils.parseDate("2021-07-17", "yyyy-MM-dd"); int recordsCount = 0; int newRecordCount = 0; while (beginDate.before(endDate)){ System.out.println("开始拆分数据,当前时间为:" + beginDate); List records = kuaishouOperationRecordMapper.selectCreativeRecordByStatDate(DateUtils.formatDate(beginDate, "yyyy-MM-dd")); //开始拆分数据,当前时间为:Tue Apr 28 00:00:00 CST 2020 // List records = kuaishouOperationRecordMapper.selectAccountRecordByStatDate(DateUtils.formatDate(beginDate, "yyyy-MM-dd")); // List records = kuaishouOperationRecordMapper.selectAppRecordByStatDate(DateUtils.formatDate(beginDate, "yyyy-MM-dd")); // List records = kuaishouOperationRecordMapper.selectCampaignRecordByStatDate(DateUtils.formatDate(beginDate, "yyyy-MM-dd")); //已执行完毕 // List records = kuaishouOperationRecordMapper.selectPackageRecordByStatDate(DateUtils.formatDate(beginDate, "yyyy-MM-dd")); // List records = kuaishouOperationRecordMapper.selectUnitRecordByStatDate(DateUtils.formatDate(beginDate, "yyyy-MM-dd")); //已执行完毕 // List records = kuaishouOperationRecordMapper.selectVideoRecordByStatDate(DateUtils.formatDate(beginDate, "yyyy-MM-dd")); if (!Check .isNull(records)) { recordsCount += records.size(); countDownLatch = new CountDownLatch(records.size()); List newRecords = new ArrayList<>(); records.forEach(record -> executorService.submit(() -> { try { String log = record.getLog(); JSONArray jsonArray = JSONArray.parseArray(log); for (int i = 0; i < jsonArray.size(); i++) { JSONObject object = jsonArray.getJSONObject(i); String originalData = object.getString("original_data"); String updateData = object.getString("update_data"); String fieldName = object.getString("field_name"); KuaishouOperationRecord newRecord = new KuaishouOperationRecord(record); newRecord.setOriginalData(originalData); newRecord.setUpdateData(updateData); newRecord.setFieldName(fieldName); if ("".equals(newRecord.getOriginalData()) && "".equals(newRecord.getUpdateData())) { continue; } newRecords.add(newRecord); } } catch (Exception e) { } finally { countDownLatch.countDown(); } })); countDownLatch.await(); newRecordCount += newRecords.size(); this.saveOperationRecordListToDb(newRecords); beginDate = DateUtils.addDay(beginDate, 1); } else { System.out.print(" -> 今日的数据为空!!!"); beginDate = DateUtils.addDay(beginDate, 1); } } System.out.println("---------------------------------------------------------------"); System.out.println("------------获取数据结束"+recordsCount+"条原始数据变成了"+newRecordCount+"条数据------------"); System.out.println("---------------------------------------------------------------"); } private void saveOperationRecordListToDb(List records) { if (Check.isNull(records)) { return; } List batchList = new ArrayList<>(); records.removeAll(Collections.singleton(null)); Integer operationTarget = records.get(0).getOperationTarget(); for (KuaishouOperationRecord sumPosition : records) { batchList.add(sumPosition); if (batchList.size() >= 1000) { if (operationTarget == 1) { kuaishouOperationRecordMapper.insertOperationRecordInfoAccountNew(batchList); } else if (operationTarget == 2) { kuaishouOperationRecordMapper.insertOperationRecordInfoCampaignNew(batchList); } else if (operationTarget == 3) { kuaishouOperationRecordMapper.insertOperationRecordInfoUnitNew(batchList); } else if (operationTarget == 4) { kuaishouOperationRecordMapper.insertOperationRecordInfoCreativeNew(batchList); } else if (operationTarget == 5) { kuaishouOperationRecordMapper.insertOperationRecordInfoVideoNew(batchList); } else if (operationTarget == 6) { kuaishouOperationRecordMapper.insertOperationRecordInfoAppNew(batchList); } else if (operationTarget == 7) { kuaishouOperationRecordMapper.insertOperationRecordInfoPackageNew(batchList); } else { System.out.println("未知的目标类型!"); } batchList.clear(); } } if (!batchList.isEmpty()) { if (operationTarget == 1) { kuaishouOperationRecordMapper.insertOperationRecordInfoAccountNew(batchList); } else if (operationTarget == 2) { kuaishouOperationRecordMapper.insertOperationRecordInfoCampaignNew(batchList); } else if (operationTarget == 3) { kuaishouOperationRecordMapper.insertOperationRecordInfoUnitNew(batchList); } else if (operationTarget == 4) { kuaishouOperationRecordMapper.insertOperationRecordInfoCreativeNew(batchList); } else if (operationTarget == 5) { kuaishouOperationRecordMapper.insertOperationRecordInfoVideoNew(batchList); } else if (operationTarget == 6) { kuaishouOperationRecordMapper.insertOperationRecordInfoAppNew(batchList); } else if (operationTarget == 7) { kuaishouOperationRecordMapper.insertOperationRecordInfoPackageNew(batchList); } else { System.out.println("未知的目标类型!"); } batchList.clear(); } } public void testfreshBytedanceUrl(){ etlReportBytedanceVideoService.etlBytedanceVideoClipIdJob(null); etlReportBytedanceVideoService.etlBytedanceVideoPlanInfoJob(); etlReportBytedanceVideoService.etlBytedanceVideoShotInfoJob(); } @Autowired IKuaishouInterfaceService2 kuaishouInterfaceServiceImpl2; @Test public void getUnitData() throws InterruptedException { List accountIdList = new ArrayList<>(); accountIdList.add(9556188l); accountIdList.add(9677284l); accountIdList.add(9677304l); accountIdList.add(10072135l); accountIdList.add(9901975l); accountIdList.add(9677344l); accountIdList.add(9901956l); accountIdList.add(9556431l); accountIdList.add(9767055l); accountIdList.add(10248827l); Date beginDate = DateUtils.parseDate("2021-06-04", "yyyy-MM-dd"); Date endDate = DateUtils.parseDate("2021-07-22", "yyyy-MM-dd"); List tokens = tokenService.selectKuaiShouToken(); while (beginDate.before(endDate)){ Date mulDate = beginDate; countDownLatch = new CountDownLatch(tokens.size()); tokens.forEach(token -> executorService.submit(() -> { try{ if(!accountIdList.contains(token.getAccountId())){ kuaishouInterfaceServiceImpl2.getOperationRecord( token.getAccountId(), token.getAccessToken(), 3, //先3组 后2计划 DateUtils.formatDate(mulDate,"yyyy-MM-dd"), DateUtils.formatDate(mulDate,"yyyy-MM-dd")); System.out.println("获取完毕 -> accountId:" + token.getAccountId() +",日期:" + DateUtils.formatDate(mulDate,"yyyy-MM-dd")); } }catch (Exception e){ }finally { countDownLatch.countDown(); } })); beginDate = DateUtils.addDay(beginDate, 1); countDownLatch.await(); } System.out.println("走完了~"); } @Autowired IKuaishouAudienceReportDailyService kuaishouAudienceReportDailyService; @Resource AiKuaishouUnitLevelOperationRecordMapper kuaishouUnitLevelOperationRecordMapper; @Test public void getAudienceReport() throws InterruptedException { int page = 1; int pageSize = 2000; Date beginDate = DateUtils.parseDate("2021-05-01", "yyyy-MM-dd"); Date endDate = DateUtils.parseDate("2021-06-01", "yyyy-MM-dd"); while (beginDate.before(endDate)){ Date mulDate = beginDate; List planIdsAndUnitIds = kuaishouUnitLevelOperationRecordMapper.selectAllByAccountId(mulDate, DateUtils.addDay(mulDate,1)); countDownLatch = new CountDownLatch(planIdsAndUnitIds.size()); planIdsAndUnitIds.forEach(planIdAndUnitId -> executorService.submit(() -> { try{ CtopOauthToken token = tokenService.getTokenByAccountId(planIdAndUnitId.getAccountId()); if(!Check.isNull(token)){ Date startDate = mulDate; Date enDate = DateUtils.addDay(startDate, 30); // province: 省级地域分布, city: 地级地域分布,gender: 性别分布,ageSegment: 年龄分布clientId: 系统分布 businessInterestTags: 商业兴趣分布 while (startDate.before(enDate)){ kuaishouAudienceReportDailyService.crowdAnalysisReport2(token, DateUtils.formatDate(startDate,"yyyy-MM-dd"), "ageSegment", planIdAndUnitId.getCampaignId(), planIdAndUnitId.getUnitId(), page, pageSize); System.out.println("获取完毕 -> unit:" + planIdAndUnitId.getUnitId() +",日期:" + DateUtils.formatDate(startDate,"yyyy-MM-dd")); startDate = DateUtils.addDay(startDate, 1); } }else { System.out.println("token异常 -》 accountId:"+planIdAndUnitId.getAccountId()); } }catch (Exception e){ }finally { countDownLatch.countDown(); } })); beginDate = DateUtils.addDay(beginDate, 1); countDownLatch.await(); } System.out.println("走完了~"); } }