SampleTest.java 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410
  1. package org.jeecg;
  2. import cn.com.ctop.common.module.entity.BindAccountLogin;
  3. import cn.com.ctop.common.module.entity.CtopOauthToken;
  4. import cn.com.ctop.common.module.service.IBindAccountLoginService;
  5. import cn.com.ctop.common.module.service.ICtopOauthTokenService;
  6. import cn.com.ctop.common.module.utils.Check;
  7. import cn.com.ctop.common.module.utils.CtopAdConstant;
  8. import cn.com.ctop.kuaishou.modules.batch.entity.KuaishouOperationRecord;
  9. import cn.com.ctop.kuaishou.modules.batch.mapper.KuaishouOperationRecordMapper;
  10. import cn.com.ctop.kuaishou.modules.batch.service.IKuaishouInterfaceService;
  11. import cn.com.ctop.kuaishou.modules.batch.service.impl.KuaishouInterfaceServiceImpl2;
  12. import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouWebInterfaceService;
  13. import cn.com.ctop.kuaishou.modules.report.mapper.EtlKuaishouAccountMaterialReportDailyMapper;
  14. import cn.com.ctop.kuaishou.modules.report.service.IKuaishouReportDailyImageService;
  15. import cn.com.ctop.oa.modules.service.IWechatNoListService;
  16. import cn.com.ctop.toutiao.modules.link.service.IETLReportBytedanceVideoService;
  17. import cn.com.ctop.toutiao.modules.material.service.IByteDanceAdvertiserDataService;
  18. import cn.com.ctop.toutiao.modules.report.entity.BytedanceCheckReportTaskInfo;
  19. import cn.com.ctop.toutiao.modules.report.service.*;
  20. import com.alibaba.fastjson.JSONArray;
  21. import com.alibaba.fastjson.JSONObject;
  22. import com.xxl.job.core.context.XxlJobHelper;
  23. import lombok.extern.slf4j.Slf4j;
  24. import org.jeecg.common.util.DateUtils;
  25. import org.junit.Test;
  26. import org.junit.runner.RunWith;
  27. import org.springframework.beans.factory.annotation.Autowired;
  28. import org.springframework.boot.test.context.SpringBootTest;
  29. import org.springframework.test.context.ActiveProfiles;
  30. import org.springframework.test.context.junit4.SpringRunner;
  31. import javax.annotation.Resource;
  32. import java.text.ParseException;
  33. import java.text.SimpleDateFormat;
  34. import java.util.*;
  35. import java.util.concurrent.CountDownLatch;
  36. import java.util.concurrent.ExecutorService;
  37. import java.util.concurrent.Executors;
  38. @RunWith(SpringRunner.class)
  39. @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
  40. @ActiveProfiles("wps")
  41. @Slf4j
  42. public class SampleTest {
  43. @Autowired
  44. private IBytedanceVideoEtlInfoService bytedanceVideoEtlInfoService;
  45. @Autowired
  46. IBytedanceVideoReportService bytedanceVideoReportService;
  47. @Autowired
  48. IETLReportBytedanceVideoService ietlReportBytedanceVideoService;
  49. @Autowired
  50. IKuaishouReportDailyImageService kuaishouReportDailyImageService;
  51. @Autowired
  52. IRuleByteDanceAccountService ruleByteDanceAccountService;
  53. @Autowired
  54. private ICtopOauthTokenService tokenService;
  55. @Autowired
  56. private IBindAccountLoginService bindAccountLoginService;
  57. @Autowired
  58. private IKuaishouWebInterfaceService kuaishouWebInterfaceService;
  59. @Autowired
  60. private IByteDanceAdvertiserDataService advertiserDataService;
  61. @Autowired
  62. private IWechatNoListService wechatNoListService;
  63. @Autowired
  64. private IETLReportBytedanceVideoService bytedanceVideoService;
  65. @Autowired
  66. private IByteDanceVideoReportDailyService videoReportDailyService;
  67. @Resource
  68. private EtlKuaishouAccountMaterialReportDailyMapper etlKuaishouAccountMaterialReportDailyMapper;
  69. @Autowired
  70. private IReportService reportService;
  71. @Resource
  72. private IBytedanceCheckReportTaskInfoService checkReportTaskInfoService;
  73. @Autowired
  74. private IEtlBytedanceReportVideoDailyService etlBytedanceReportVideoDailyService;
  75. static ExecutorService executorService = null;
  76. static CountDownLatch countDownLatch = null;
  77. @Autowired
  78. private IBytedanceReportService bytedanceReportService;
  79. @Autowired
  80. private IETLReportBytedanceVideoService etlReportBytedanceVideoService;
  81. @Autowired
  82. private IKuaishouInterfaceService iKuaishouInterfaceService;
  83. @Test
  84. public void loadKuaishouCookie() throws ParseException {
  85. CtopOauthToken token = tokenService.getTokenByAccountId(10291211L);
  86. SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
  87. Date parse = simpleDateFormat.parse("2021-07-11");
  88. Date pars1 = simpleDateFormat.parse("2021-07-12");
  89. iKuaishouInterfaceService.getGroupList(token, parse, pars1);
  90. }
  91. @Test
  92. public void deleteKuaishouComment() {
  93. Long start = System.currentTimeMillis();
  94. List<BindAccountLogin> loginList = bindAccountLoginService.getListByParams(CtopAdConstant.PLATFORM_TYPE_KUAISHOU_PY, 1);
  95. if (loginList != null && !loginList.isEmpty()) {
  96. executorService = Executors.newFixedThreadPool(5);
  97. countDownLatch = new CountDownLatch(loginList.size());
  98. loginList.forEach(login -> executorService.submit(() -> {
  99. try {
  100. kuaishouWebInterfaceService.deleteAllComment(new HashMap<>(), login);
  101. } catch (Exception e) {
  102. log.error(e.getMessage(), e);
  103. } finally {
  104. countDownLatch.countDown();
  105. }
  106. }));
  107. try {
  108. countDownLatch.await();
  109. } catch (InterruptedException e) {
  110. e.printStackTrace();
  111. }
  112. Long end = System.currentTimeMillis();
  113. log.info("快手删评论所用时长:{}毫秒", end - start);
  114. }
  115. }
  116. @Test
  117. public void loadBytedanceVideoReportData() {
  118. CtopOauthToken token = tokenService.getTokenByAccountId(1696831659659278L);
  119. advertiserDataService.getMaterialList(token);
  120. }
  121. @Test
  122. public void etlBytedanceVideoInfo() {
  123. String startDate = "2021-02-01";
  124. for (int i = 0; i < 100; i++) {
  125. bytedanceVideoService.cleanData(DateUtils.addDay(startDate, i));
  126. }
  127. }
  128. @Test
  129. public void getData() {
  130. //汇创 人员
  131. // wechatUserInfoService.getUserList();
  132. // //考勤记录
  133. // wechatCheckinDataService.getCheckinData("2021-03-01 00:00:00", "2021-03-31 23:59:59");
  134. //异常记录
  135. wechatNoListService.getNoDateByNo("2021-03-01 00:00:00", "2021-03-31 23:59:59", "0");
  136. //优容
  137. // wechatUserInfoService.getYRUserList();
  138. // wechatCheckinDataService.getYRCheckinData("2021-03-01 00:00:00", "2021-03-31 23:59:59");
  139. wechatNoListService.getNoDateByNo("2021-03-01 00:00:00", "2021-03-31 23:59:59", "1");
  140. }
  141. @Test
  142. public void formatVideoReportData() {
  143. for (int i = 1; i < 45; i++) {
  144. Date getDate = DateUtils.addDay(new Date(), -i);
  145. String date = DateUtils.formatDate(getDate);
  146. videoReportDailyService.videoInfoList(date, date);
  147. }
  148. }
  149. @Test
  150. public void loadBDAccoutData() {
  151. Date startDate = DateUtils.addDay(new Date(), -2000);
  152. for (int i = 0; i <= 2000; i++) {
  153. Date getDate = DateUtils.addDay(startDate, i);
  154. bytedanceVideoEtlInfoService.etlBytedanceVideoInfo(getDate);
  155. }
  156. }
  157. @Test
  158. public void etlKuaishouVideoInfo() {
  159. etlKuaishouAccountMaterialReportDailyMapper.etlKuaishouAccountMaterialReportDailyData(DateUtils.formatDate());
  160. }
  161. @Test
  162. public void testLoadBytedanceCampaign() {
  163. CtopOauthToken token = tokenService.getTokenByAccountId(1681669009084429L);
  164. bytedanceReportService.bytedanceMaterialReport(token, "2021-05-01", "2021-05-06");
  165. }
  166. @Test
  167. public void etlBytedanceVideoReportInfo() throws Exception {
  168. etlBytedanceReportVideoDailyService.etlBytedanceVideo();
  169. }
  170. @Test
  171. public void loadBytedanceAccountReport() {
  172. CtopOauthToken token1 = tokenService.getTokenByAccountId(1693648201498637L);
  173. reportService.getAdvertiserReport(token1, DateUtils.parseDate("2021-04-01", "yyyy-MM-dd"), DateUtils.parseDate("2021-04-12", "yyyy-MM-dd"), CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
  174. }
  175. @Test
  176. public void checkCostData() {
  177. String date = "2021-06-11";
  178. for (int i = 0; i < 7; i++) {
  179. String getDate = DateUtils.addDay(date, i);
  180. //1:初始化
  181. checkReportTaskInfoService.initTask(getDate);
  182. //2: checkCost
  183. List<BytedanceCheckReportTaskInfo> taskInfos = checkReportTaskInfoService.getListByParams(null, getDate, 0);
  184. if (null != taskInfos && !taskInfos.isEmpty()) {
  185. for (BytedanceCheckReportTaskInfo task : taskInfos) {
  186. checkReportTaskInfoService.checkCost(task);
  187. }
  188. log.info("{}:数据获取完成", getDate);
  189. }
  190. }
  191. }
  192. @Test
  193. public void refreshReportData() throws Exception {
  194. //1:查询异常任务数据
  195. List<BytedanceCheckReportTaskInfo> taskInfos = checkReportTaskInfoService.getListByParams(null, null, 50001);
  196. if (null != taskInfos && !taskInfos.isEmpty()) {
  197. countDownLatch = new CountDownLatch(taskInfos.size());
  198. taskInfos.forEach(taskInfo -> {
  199. try {
  200. checkReportTaskInfoService.reloadData(taskInfo);
  201. checkReportTaskInfoService.checkCost(taskInfo);
  202. } catch (Exception e) {
  203. } finally {
  204. countDownLatch.countDown();
  205. }
  206. });
  207. countDownLatch.await();
  208. XxlJobHelper.log("{}数据更新完成");
  209. }
  210. }
  211. @Test
  212. public void testfreshBytedanceUrl() {
  213. etlReportBytedanceVideoService.etlBytedanceVideoClipIdJob();
  214. etlReportBytedanceVideoService.etlBytedanceVideoPlanInfoJob();
  215. etlReportBytedanceVideoService.etlBytedanceVideoShotInfoJob();
  216. }
  217. @Test
  218. public void loadEffecfData() {
  219. String startDate = "2021-05-01";
  220. for (int i = 0; i <= 70; i++) {
  221. String getDate = DateUtils.addDay(startDate, i);
  222. bytedanceVideoEtlInfoService.etlBytedanceVideoInfo(DateUtils.parseDate(getDate, "yyyy-MM-dd"));
  223. }
  224. }
  225. @Resource
  226. private KuaishouOperationRecordMapper kuaishouOperationRecordMapper;
  227. @Test
  228. public void dealOperationRecordData() throws InterruptedException {
  229. Date beginDate = DateUtils.parseDate("2021-06-01", "yyyy-MM-dd");
  230. Date endDate = DateUtils.parseDate("2021-07-19", "yyyy-MM-dd");
  231. // Date beginDate = DateUtils.parseDate("2020-04-21", "yyyy-MM-dd");
  232. // Date endDate = DateUtils.parseDate("2021-07-17", "yyyy-MM-dd");
  233. int recordsCount = 0;
  234. int newRecordCount = 0;
  235. while (beginDate.before(endDate)){
  236. System.out.println("开始拆分数据,当前时间为:" + beginDate);
  237. // List<KuaishouOperationRecord> records = kuaishouOperationRecordMapper.selectCreativeRecordByStatDate(DateUtils.formatDate(beginDate, "yyyy-MM-dd")); //开始拆分数据,当前时间为:Tue Apr 28 00:00:00 CST 2020
  238. // List<KuaishouOperationRecord> records = kuaishouOperationRecordMapper.selectAccountRecordByStatDate(DateUtils.formatDate(beginDate, "yyyy-MM-dd"));
  239. // List<KuaishouOperationRecord> records = kuaishouOperationRecordMapper.selectAppRecordByStatDate(DateUtils.formatDate(beginDate, "yyyy-MM-dd"));
  240. // List<KuaishouOperationRecord> records = kuaishouOperationRecordMapper.selectCampaignRecordByStatDate(DateUtils.formatDate(beginDate, "yyyy-MM-dd")); //已执行完毕
  241. // List<KuaishouOperationRecord> records = kuaishouOperationRecordMapper.selectPackageRecordByStatDate(DateUtils.formatDate(beginDate, "yyyy-MM-dd"));
  242. List<KuaishouOperationRecord> records = kuaishouOperationRecordMapper.selectUnitRecordByStatDate(DateUtils.formatDate(beginDate, "yyyy-MM-dd")); //已执行完毕
  243. // List<KuaishouOperationRecord> records = kuaishouOperationRecordMapper.selectVideoRecordByStatDate(DateUtils.formatDate(beginDate, "yyyy-MM-dd"));
  244. if(Check.isNull(records)){
  245. System.out.print(" -> 今日的数据为空!!!");
  246. beginDate = DateUtils.addDay(beginDate, 1);
  247. continue;
  248. }
  249. recordsCount += records.size();
  250. countDownLatch = new CountDownLatch(records.size());
  251. List<KuaishouOperationRecord> newRecords = new ArrayList<>();
  252. records.forEach(record -> executorService.submit(() -> {
  253. try{
  254. String log = record.getLog();
  255. JSONArray jsonArray = JSONArray.parseArray(log);
  256. for(int i=0;i<jsonArray.size();i++){
  257. JSONObject object = jsonArray.getJSONObject(i);
  258. String originalData = object.getString("original_data");
  259. String updateData = object.getString("update_data");
  260. String fieldName = object.getString("field_name");
  261. KuaishouOperationRecord newRecord = new KuaishouOperationRecord(record);
  262. newRecord.setOriginalData(originalData);
  263. newRecord.setUpdateData(updateData);
  264. newRecord.setFieldName(fieldName);
  265. if("".equals(newRecord.getOriginalData()) && "".equals(newRecord.getUpdateData())){
  266. continue;
  267. }
  268. newRecords.add(newRecord);
  269. }
  270. }catch (Exception e){
  271. }finally {
  272. countDownLatch.countDown();
  273. }}));
  274. countDownLatch.await();
  275. newRecordCount += newRecords.size();
  276. this.saveOperationRecordListToDb(newRecords);
  277. beginDate = DateUtils.addDay(beginDate, 1);
  278. }
  279. System.out.println("---------------------------------------------------------------");
  280. System.out.println("------------获取数据结束"+recordsCount+"条原始数据变成了"+newRecordCount+"条数据------------");
  281. System.out.println("---------------------------------------------------------------");
  282. }
  283. private void saveOperationRecordListToDb(List<KuaishouOperationRecord> records) {
  284. if (Check.isNull(records)) {
  285. return;
  286. }
  287. List<KuaishouOperationRecord> batchList = new ArrayList<>();
  288. records.removeAll(Collections.singleton(null));
  289. Integer operationTarget = records.get(0).getOperationTarget();
  290. for (KuaishouOperationRecord sumPosition : records) {
  291. batchList.add(sumPosition);
  292. if (batchList.size() >= 1000) {
  293. if (operationTarget == 1) {
  294. kuaishouOperationRecordMapper.insertOperationRecordInfoAccountNew(batchList);
  295. } else if (operationTarget == 2) {
  296. kuaishouOperationRecordMapper.insertOperationRecordInfoCampaignNew(batchList);
  297. } else if (operationTarget == 3) {
  298. kuaishouOperationRecordMapper.insertOperationRecordInfoUnitNew(batchList);
  299. } else if (operationTarget == 4) {
  300. kuaishouOperationRecordMapper.insertOperationRecordInfoCreativeNew(batchList);
  301. } else if (operationTarget == 5) {
  302. kuaishouOperationRecordMapper.insertOperationRecordInfoVideoNew(batchList);
  303. } else if (operationTarget == 6) {
  304. kuaishouOperationRecordMapper.insertOperationRecordInfoAppNew(batchList);
  305. } else if (operationTarget == 7) {
  306. kuaishouOperationRecordMapper.insertOperationRecordInfoPackageNew(batchList);
  307. } else {
  308. System.out.println("未知的目标类型!");
  309. }
  310. batchList.clear();
  311. }
  312. }
  313. if (!batchList.isEmpty()) {
  314. if (operationTarget == 1) {
  315. kuaishouOperationRecordMapper.insertOperationRecordInfoAccountNew(batchList);
  316. } else if (operationTarget == 2) {
  317. kuaishouOperationRecordMapper.insertOperationRecordInfoCampaignNew(batchList);
  318. } else if (operationTarget == 3) {
  319. kuaishouOperationRecordMapper.insertOperationRecordInfoUnitNew(batchList);
  320. } else if (operationTarget == 4) {
  321. kuaishouOperationRecordMapper.insertOperationRecordInfoCreativeNew(batchList);
  322. } else if (operationTarget == 5) {
  323. kuaishouOperationRecordMapper.insertOperationRecordInfoVideoNew(batchList);
  324. } else if (operationTarget == 6) {
  325. kuaishouOperationRecordMapper.insertOperationRecordInfoAppNew(batchList);
  326. } else if (operationTarget == 7) {
  327. kuaishouOperationRecordMapper.insertOperationRecordInfoPackageNew(batchList);
  328. } else {
  329. System.out.println("未知的目标类型!");
  330. }
  331. batchList.clear();
  332. }
  333. }
  334. @Autowired
  335. KuaishouInterfaceServiceImpl2 kuaishouInterfaceServiceImpl2;
  336. @Test
  337. public void getUnitData(){
  338. List<Long> accountIdList = new ArrayList<>();
  339. accountIdList.add(9556188l);
  340. accountIdList.add(9677284l);
  341. accountIdList.add(9677304l);
  342. accountIdList.add(10072135l);
  343. accountIdList.add(9901975l);
  344. accountIdList.add(9677344l);
  345. accountIdList.add(9901956l);
  346. accountIdList.add(9556431l);
  347. accountIdList.add(9767055l);
  348. accountIdList.add(10248827l);
  349. Date beginDate = DateUtils.parseDate("2021-06-01", "yyyy-MM-dd");
  350. Date endDate = DateUtils.parseDate("2021-07-19", "yyyy-MM-dd");
  351. while (beginDate.before(endDate)){
  352. for(Long accountId : accountIdList){
  353. kuaishouInterfaceServiceImpl2.getOperationRecord( accountId,
  354. tokenService.getOauthTokenByAccountId(String.valueOf(accountId)).getAccessToken(),
  355. 3,
  356. DateUtils.formatDate(beginDate,"yyyy-MM-dd"),
  357. DateUtils.formatDate(beginDate,"yyyy-MM-dd"));
  358. System.out.println("获取完毕 -> accountId:" + accountId +",日期:" + DateUtils.formatDate(beginDate,"yyyy-MM-dd"));
  359. }
  360. beginDate = DateUtils.addDay(beginDate, 1);
  361. }
  362. System.out.println("走完了~");
  363. }
  364. }