|
@@ -4,8 +4,13 @@ import cn.com.ctop.common.module.entity.BindAccountLogin;
|
|
|
import cn.com.ctop.common.module.entity.CtopOauthToken;
|
|
|
import cn.com.ctop.common.module.service.IBindAccountLoginService;
|
|
|
import cn.com.ctop.common.module.service.ICtopOauthTokenService;
|
|
|
+import cn.com.ctop.common.module.utils.Check;
|
|
|
import cn.com.ctop.common.module.utils.CtopAdConstant;
|
|
|
+import cn.com.ctop.kuaishou.modules.batch.entity.KuaishouOperationRecord;
|
|
|
+import cn.com.ctop.kuaishou.modules.batch.mapper.KuaishouOperationRecordMapper;
|
|
|
import cn.com.ctop.kuaishou.modules.batch.service.IKuaiShouReportDailyMaterialService;
|
|
|
+import cn.com.ctop.kuaishou.modules.batch.service.IKuaishouInterfaceService2;
|
|
|
+import cn.com.ctop.kuaishou.modules.batch.service.impl.KuaishouInterfaceServiceImpl2;
|
|
|
import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouWebInterfaceService;
|
|
|
import cn.com.ctop.kuaishou.modules.material.service.IEtlKuaishouVideoInfoService;
|
|
|
import cn.com.ctop.kuaishou.modules.report.entity.EtlKuaishouAccountMaterialReportDaily;
|
|
@@ -16,6 +21,8 @@ import cn.com.ctop.toutiao.modules.link.service.IETLReportBytedanceVideoService;
|
|
|
import cn.com.ctop.toutiao.modules.material.service.IByteDanceAdvertiserDataService;
|
|
|
import cn.com.ctop.toutiao.modules.report.entity.BytedanceCheckReportTaskInfo;
|
|
|
import cn.com.ctop.toutiao.modules.report.service.*;
|
|
|
+import com.alibaba.fastjson.JSONArray;
|
|
|
+import com.alibaba.fastjson.JSONObject;
|
|
|
import com.xxl.job.core.context.XxlJobHelper;
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
import org.jeecg.common.util.DateUtils;
|
|
@@ -28,9 +35,7 @@ import org.springframework.test.context.junit4.SpringRunner;
|
|
|
|
|
|
import javax.annotation.Resource;
|
|
|
import java.text.ParseException;
|
|
|
-import java.util.Date;
|
|
|
-import java.util.HashMap;
|
|
|
-import java.util.List;
|
|
|
+import java.util.*;
|
|
|
import java.util.concurrent.CountDownLatch;
|
|
|
import java.util.concurrent.ExecutorService;
|
|
|
import java.util.concurrent.Executors;
|
|
@@ -79,6 +84,8 @@ public class SampleTest {
|
|
|
@Autowired
|
|
|
private IETLReportBytedanceVideoService etlReportBytedanceVideoService;
|
|
|
|
|
|
+
|
|
|
+
|
|
|
@Test
|
|
|
public void loadKuaishouCookie() throws ParseException {
|
|
|
CtopOauthToken tokenByAccountId = tokenService.getTokenByAccountId(1700996735138951L);
|
|
@@ -271,5 +278,189 @@ public class SampleTest {
|
|
|
countDownLatch.await();
|
|
|
System.out.println("数据获取完成");
|
|
|
}
|
|
|
+ @Test
|
|
|
+ public void loadKuaiShouMatHourlyData() throws InterruptedException{
|
|
|
+// kuaiShouReportDailyMaterialService.getMaterialReportByAccountIdAndStatDate(7411077l, tokenService.getOauthTokenByAccountId("7411077").getAccessToken(), "2021-07-02", "2021-07-02", "HOURLY", 1);
|
|
|
+
|
|
|
+ String startDate = "2021-07-19";
|
|
|
+ String endDate = "2021-07-19";
|
|
|
+ System.out.println(DateUtils.now());
|
|
|
+ List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
|
|
|
+ System.out.println(tokens.size());
|
|
|
+ countDownLatch = new CountDownLatch(tokens.size());
|
|
|
+ tokens.forEach(token -> executorService.submit(() -> {
|
|
|
+ try {
|
|
|
+ kuaiShouReportDailyMaterialService.getMaterialReportByAccountIdAndStatDate(token.getAccountId(), token.getAccessToken(), startDate, endDate, "HOURLY", 1);
|
|
|
+ }catch (Exception e){
|
|
|
+ }finally {
|
|
|
+ countDownLatch.countDown();
|
|
|
+ }
|
|
|
+ }));
|
|
|
+ countDownLatch.await();
|
|
|
+ System.out.println("数据获取完成");
|
|
|
+ System.out.println(DateUtils.now());
|
|
|
+ }
|
|
|
+
|
|
|
+ @Autowired
|
|
|
+ IKuaishouInterfaceService2 kuaishouInterfaceService2;
|
|
|
+ @Resource
|
|
|
+ private KuaishouOperationRecordMapper kuaishouOperationRecordMapper;
|
|
|
+
|
|
|
+
|
|
|
+ @Test
|
|
|
+ public void dealOperationRecordData() throws InterruptedException {
|
|
|
+ Date beginDate = DateUtils.parseDate("2021-06-01", "yyyy-MM-dd");
|
|
|
+ Date endDate = DateUtils.parseDate("2021-07-19", "yyyy-MM-dd");
|
|
|
+// Date beginDate = DateUtils.parseDate("2020-04-21", "yyyy-MM-dd");
|
|
|
+// Date endDate = DateUtils.parseDate("2021-07-17", "yyyy-MM-dd");
|
|
|
+
|
|
|
+ int recordsCount = 0;
|
|
|
+ int newRecordCount = 0;
|
|
|
+
|
|
|
+ while (beginDate.before(endDate)){
|
|
|
+ System.out.println("开始拆分数据,当前时间为:" + beginDate);
|
|
|
+// List<KuaishouOperationRecord> records = kuaishouOperationRecordMapper.selectCreativeRecordByStatDate(DateUtils.formatDate(beginDate, "yyyy-MM-dd")); //开始拆分数据,当前时间为:Tue Apr 28 00:00:00 CST 2020
|
|
|
+// List<KuaishouOperationRecord> records = kuaishouOperationRecordMapper.selectAccountRecordByStatDate(DateUtils.formatDate(beginDate, "yyyy-MM-dd"));
|
|
|
+// List<KuaishouOperationRecord> records = kuaishouOperationRecordMapper.selectAppRecordByStatDate(DateUtils.formatDate(beginDate, "yyyy-MM-dd"));
|
|
|
+// List<KuaishouOperationRecord> records = kuaishouOperationRecordMapper.selectCampaignRecordByStatDate(DateUtils.formatDate(beginDate, "yyyy-MM-dd")); //已执行完毕
|
|
|
+// List<KuaishouOperationRecord> records = kuaishouOperationRecordMapper.selectPackageRecordByStatDate(DateUtils.formatDate(beginDate, "yyyy-MM-dd"));
|
|
|
+ List<KuaishouOperationRecord> records = kuaishouOperationRecordMapper.selectUnitRecordByStatDate(DateUtils.formatDate(beginDate, "yyyy-MM-dd")); //已执行完毕
|
|
|
+// List<KuaishouOperationRecord> records = kuaishouOperationRecordMapper.selectVideoRecordByStatDate(DateUtils.formatDate(beginDate, "yyyy-MM-dd"));
|
|
|
+ if(Check.isNull(records)){
|
|
|
+ System.out.print(" -> 今日的数据为空!!!");
|
|
|
+ beginDate = DateUtils.addDay(beginDate, 1);
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ recordsCount += records.size();
|
|
|
+ countDownLatch = new CountDownLatch(records.size());
|
|
|
+
|
|
|
+ List<KuaishouOperationRecord> newRecords = new ArrayList<>();
|
|
|
+
|
|
|
+ records.forEach(record -> executorService.submit(() -> {
|
|
|
+ try{
|
|
|
+ String log = record.getLog();
|
|
|
+ JSONArray jsonArray = JSONArray.parseArray(log);
|
|
|
+ for(int i=0;i<jsonArray.size();i++){
|
|
|
+ JSONObject object = jsonArray.getJSONObject(i);
|
|
|
+ String originalData = object.getString("original_data");
|
|
|
+ String updateData = object.getString("update_data");
|
|
|
+ String fieldName = object.getString("field_name");
|
|
|
+ KuaishouOperationRecord newRecord = new KuaishouOperationRecord(record);
|
|
|
+ newRecord.setOriginalData(originalData);
|
|
|
+ newRecord.setUpdateData(updateData);
|
|
|
+ newRecord.setFieldName(fieldName);
|
|
|
+
|
|
|
+ if("".equals(newRecord.getOriginalData()) && "".equals(newRecord.getUpdateData())){
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ newRecords.add(newRecord);
|
|
|
+ }
|
|
|
+ }catch (Exception e){
|
|
|
+ }finally {
|
|
|
+
|
|
|
+ countDownLatch.countDown();
|
|
|
+ }}));
|
|
|
+ countDownLatch.await();
|
|
|
+ newRecordCount += newRecords.size();
|
|
|
+ this.saveOperationRecordListToDb(newRecords);
|
|
|
+ beginDate = DateUtils.addDay(beginDate, 1);
|
|
|
+ }
|
|
|
+ System.out.println("---------------------------------------------------------------");
|
|
|
+ System.out.println("------------获取数据结束"+recordsCount+"条原始数据变成了"+newRecordCount+"条数据------------");
|
|
|
+ System.out.println("---------------------------------------------------------------");
|
|
|
+ }
|
|
|
+
|
|
|
+ private void saveOperationRecordListToDb(List<KuaishouOperationRecord> records) {
|
|
|
+ if (Check.isNull(records)) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ List<KuaishouOperationRecord> batchList = new ArrayList<>();
|
|
|
+ records.removeAll(Collections.singleton(null));
|
|
|
+ Integer operationTarget = records.get(0).getOperationTarget();
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+ for (KuaishouOperationRecord sumPosition : records) {
|
|
|
+ batchList.add(sumPosition);
|
|
|
+
|
|
|
+ if (batchList.size() >= 1000) {
|
|
|
+ if (operationTarget == 1) {
|
|
|
+ kuaishouOperationRecordMapper.insertOperationRecordInfoAccountNew(batchList);
|
|
|
+ } else if (operationTarget == 2) {
|
|
|
+ kuaishouOperationRecordMapper.insertOperationRecordInfoCampaignNew(batchList);
|
|
|
+ } else if (operationTarget == 3) {
|
|
|
+ kuaishouOperationRecordMapper.insertOperationRecordInfoUnitNew(batchList);
|
|
|
+ } else if (operationTarget == 4) {
|
|
|
+ kuaishouOperationRecordMapper.insertOperationRecordInfoCreativeNew(batchList);
|
|
|
+ } else if (operationTarget == 5) {
|
|
|
+ kuaishouOperationRecordMapper.insertOperationRecordInfoVideoNew(batchList);
|
|
|
+ } else if (operationTarget == 6) {
|
|
|
+ kuaishouOperationRecordMapper.insertOperationRecordInfoAppNew(batchList);
|
|
|
+ } else if (operationTarget == 7) {
|
|
|
+ kuaishouOperationRecordMapper.insertOperationRecordInfoPackageNew(batchList);
|
|
|
+ } else {
|
|
|
+ System.out.println("未知的目标类型!");
|
|
|
+ }
|
|
|
+ batchList.clear();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if (!batchList.isEmpty()) {
|
|
|
+
|
|
|
+ if (operationTarget == 1) {
|
|
|
+ kuaishouOperationRecordMapper.insertOperationRecordInfoAccountNew(batchList);
|
|
|
+ } else if (operationTarget == 2) {
|
|
|
+ kuaishouOperationRecordMapper.insertOperationRecordInfoCampaignNew(batchList);
|
|
|
+ } else if (operationTarget == 3) {
|
|
|
+ kuaishouOperationRecordMapper.insertOperationRecordInfoUnitNew(batchList);
|
|
|
+ } else if (operationTarget == 4) {
|
|
|
+ kuaishouOperationRecordMapper.insertOperationRecordInfoCreativeNew(batchList);
|
|
|
+ } else if (operationTarget == 5) {
|
|
|
+ kuaishouOperationRecordMapper.insertOperationRecordInfoVideoNew(batchList);
|
|
|
+ } else if (operationTarget == 6) {
|
|
|
+ kuaishouOperationRecordMapper.insertOperationRecordInfoAppNew(batchList);
|
|
|
+ } else if (operationTarget == 7) {
|
|
|
+ kuaishouOperationRecordMapper.insertOperationRecordInfoPackageNew(batchList);
|
|
|
+ } else {
|
|
|
+ System.out.println("未知的目标类型!");
|
|
|
+ }
|
|
|
+ batchList.clear();
|
|
|
+ }
|
|
|
+
|
|
|
+ }
|
|
|
+ @Autowired
|
|
|
+ KuaishouInterfaceServiceImpl2 kuaishouInterfaceServiceImpl2;
|
|
|
+
|
|
|
+ @Test
|
|
|
+ public void getUnitData(){
|
|
|
+ List<Long> accountIdList = new ArrayList<>();
|
|
|
+ accountIdList.add(9556188l);
|
|
|
+ accountIdList.add(9677284l);
|
|
|
+ accountIdList.add(9677304l);
|
|
|
+ accountIdList.add(10072135l);
|
|
|
+ accountIdList.add(9901975l);
|
|
|
+ accountIdList.add(9677344l);
|
|
|
+ accountIdList.add(9901956l);
|
|
|
+ accountIdList.add(9556431l);
|
|
|
+ accountIdList.add(9767055l);
|
|
|
+ accountIdList.add(10248827l);
|
|
|
+
|
|
|
+ Date beginDate = DateUtils.parseDate("2021-06-01", "yyyy-MM-dd");
|
|
|
+ Date endDate = DateUtils.parseDate("2021-07-19", "yyyy-MM-dd");
|
|
|
+
|
|
|
+ while (beginDate.before(endDate)){
|
|
|
+ for(Long accountId : accountIdList){
|
|
|
+ kuaishouInterfaceServiceImpl2.getOperationRecord( accountId,
|
|
|
+ tokenService.getOauthTokenByAccountId(String.valueOf(accountId)).getAccessToken(),
|
|
|
+ 3,
|
|
|
+ DateUtils.formatDate(beginDate,"yyyy-MM-dd"),
|
|
|
+ DateUtils.formatDate(beginDate,"yyyy-MM-dd"));
|
|
|
+ System.out.println("获取完毕 -> accountId:" + accountId +",日期:" + DateUtils.formatDate(beginDate,"yyyy-MM-dd"));
|
|
|
+
|
|
|
+ }
|
|
|
+ beginDate = DateUtils.addDay(beginDate, 1);
|
|
|
+ }
|
|
|
+ System.out.println("走完了~");
|
|
|
+ }
|
|
|
|
|
|
}
|