Sfoglia il codice sorgente

fix of '头条视频报表重构' on 2021.02.02

jiequan.bi 4 anni fa
parent
commit
d668e6eeb7

+ 66 - 39
jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java

@@ -3,12 +3,15 @@ package org.jeecg;
 import cn.com.ctop.common.module.entity.BindAccountLogin;
 import cn.com.ctop.common.module.entity.CtopOauthToken;
 import cn.com.ctop.common.module.entity.UserAllocation;
-import cn.com.ctop.common.module.mapper.CtopOauthTokenMapper;
-import cn.com.ctop.common.module.service.*;
+import cn.com.ctop.common.module.service.IBindAccountLoginService;
+import cn.com.ctop.common.module.service.ICtopOauthTokenService;
+import cn.com.ctop.common.module.service.IUserAllocationService;
+import cn.com.ctop.common.module.utils.Check;
 import cn.com.ctop.common.module.utils.CtopAdConstant;
 import cn.com.ctop.kuaishou.modules.ai.entity.AiKuaishouAccountTargetTemplate;
 import cn.com.ctop.kuaishou.modules.ai.service.IAiKuaishouAccountTargetTemplateService;
 import cn.com.ctop.kuaishou.modules.ai.service.IAiKuaishouNewCreateCampaign;
+import cn.com.ctop.kuaishou.modules.batch.entity.KuaiShouVideoGet;
 import cn.com.ctop.kuaishou.modules.batch.service.*;
 import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouWebInterfaceService;
 import cn.com.ctop.kuaishou.modules.report.service.IKuaishouReportDailyAgentService;
@@ -17,14 +20,18 @@ import cn.com.ctop.kuaishou.modules.report.service.IRuleKuaiShouPlanService;
 import cn.com.ctop.oa.modules.service.IWechatCheckinDataService;
 import cn.com.ctop.oa.modules.service.IWechatNoListService;
 import cn.com.ctop.oa.modules.service.IWechatUserListService;
+import cn.com.ctop.toutiao.modules.link.service.IETLReportBytedanceVideoService;
 import cn.com.ctop.toutiao.modules.material.service.IByteDanceAdvertiserDataService;
 import cn.com.ctop.toutiao.modules.report.entity.BytedanceAccountReportTaskRecord;
 import cn.com.ctop.toutiao.modules.report.mapper.BytedanceReportMaterialDailyMapper;
 import cn.com.ctop.toutiao.modules.report.service.*;
+import com.alibaba.fastjson.JSONArray;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.catalina.User;
 import org.jeecg.common.util.DateUtils;
 import org.junit.Test;
 import org.junit.runner.RunWith;
+import org.openqa.selenium.By;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.test.context.ActiveProfiles;
@@ -36,6 +43,7 @@ import java.text.SimpleDateFormat;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -63,47 +71,12 @@ public class SampleTest {
     private IBindAccountLoginService bindAccountLoginService;
     @Autowired
     private IKuaishouWebInterfaceService kuaishouWebInterfaceService;
-    @Autowired
-    private IOauthConfigService configService;
-    @Autowired
-    private CtopOauthTokenMapper oauthTokenMapper;
-    @Autowired
-    private IKuaishouInterfaceService iKuaishouInterfaceService;
-    @Autowired
-    private IRefreshTokenService refreshTokenService;
 
 
     @Test
     public void loadBytedanceCreativeData() throws ParseException {
-        CtopOauthToken token = oauthTokenMapper.selectByAccountId(9784689L);
-
-        refreshTokenService.getKuaiShouRefresh();
-
-
-     //   iKuaishouInterfaceService.getVideoList(token, null, null);
-
-      /*  List<CtopOauthToken> oauthTokens = oauthTokenMapper.selectKuaiShouTokens(CtopAdConstant.PLATFORM_TYPE_KUAISHOU);
-        if (!Check.isNull(oauthTokens)) {
-            Map<Long, String> configMap = configService.getConfigMapByMediaId(CtopAdConstant.PLATFORM_TYPE_KUAISHOU_INT);
-            for (CtopOauthToken oauthToken : oauthTokens) {
-                XxlJobHelper.log("刷新token,accountId:{}", oauthToken.getAccountId());
-                //快手
-                if (CtopAdConstant.PLATFORM_TYPE_KUAISHOU.equals(oauthToken.getMediaId().trim())) {
-                    String secret = configMap.get(oauthToken.getAppId());
-                    System.err.println(secret);
-                    oauthTokenService.getKuaiShouRefreshToken(oauthToken.getAccountId(), oauthToken.getAccessToken(), oauthToken.getRefreshToken(), oauthToken.getAppId(), secret);
-                }
-            }
-        } else {
-            XxlJobHelper.log("----------------------刷新token获取数据返回结果为空----------------------------------");
-        }*/
-
-
-      /*  Map<Long, String> configMap = configService.getConfigMapByMediaId(CtopAdConstant.PLATFORM_TYPE_KUAISHOU_INT);
-        System.err.println(configMap);*/
 
-
-      /*  List<KuaiShouVideoGet> KuaiShouVideoGets = videoGetService.selectPhotoIdsByAccountId(9743746L);
+        List<KuaiShouVideoGet> KuaiShouVideoGets = videoGetService.selectPhotoIdsByAccountId(9743746L);
         if (!Check.isNull(KuaiShouVideoGets)) {
             JSONArray dateList = new JSONArray();
             JSONArray photoIds = new JSONArray();
@@ -124,7 +97,7 @@ public class SampleTest {
             }
             dateList.add(photoIds);
             kuaishouVideoRelateCreativesService.videoRelateCreatives(9743746L, dateList, statdateMap);
-        }*/
+        }
     }
 
     @Test
@@ -564,4 +537,58 @@ public class SampleTest {
         CtopOauthToken token = tokenService.getTokenByAccountId(1660666858777607L);
         reportService.getAdvertiserPlanReport(token, DateUtils.parseDate("2021-01-25", "yyyy-MM-dd"), DateUtils.parseDate("2021-01-25", "yyyy-MM-dd"), CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
     }
+
+    @Autowired
+    IBytedanceVideoReportService bytedanceVideoReportService;
+
+    @Test
+    public void etlBytedanceVideoReport() {
+
+        for (int i = 1; i <= 62; i++) {
+            etlBytedanceVideo(DateUtils.formatDate(DateUtils.addDay(new Date(), -i)));
+        }
+        //etlBytedanceVideo("2021-02-01");
+
+    }
+
+    private void etlBytedanceVideo(String statDate) {
+        List<CtopOauthToken> tokens = oauthTokenService.selectToutiaoToken();
+        executorService = Executors.newFixedThreadPool(30);
+        countDownLatch = new CountDownLatch(tokens.size());
+        tokens.forEach(token -> executorService.submit(() -> {
+            try {
+                bytedanceVideoReportService.pullVideoReport(token, statDate, statDate, 1);
+            } catch (Exception e) {
+
+            } finally {
+                countDownLatch.countDown();
+            }
+        }));
+        try {
+            countDownLatch.await();
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+        }
+    }
+
+    @Autowired
+    IETLReportBytedanceVideoService ietlReportBytedanceVideoService;
+
+    @Test
+    public void etlBytedanceVideoReportClean() {
+        for (int i = 1; i <= 1; i++) {
+            String statDate = DateUtils.formatDate(DateUtils.addDay(new Date(), -i));
+            ietlReportBytedanceVideoService.cleanETL(statDate);
+        }
+    }
+
+    @Test
+    public void etlBytedanceVideoReportClean1() {
+        //executorService = Executors.newFixedThreadPool(31);
+        //countDownLatch = new CountDownLatch(31);
+        for (int i = 3; i <= 31; i++) {
+            String statDate = DateUtils.formatDate(DateUtils.addDay(new Date(), -i));
+                    ietlReportBytedanceVideoService.cleanETL(statDate);
+        }
+    }
 }

+ 1 - 1
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/BytedanceVideoReportServiceImpl.java

@@ -87,7 +87,7 @@ public class BytedanceVideoReportServiceImpl implements IBytedanceVideoReportSer
             Long materialId= videoReportJSONObject.getJSONObject("dimensions").getLong("material_id");
             JSONObject metrics= videoReportJSONObject.getJSONObject("metrics");
             BaseReportBytedanceVideoDaily temp= JSONObject.toJavaObject(metrics,BaseReportBytedanceVideoDaily.class);
-            temp.setMaterialShow(videoReportJSONObject.getLong("show"));
+            temp.setMaterialShow(metrics.getLong("show"));
             temp.setAccountId(oauthToken.getAccountId());
             temp.setMaterialId(materialId);
             temp.setStatDatetime(startDate);