Ver código fonte

Merge remote-tracking branch 'origin/master'

zhaoxian 4 anos atrás
pai
commit
8ac603ee16

+ 26 - 76
jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java

@@ -5,21 +5,18 @@ import cn.com.ctop.common.module.entity.CtopOauthToken;
 import cn.com.ctop.common.module.service.IBindAccountLoginService;
 import cn.com.ctop.common.module.service.ICtopOauthTokenService;
 import cn.com.ctop.common.module.utils.CtopAdConstant;
-import cn.com.ctop.kuaishou.modules.batch.service.IKuaishouInterfaceService;
 import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouWebInterfaceService;
+import cn.com.ctop.kuaishou.modules.report.entity.EtlKuaishouAccountMaterialReportDaily;
 import cn.com.ctop.kuaishou.modules.report.mapper.EtlKuaishouAccountMaterialReportDailyMapper;
 import cn.com.ctop.kuaishou.modules.report.service.IKuaishouReportDailyImageService;
 import cn.com.ctop.oa.modules.service.IWechatNoListService;
-import cn.com.ctop.oa.modules.service.IWechatUserListService;
 import cn.com.ctop.toutiao.modules.link.service.IETLReportBytedanceVideoService;
 import cn.com.ctop.toutiao.modules.material.service.IByteDanceAdvertiserDataService;
-import cn.com.ctop.toutiao.modules.material.service.IByteDanceCampaignService;
 import cn.com.ctop.toutiao.modules.report.entity.BytedanceCheckReportTaskInfo;
 import cn.com.ctop.toutiao.modules.report.service.*;
 import com.xxl.job.core.context.XxlJobHelper;
 import lombok.extern.slf4j.Slf4j;
 import org.jeecg.common.util.DateUtils;
-import org.jeecg.common.util.RedisUtil;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -58,8 +55,6 @@ public class SampleTest {
     @Autowired
     private IKuaishouWebInterfaceService kuaishouWebInterfaceService;
     @Autowired
-    private IWechatUserListService wechatUserInfoService;
-    @Autowired
     private IByteDanceAdvertiserDataService advertiserDataService;
     @Autowired
     private IWechatNoListService wechatNoListService;
@@ -70,78 +65,27 @@ public class SampleTest {
     @Resource
     private EtlKuaishouAccountMaterialReportDailyMapper etlKuaishouAccountMaterialReportDailyMapper;
     @Autowired
-    private IByteDanceCampaignService campaignService;
-    @Autowired
     private IReportService reportService;
     @Resource
     private IBytedanceCheckReportTaskInfoService checkReportTaskInfoService;
     @Autowired
     private IEtlBytedanceReportVideoDailyService etlBytedanceReportVideoDailyService;
-    //    @Autowired
-    //    private IAppchatCreateService appchatCreateService;
-
-
-
+    static ExecutorService executorService = null;
+    static CountDownLatch countDownLatch = null;
+    @Autowired
+    private IBytedanceReportService bytedanceReportService;
+    @Autowired
+    private IETLReportBytedanceVideoService etlReportBytedanceVideoService;
 
     @Test
     public void loadKuaishouCookie() throws ParseException {
-
         CtopOauthToken tokenByAccountId = tokenService.getTokenByAccountId(1700996735138951L);
         advertiserDataService.getMaterialList(tokenByAccountId);
 
-
-
         CtopOauthToken tokenByAccountId1 = tokenService.getTokenByAccountId(1700996735589383L);
         advertiserDataService.getMaterialList(tokenByAccountId1);
-
-/*
-        wechatUserInfoService.getUserList();*/
-
-        // appchatCreateService.createAppChat(null, null, null);
-   //     appchatCreateService.getChatInfo(null);
-//        appchatCreateService.getChatInfo(null);
-       /* redisUtil.set("11sdfhgjksdhfjksdhfjksdhfkjsjkfhkjsddhfjksdhfkjhsjkfhjkshfjksdhfjksdhfjkshdjkfhsjkdfhkjsd1","hahahaha",5);
-        Object o = redisUtil.get("11sdfhgjksdhfjksdhfjksdhfkjsjkfhkjsddhfjksdhfkjhsjkfhjkshfjksdhfjksdhfjkshdjkfhsjkdfhkjsd1");
-        System.err.println(o);*/
-
-
-        /*try {
-            // 用来保存数据
-            ArrayList<String[]> csvFileList = new ArrayList<String[]>();
-            // 定义一个CSV路径
-            String csvFilePath = "D:\\test\\123.csv";
-            // 创建CSV读对象 例如:CsvReader(文件路径,分隔符,编码格式);
-            CsvReader reader = new CsvReader(csvFilePath, ',', Charset.forName("UTF-8"));
-            // 跳过表头 如果需要表头的话,这句可以忽略
-            reader.readHeaders();
-            // 逐行读入除表头的数据
-            while (reader.readRecord()) {
-                System.out.println(reader.getRawRecord());
-                csvFileList.add(reader.getValues());
-            }
-            reader.close();
-
-            // 遍历读取的CSV文件
-            for (int row = 0; row < csvFileList.size(); row++) {
-                // 取得第row行第0列的数据
-                String cell = csvFileList.get(row)[0];
-                System.out.println("------------>" + cell);
-            }
-        } catch (IOException e) {
-            e.printStackTrace();
-        }*/
-
-
-
-
-
-       /* CtopOauthToken tokenByAccountId = tokenService.getTokenByAccountId(9792526L);
-        SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
-        Date parse = dateFormat.parse("2021-04-02");
-        kuaishouInterfaceService.getAdvertiserReportDaily(tokenByAccountId, parse, parse, null);*/
     }
-    static ExecutorService executorService = null;
-    static CountDownLatch countDownLatch = null;
+
 
 
 
@@ -228,8 +172,6 @@ public class SampleTest {
         etlKuaishouAccountMaterialReportDailyMapper.etlKuaishouAccountMaterialReportDailyData(DateUtils.formatDate());
     }
 
-    @Autowired
-    private IBytedanceReportService bytedanceReportService;
 
     @Test
     public void testLoadBytedanceCampaign() {
@@ -245,12 +187,6 @@ public class SampleTest {
 
     @Test
     public void loadBytedanceAccountReport() {
-        /**
-         * 1692555408132110
-         * 1692555631713287
-         * 1696734332089352
-         * 1696734333251598
-         */
         CtopOauthToken token1 = tokenService.getTokenByAccountId(1693648201498637L);
         reportService.getAdvertiserReport(token1, DateUtils.parseDate("2021-04-01", "yyyy-MM-dd"), DateUtils.parseDate("2021-04-12", "yyyy-MM-dd"), CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
     }
@@ -271,7 +207,24 @@ public class SampleTest {
                 log.info("{}:数据获取完成",getDate);
             }
         }
-// 3:重复异常数据
+    }
+
+    @Test
+    public void reloadKuaishouData(){
+        String formatDate = "2021-06-01";
+        List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
+        tokens.forEach(token -> {
+            try {
+                List<EtlKuaishouAccountMaterialReportDaily> getList = etlKuaishouAccountMaterialReportDailyMapper.etlKuaishouAccountMaterialReportDailyList(formatDate,token.getAccountId());
+                if(null!=getList&&!getList.isEmpty()){
+                    getList.forEach(report->etlKuaishouAccountMaterialReportDailyMapper.replace(report));
+                }
+            }catch (Exception e){
+                e.printStackTrace();
+            }
+        });
+
+        log.info("{}数据获取完成",formatDate);
     }
 
     @Test
@@ -300,7 +253,4 @@ public class SampleTest {
         etlReportBytedanceVideoService.etlBytedanceVideoPlanInfoJob();
         etlReportBytedanceVideoService.etlBytedanceVideoShotInfoJob();
     }
-
-    @Autowired
-    private IETLReportBytedanceVideoService etlReportBytedanceVideoService;
 }

+ 6 - 15
module-job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedanceCheckAccountReportJob.java

@@ -23,14 +23,10 @@ public class BytedanceCheckAccountReportJob {
     private IBytedanceCheckReportTaskInfoService checkReportTaskInfoService;
     @XxlJob("initCheckReportJob")
     public void initCheckReportJob() throws Exception {
-        String params = DateUtils.formatDate();
-        String date = XxlJobHelper.getJobParam();
-        if (date==null || date.trim().length()==0) {
-            date = params;
-        }
-        checkReportTaskInfoService.initTask(date);
+        String params = DateUtils.addDay(DateUtils.formatDate(),-1);
+        checkReportTaskInfoService.initTask(params);
         //查询初始化的任务
-        List<BytedanceCheckReportTaskInfo>taskInfos = checkReportTaskInfoService.getListByParams(null,date,0);
+        List<BytedanceCheckReportTaskInfo>taskInfos = checkReportTaskInfoService.getListByParams(null,params,0);
         if(null!=taskInfos&&!taskInfos.isEmpty()){
             countDownLatch = new CountDownLatch(taskInfos.size());
             taskInfos.forEach(taskInfo -> {
@@ -42,19 +38,14 @@ public class BytedanceCheckAccountReportJob {
                 }
             });
             countDownLatch.await();
-            XxlJobHelper.log("{}数据初始化完成",date);
+            XxlJobHelper.log("{}数据初始化完成",params);
         }
     }
 
     @XxlJob("refreshCheckReportJob")
     public void refreshCheckReportJob() throws Exception {
-        String params = DateUtils.formatDate();
-        String date = XxlJobHelper.getJobParam();
-        if (date==null || date.trim().length()==0) {
-            date = params;
-        }
         //1:查询异常任务数据
-        List<BytedanceCheckReportTaskInfo>taskInfos = checkReportTaskInfoService.getListByParams(null,date,50001);
+        List<BytedanceCheckReportTaskInfo>taskInfos = checkReportTaskInfoService.getListByParams(null,null,50001);
         if(null!=taskInfos&&!taskInfos.isEmpty()){
             countDownLatch = new CountDownLatch(taskInfos.size());
             taskInfos.forEach(taskInfo -> {
@@ -67,7 +58,7 @@ public class BytedanceCheckAccountReportJob {
                 }
             });
             countDownLatch.await();
-            XxlJobHelper.log("{}数据更新完成",date);
+            XxlJobHelper.log("数据更新完成");
         }
     }
 }

+ 2 - 2
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/ai/service/impl/AiKuaishouCreateCreativeServiceImpl.java

@@ -802,9 +802,9 @@ public class AiKuaishouCreateCreativeServiceImpl implements IAiKuaishouCreateCre
             List<KuaiShouVideoGet> customVideos = new ArrayList<>();
 
             if (Check.isNull(strategy.getMaterialKeyword())) {
-                generalVideos = videoGetService.getNewVideoBetweenTime(strategy.getAccountId(), startTime, endTime, strategy.getChannelType(), "all", null);
+                customVideos = videoGetService.getNewVideoBetweenTime(strategy.getAccountId(), startTime, endTime, strategy.getChannelType(), "all", null);
                 //2:查询特定素材
-                customVideos = videoGetService.getNewVideoBetweenTime(strategy.getAccountId(), startTime, endTime, strategy.getChannelType(), appInfo.getAppVersion(), null);
+                generalVideos = videoGetService.getNewVideoBetweenTime(strategy.getAccountId(), startTime, endTime, strategy.getChannelType(), appInfo.getAppVersion(), null);
             } else { //  tt语音特殊素材查询
                 generalVideos = videoGetService.getNewVideoBetweenTime(strategy.getAccountId(), startTime, endTime, strategy.getChannelType(), strategy.getMaterialKeyword(), null);
 

+ 2 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/EtlKuaishouAccountMaterialReportDailyMapper.java

@@ -19,4 +19,6 @@ public interface EtlKuaishouAccountMaterialReportDailyMapper extends BaseMapper<
     List<EtlKuaishouAccountMaterialReportDaily> etlKuaishouAccountMaterialReportDailyList(@Param(value = "date")String date,@Param(value = "accountId")Long accountId);
 
     void replaceBatch(@Param(value = "records")List<EtlKuaishouAccountMaterialReportDaily> getList);
+
+    void replace(@Param(value = "report")EtlKuaishouAccountMaterialReportDaily report);
 }

+ 67 - 1
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/xml/EtlKuaishouAccountMaterialReportDailyMapper.xml

@@ -142,6 +142,72 @@
             )
         </foreach>
     </insert>
+    <insert id="replace">
+    replace into ctop_etl_kuaishou_account_material_report_daily(
+        company_id ,
+        company_name ,
+        project_id ,
+        project_name,
+        account_id,
+        signature,
+        charge,
+        photo_show ,
+        aclick ,
+        bclick ,
+        download_started ,
+        download_completed ,
+        activation ,
+        submit ,
+        stat_date ,
+        photo_click ,
+        event_pay_first_day ,
+        event_pay_purchase_amount_first_day ,
+        event_pay ,
+        event_pay_purchase_amount ,
+        event_register ,
+        event_jin_jian_app ,
+        event_credit_grant_app ,
+        event_order_paid ,
+        event_order_paid_purchase_amount ,
+        event_jin_jian_landing_page ,
+        event_credit_grant_landing_page ,
+        event_next_day_stay ,
+        form_count,
+        play_3s_count)
+        values
+        (
+            #{report.companyId},
+            #{report.companyName},
+            #{report.projectId},
+            #{report.projectName},
+            #{report.accountId},
+            #{report.signature},
+            #{report.charge},
+            #{report.photoShow},
+            #{report.aclick},
+            #{report.bclick},
+            #{report.downloadStarted} ,
+            #{report.downloadCompleted} ,
+            #{report.activation},
+            #{report.submit},
+            #{report.statDate},
+            #{report.photoClick},
+            #{report.eventPayFirstDay},
+            #{report.eventPayPurchaseAmountFirstDay},
+            #{report.eventPay},
+            #{report.eventPayPurchaseAmount},
+            #{report.eventRegister},
+            #{report.eventJinJianApp},
+            #{report.eventCreditGrantApp},
+            #{report.eventOrderPaid},
+            #{report.eventOrderPaidPurchaseAmount},
+            #{report.eventJinJianLandingPage},
+            #{report.eventCreditGrantLandingPage} ,
+            #{report.eventNextDayStay},
+            #{report.formCount},
+            #{report.play3sCount}
+        )
+    </insert>
     <select id="etlKuaishouAccountMaterialReportDailyList"
             resultType="cn.com.ctop.kuaishou.modules.report.entity.EtlKuaishouAccountMaterialReportDaily">
     select
@@ -178,7 +244,7 @@
     from ctop_kuaishou_report_daily_material m left join ctop_user_allocation u on u.account_id = m.account_id
 left join ctop_project p on p.id = u.project_id
 left join user_company c on c.user_id = u.account_id
-where m.signature is not null
+where 1=1
 and m.stat_date = #{date} and m.account_id = #{accountId}
 group by m.account_id,m.signature
     </select>

+ 3 - 1
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/service/impl/EtlKuaishouAccountMaterialReportDailyServiceImpl.java

@@ -60,7 +60,9 @@ public class EtlKuaishouAccountMaterialReportDailyServiceImpl extends ServiceImp
         tokens.forEach(token -> etlExecutorService.submit(()->{
             try {
                 List<EtlKuaishouAccountMaterialReportDaily> getList = etlKuaishouAccountMaterialReportDailyMapper.etlKuaishouAccountMaterialReportDailyList(formatDate,token.getAccountId());
-                etlKuaishouAccountMaterialReportDailyMapper.replaceBatch(getList);
+                if(null!=getList&&!getList.isEmpty()){
+                    getList.forEach(report->etlKuaishouAccountMaterialReportDailyMapper.replace(report));
+                }
             }catch (Exception e){
                 e.printStackTrace();
             }finally {