Explorar o código

Merge remote-tracking branch 'origin/master'

zhaoxian %!s(int64=4) %!d(string=hai) anos
pai
achega
5f28ef35f0

+ 31 - 326
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/TestController.java

@@ -35,6 +35,7 @@ import cn.com.ctop.toutiao.modules.report.service.*;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import com.xxl.job.core.context.XxlJobHelper;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.ibatis.annotations.Param;
@@ -120,12 +121,9 @@ public class TestController {
     private IUserAllocationService allocationService;
     @Autowired
     private ICtopOauthTokenService tokenService;
-
     @Autowired
     private IKuaiShouProjectDataService kuaiShouProjectDataService;
     @Autowired
-    private IKuaishouReportHourlyGroupService hourlyGroupService;
-    @Autowired
     private IKuaiShouReportDailyMaterialService kuaiShouReportDailyMaterialService;
 
     @Autowired
@@ -177,207 +175,6 @@ public class TestController {
         return "success";
     }
 
-
-    @GetMapping(value = "/getProgramCreative")
-    public void getProgramCreative() {
-        List<Long> list = new ArrayList();
-        list.add(10045052L);
-        list.add(10045823L);
-        list.add(10045830L);
-        list.add(10045831L);
-
-        for (int i = 0; i < list.size(); i++) {
-            Long accountId = list.get(i);
-            CtopOauthToken oauthToken = oauthTokenService.getTokenByAccountId(accountId);
-            creativeService.getProgramCreative(oauthToken, 1);
-        }
-
-
-    }
-
-
-    @GetMapping(value = "/campaignList")
-    public void campaignList() {
-        List<Long> list = new ArrayList();
-        list.add(10045052L);
-        list.add(10045823L);
-        list.add(10045830L);
-        list.add(10045831L);
-        for (int i = 0; i < list.size(); i++) {
-            Long accountId = list.get(i);
-            CtopOauthToken oauthToken = oauthTokenService.getTokenByAccountId(accountId);
-            // kuaishouInterfaceService.getCampaignList(oauthToken, null, null);
-            videoService.submit(new Runnable() {
-                @Override
-                public void run() {
-                    try {
-                        kuaishouInterfaceService.getCampaignList(oauthToken, null, null);
-                        //   kuaishouInterfaceService.getGroupList(oauthToken, null, null);
-                        //     kuaishouInterfaceService.getCreativeList(oauthToken, null, null);
-                        Thread.sleep(1500);
-                    } catch (Exception e) {
-                        e.printStackTrace();
-                    }
-                }
-            });
-
-        }
-
-    }
-
-
-    @GetMapping(value = "/creativeList")
-    public void creativeList() {
-        List<Long> list = new ArrayList();
-        list.add(10045052L);
-        list.add(10045823L);
-        list.add(10045830L);
-        list.add(10045831L);
-        for (int i = 0; i < list.size(); i++) {
-            Long accountId = list.get(i);
-            CtopOauthToken oauthToken = oauthTokenService.getTokenByAccountId(accountId);
-            QueryWrapper<KuaiShouCampaign> campaignQueryWrapper = new QueryWrapper<>();
-            campaignQueryWrapper.eq("account_id", accountId);
-
-            List<KuaiShouCampaign> list1 = campaignService.list(campaignQueryWrapper);
-            for (KuaiShouCampaign campaign : list1) {
-                executorService.submit(new Runnable() {
-                    @Override
-                    public void run() {
-                        kuaishouInterfaceService.getCreativeList(oauthToken, null, null);
-                    }
-                });
-                try {
-                    Thread.sleep(400L);
-                } catch (InterruptedException e) {
-                    e.printStackTrace();
-                }
-            }
-
-
-        }
-
-    }
-
-
-    @GetMapping(value = "/updateUnit")
-    public void updateUnit() throws Exception {
-        List<Long> list = new ArrayList();
-        list.add(9556344L);
-        list.add(9743738L);
-        list.add(9743727L);
-        list.add(9812671L);
-        list.add(9838554L);
-        list.add(9743746L);
-        list.add(9767055L);
-        list.add(9556274L);
-        list.add(9767096L);
-        list.add(9792538L);
-        list.add(9766952L);
-        list.add(9677440L);
-        list.add(9677334L);
-        list.add(9556465L);
-        list.add(9677236L);
-        list.add(9556227L);
-        list.add(9556188L);
-        list.add(9556431L);
-        list.add(9677408L);
-        list.add(9677284L);
-
-        for (int i = 0; i < list.size(); i++) {
-            Long accountId = list.get(i);
-            CtopOauthToken oauthToken = oauthTokenService.getTokenByAccountId(accountId);
-            QueryWrapper<KuaiShouGroup> groupQueryWrapper = new QueryWrapper<>();
-            groupQueryWrapper.eq("account_id", accountId);
-            //      groupQueryWrapper.ne("schedule_time","000000111111111111111111000000111111111111111111000000111111111111111111000000111111111111111111000000111111111111111111000000111111111111111111000000111111111111111111");
-            //     groupQueryWrapper.eq("use_app_market", 1);
-            //    groupQueryWrapper.gt("cpa_bid", 19000);
-
-
-            List<KuaiShouGroup> list1 = groupService.list(groupQueryWrapper);
-          /*  for (KuaiShouGroup group : list1) {
-
-                JSONObject json = new JSONObject();
-                json.put("unitId", group.getUnitId());
-                json.put("useAppMarket", 0);
-                json.put("appStore", new JSONArray());
-                batchService.updateUnit(json, oauthToken);
-            }*/
-
-            for (KuaiShouGroup group : list1) {
-                executorService.submit(new Runnable() {
-                    @Override
-                    public void run() {
-                        try {
-                            JSONObject json = new JSONObject();
-                            json.put("unitId", group.getUnitId());
-                            json.put("cpaBid", 18 * 1000);
-                            //     json.put("gender", "2");
-                            batchService.updateUnit(json, oauthToken);
-                            Thread.sleep(1500);
-                        } catch (Exception e) {
-                            e.printStackTrace();
-                        }
-                    }
-                });
-
-
-            }
-
-
-        }
-
-
-    }
-
-
-    @GetMapping(value = "/updateCreative")
-    public void updateCreative() {
-        List<Long> list = new ArrayList();
-        list.add(10045052L);
-        list.add(10045823L);
-        list.add(10045830L);
-        list.add(10045831L);
-
-        for (int i = 0; i < list.size(); i++) {
-            Long accountId = list.get(i);
-            CtopOauthToken oauthToken = oauthTokenService.getTokenByAccountId(accountId);
-
-
-            QueryWrapper<KuaiShouCreative> creativeQueryWrapper = new QueryWrapper<>();
-            creativeQueryWrapper.eq("account_id", accountId);
-
-
-            List<KuaiShouCreative> creativeList = creativeService.list(creativeQueryWrapper);
-            for (KuaiShouCreative creative : creativeList) {
-                executorService.submit(new Runnable() {
-                    @Override
-                    public void run() {
-                        try {
-                            JSONObject json = new JSONObject();
-                            json.put("creativeId", creative.getCreativeId());
-                            String url = "https://click-wailaxin.1sapp.com/vendor/feedback/reportClick?platform=kuaishou&feedback_type=android&account_id=19&plan_id=__DID__&adgroup_id=__AID__&creative_id=__CID__&imei=__IMEI2__&mac=__MAC__&androidid=__ANDROIDID2__&ts=__TS__&ip=__IP__&ua=__UA__&callback=__CALLBACK__&oaid=__OAID__&app=dxqlb";
-
-                            if (creative.getCreativeMaterialType() == 1) {
-                                json.put("clickTrackUrl", url);
-                            } else if (creative.getCreativeMaterialType() == 4) {
-                                json.put("impressionUrl", url);
-                                json.put("clickTrackUrl", url);
-
-                            }
-                            JSONObject jsonObject = batchService.updateCreative(json, oauthToken);
-                            log.info("修改创意返回信息:creative:{},结果:{}", creative.getCreativeId(), jsonObject);
-                            Thread.sleep(1000L);
-                        } catch (Exception e) {
-                            e.printStackTrace();
-                        }
-                    }
-                });
-            }
-        }
-    }
-
-
     @GetMapping("/cleanNewlyData")
     public void cleanNewlyData(String startDate, String endDate) throws ParseException {
 
@@ -423,19 +220,6 @@ public class TestController {
 
     }
 
-    @GetMapping("/getVideoByProjectId")
-    public String getVideoByProjectId(Long projectId) {
-        QueryWrapper<UserAllocation> queryWrapper = new QueryWrapper<>();
-        queryWrapper.eq("project_id", projectId);
-        List<UserAllocation> list = allocationService.list(queryWrapper);
-        for (UserAllocation allocation : list) {
-            CtopOauthToken token = tokenService.getTokenByAccountId(allocation.getAccountId());
-            kuaishouInterfaceService.getVideoList(token, null, null);
-        }
-        return "success";
-    }
-
-
     @GetMapping("/testKuaishou")
     public String kuaishouCallback(@RequestParam("auth_code") String authCode, @RequestParam("state") String state) {
         try {
@@ -545,115 +329,6 @@ public class TestController {
         return "授权绑定失败!";
     }
 
-
-    @GetMapping("/getBytedanceMatReport")
-    public Map<String, Object> getData() {
-        List<CtopOauthToken> tokens = tokenService.selectToutiaoToken();
-        executorService = Executors.newFixedThreadPool(10);
-        CountDownLatch countDownLatch = new CountDownLatch(tokens.size());
-        tokens.forEach(record -> executorService.submit(() -> {
-            try {
-                reportService.getAdvertiserReport(record, DateUtils.parseDate("2021-01-01", "yyyy-MM-dd"), DateUtils.parseDate("2021-01-31", "yyyy-MM-dd"), CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
-                log.info("账户数据获取完成:account:{},时间:{}-{}", record.getAccountId(), "2021-01-01", "2020-01-31");
-            } catch (Exception e) {
-
-            } finally {
-                countDownLatch.countDown();
-            }
-        }));
-        try {
-            countDownLatch.await();
-        } catch (InterruptedException e) {
-            e.printStackTrace();
-        }
-        Map<String, Object> result = new HashMap<>();
-        ResultMapUtils.setResultMap(new HashMap<>(), StatusCode.COMMON_SUCCESS);
-        return result;
-    }
-
-
-    @GetMapping(value = "/updateTemplate")
-    public void updateTemplate(Long projectId, String startDate, String endDate) {
-
-        List<Long> list = new ArrayList();
-        list.add(7022550L);
-        list.add(7409586L);
-        list.add(5473041L);
-        list.add(5852766L);
-        list.add(6962896L);
-        list.add(7409919L);
-        for (int i = 0; i < list.size(); i++) {
-            Long accountId = list.get(i);
-            CtopOauthToken oauthToken = oauthTokenService.getTokenByAccountId(accountId);
-            templateService.getTemplateByAccountId(oauthToken.getAccountId(), oauthToken.getAccessToken(), 1);
-
-            QueryWrapper<KuaishouTemplate> queryWrapper = new QueryWrapper<>();
-            queryWrapper.eq("account_id", accountId);
-            List<KuaishouTemplate> list1 = templateService.list(queryWrapper);
-            if (!Check.isNull(list1)) {
-                for (KuaishouTemplate template : list1) {
-                    JSONObject param = new JSONObject();
-                    param.put("templateId", template.getTemplateId());
-                    param.put("accountId", template.getAccountId());
-                    param.put("templateName", template.getTemplateName());
-                    param.put("filterConvertedLevel", 5);
-                    templateService.updateTemplate(oauthToken.getAccountId(), oauthToken.getAccessToken(), param);
-
-                }
-
-            }
-
-
-        }
-
-
-    }
-
-
-    @GetMapping(value = "/getCreativeByProjectId")
-    public void gerCreative(Long projectId, String startDate, String endDate) {
-
-        Result<String> result = new Result<>();
-        try {
-            SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
-            Date start = simpleDateFormat.parse(startDate);
-            Date end = simpleDateFormat.parse(endDate);
-            QueryWrapper<UserAllocation> userAllocationQueryWrapper = new QueryWrapper<>();
-            userAllocationQueryWrapper.eq("project_id", projectId);
-            List<UserAllocation> list = userAllocationService.list(userAllocationQueryWrapper);
-            if (!Check.isNull(list)) {
-                for (UserAllocation allocation : list) {
-                    Long accountId = allocation.getAccountId();
-                    QueryWrapper<CtopOauthToken> tokenQueryWrapper = new QueryWrapper<>();
-                    tokenQueryWrapper.eq("media_id", 2);
-                    tokenQueryWrapper.eq("account_id", accountId);
-                    CtopOauthToken token = tokenMapper.selectOne(tokenQueryWrapper);
-                    if (!Check.isNull(token)) {
-                        suzhaoService.submit(new Runnable() {
-                            @Override
-                            public void run() {
-                                JSONArray arr = new JSONArray();
-                                kuaishouInterfaceService.getAdvertiserCreativeReportDaily(token, start, end, arr, arr);
-
-                            }
-                        });
-
-
-                    }
-                }
-
-            }
-
-
-        } catch (Exception e) {
-            e.printStackTrace();
-            result.setSuccess(false);
-        }
-
-
-    }
-
-
     @GetMapping(value = "/agentReport")
     public void agentReport(String startDate, String endDate) {
         dailyAgentService.getAgentReportByPage(startDate, endDate);
@@ -1585,4 +1260,34 @@ public class TestController {
         ResultMapUtils.setResultMap(result, StatusCode.COMMON_SUCCESS);
         return result;
     }
+
+    @GetMapping("checkCostData")
+    public void checkCostData(String date){
+            //1:初始化
+            checkReportTaskInfoService.initTask(date);
+            //2: checkCost
+            List<BytedanceCheckReportTaskInfo>taskInfos = checkReportTaskInfoService.getListByParams(null,date,0);
+            if(null!=taskInfos&&!taskInfos.isEmpty()){
+                for (BytedanceCheckReportTaskInfo task:taskInfos) {
+                    checkReportTaskInfoService.checkCost(task);
+                }
+                log.info("{}:数据获取完成",date);
+            }
+    }
+
+    @GetMapping("refreshReportData")
+    public void refreshReportData()throws Exception{
+        //1:查询异常任务数据
+        List<BytedanceCheckReportTaskInfo>taskInfos = checkReportTaskInfoService.getListByParams(null,null,50001);
+        if(null!=taskInfos&&!taskInfos.isEmpty()){
+            taskInfos.forEach(taskInfo -> {
+                try {
+                    checkReportTaskInfoService.reloadData(taskInfo);
+                    checkReportTaskInfoService.checkCost(taskInfo);
+                }catch (Exception e) {
+                }
+            });
+            log.info("数据更新完成");
+        }
+    }
 }

+ 2 - 20
jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java

@@ -193,8 +193,8 @@ public class SampleTest {
 
     @Test
     public void checkCostData(){
-        String date = "2021-06-01";
-        for (int i=0;i<14;i++){
+        String date = "2021-06-11";
+        for (int i=0;i<7;i++){
             String getDate = DateUtils.addDay(date,i);
             //1:初始化
             checkReportTaskInfoService.initTask(getDate);
@@ -210,24 +210,6 @@ public class SampleTest {
     }
 
     @Test
-    public void reloadKuaishouData(){
-        String formatDate = "2021-06-01";
-        List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
-        tokens.forEach(token -> {
-            try {
-                List<EtlKuaishouAccountMaterialReportDaily> getList = etlKuaishouAccountMaterialReportDailyMapper.etlKuaishouAccountMaterialReportDailyList(formatDate,token.getAccountId());
-                if(null!=getList&&!getList.isEmpty()){
-                    getList.forEach(report->etlKuaishouAccountMaterialReportDailyMapper.replace(report));
-                }
-            }catch (Exception e){
-                e.printStackTrace();
-            }
-        });
-
-        log.info("{}数据获取完成",formatDate);
-    }
-
-    @Test
     public void refreshReportData()throws Exception{
         //1:查询异常任务数据
         List<BytedanceCheckReportTaskInfo>taskInfos = checkReportTaskInfoService.getListByParams(null,null,50001);

+ 10 - 6
module-job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedanceCheckAccountReportJob.java

@@ -23,22 +23,26 @@ public class BytedanceCheckAccountReportJob {
     private IBytedanceCheckReportTaskInfoService checkReportTaskInfoService;
     @XxlJob("initCheckReportJob")
     public void initCheckReportJob() throws Exception {
-        String params = DateUtils.addDay(DateUtils.formatDate(),-1);
-        checkReportTaskInfoService.initTask(params);
+        String date = DateUtils.addDay(DateUtils.formatDate(),-1);
+        String param = XxlJobHelper.getJobParam();
+        if (param==null || param.trim().length()==0) {
+            param = date;
+        }
+        checkReportTaskInfoService.initTask(date);
         //查询初始化的任务
-        List<BytedanceCheckReportTaskInfo>taskInfos = checkReportTaskInfoService.getListByParams(null,params,0);
+        List<BytedanceCheckReportTaskInfo>taskInfos = checkReportTaskInfoService.getListByParams(null,date,0);
         if(null!=taskInfos&&!taskInfos.isEmpty()){
             countDownLatch = new CountDownLatch(taskInfos.size());
-            taskInfos.forEach(taskInfo -> {
+            taskInfos.forEach(taskInfo ->executorService.submit(()->{
                 try {
                     checkReportTaskInfoService.checkCost(taskInfo);
                 }catch (Exception e){
                 }finally {
                     countDownLatch.countDown();
                 }
-            });
+            }));
             countDownLatch.await();
-            XxlJobHelper.log("{}数据初始化完成",params);
+            XxlJobHelper.log("{}数据初始化完成",date);
         }
     }
 

+ 11 - 0
module-job-kuaishou/src/main/java/cn/com/ctop/job/kuaishou/handler/KuaishouEtlVideoJob.java

@@ -60,4 +60,15 @@ public class KuaishouEtlVideoJob {
             materialReportDailyService.etlKuaishouAccountMaterialReportInfo(DateUtils.formatDate(DateUtils.addDay(new Date(),-i)));
         }
     }
+
+    @XxlJob("etlKuaishouReportCompanyInfo")
+    public void etlKuaishouReportCompanyInfo(){
+        Date date = DateUtils.addDay(new Date(),-1);
+        materialReportDailyService.etlKuaishouReportCompanyInfo(DateUtils.formatDate(date));
+    }
+
+    @XxlJob("etlKuaishouVideoReportStateDate")
+    public void etlKuaishouVideoReportStateDate(){
+        materialReportDailyService.etlKuaishouVideoReportStateDate();
+    }
 }

+ 4 - 3
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/ai/service/impl/KuaishouLandpagePackageServiceImpl.java

@@ -26,6 +26,7 @@ import org.jeecg.common.system.vo.LoginUser;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
+import javax.annotation.Resource;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -41,14 +42,14 @@ import java.util.Map;
 @Service
 public class KuaishouLandpagePackageServiceImpl extends ServiceImpl<KuaishouLandpagePackageMapper, KuaishouLandpagePackage> implements IKuaishouLandpagePackageService {
 
-    @Autowired
+    @Resource
     private KuaishouLandpagePackageMapper landpagePackageMapper;
-    @Autowired
+    @Resource
     private KuaishouLandpagePackageRelMapper landpagePackageRelMapper;
     @Autowired
     private KuaishouLandpagePackageRelServiceImpl landpagePackageRelService;
 
-    @Autowired
+    @Resource
     private UserAllocationMapper userAllocationMapper;
     @Autowired
     private ICtopOauthTokenService oauthTokenService;

+ 4 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/EtlKuaishouAccountMaterialReportDailyMapper.java

@@ -21,4 +21,8 @@ public interface EtlKuaishouAccountMaterialReportDailyMapper extends BaseMapper<
     void replaceBatch(@Param(value = "records")List<EtlKuaishouAccountMaterialReportDaily> getList);
 
     void replace(@Param(value = "report")EtlKuaishouAccountMaterialReportDaily report);
+
+    void etlKuaishouReportCompanyInfo(@Param(value = "formatDate") String formatDate);
+
+    void etlKuaishouVideoReportStateDate();
 }

+ 18 - 1
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/xml/EtlKuaishouAccountMaterialReportDailyMapper.xml

@@ -208,6 +208,22 @@
             #{report.play3sCount}
         )
     </insert>
+    <update id="etlKuaishouReportCompanyInfo">
+        update ctop_etl_kuaishou_account_material_report_daily a left join ctop_user_allocation cua on a.account_id = cua.account_id
+            left join user_company uc on cua.user_id = uc.user_id
+            set a.company_name = uc.company_name
+        where a.company_name is null
+          and uc.company_name is not null
+          and a.stat_date = #{formatDate}
+    </update>
+    <update id="etlKuaishouVideoReportStateDate">
+        UPDATE ctop_etl_kuaishou_video_info info
+            LEFT JOIN ( SELECT min( stat_date ) AS minDate, signature FROM ctop_kuaishou_report_daily_material GROUP BY signature ) a ON a.signature = info.video_code
+            SET info.state_date = a.minDate
+        WHERE
+            a.minDate IS NOT NULL
+          AND info.state_date IS NULL
+    </update>
     <select id="etlKuaishouAccountMaterialReportDailyList"
             resultType="cn.com.ctop.kuaishou.modules.report.entity.EtlKuaishouAccountMaterialReportDaily">
     select
@@ -216,7 +232,7 @@
     u.project_id ,
     p.project_name,
     m.account_id,
-    m.signature,
+    video.signature,
     sum(m.charge)as charge ,
     sum(m.photo_show)as photo_show ,
     sum(m.aclick)as aclick ,
@@ -242,6 +258,7 @@
     sum(m.form_count)as form_count,
      round(sum(m.play_3s_ratio*m.photo_show))as play_3s_count
     from ctop_kuaishou_report_daily_material m left join ctop_user_allocation u on u.account_id = m.account_id
+    left  join ctop_kuaishou_video_get video on video.account_id = m.account_id and m.photo_id = video.photo_id
 left join ctop_project p on p.id = u.project_id
 left join user_company c on c.user_id = u.account_id
 where 1=1

+ 4 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/service/IEtlKuaishouAccountMaterialReportDailyService.java

@@ -14,4 +14,8 @@ public interface IEtlKuaishouAccountMaterialReportDailyService extends IService<
     void etlKuaishouAccountMaterialReportDailyData(String date);
 
     void etlKuaishouAccountMaterialReportInfo(String formatDate);
+
+    void etlKuaishouReportCompanyInfo(String formatDate);
+
+    void etlKuaishouVideoReportStateDate();
 }

+ 10 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/service/impl/EtlKuaishouAccountMaterialReportDailyServiceImpl.java

@@ -76,4 +76,14 @@ public class EtlKuaishouAccountMaterialReportDailyServiceImpl extends ServiceImp
         }
         log.info("{}数据获取完成",formatDate);
     }
+
+    @Override
+    public void etlKuaishouReportCompanyInfo(String formatDate) {
+        etlKuaishouAccountMaterialReportDailyMapper.etlKuaishouReportCompanyInfo(formatDate);
+    }
+
+    @Override
+    public void etlKuaishouVideoReportStateDate() {
+        etlKuaishouAccountMaterialReportDailyMapper.etlKuaishouVideoReportStateDate();
+    }
 }