浏览代码

盯盘 数据清洗

yumeng 4 年之前
父节点
当前提交
772a3fe749

+ 64 - 8
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/TestController.java

@@ -15,10 +15,7 @@ import cn.com.ctop.kuaishou.modules.batch.service.*;
 import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouWebInterfaceService;
 import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouWebInterfaceService;
 import cn.com.ctop.kuaishou.modules.material.entity.KuaishouAccessToken;
 import cn.com.ctop.kuaishou.modules.material.entity.KuaishouAccessToken;
 import cn.com.ctop.kuaishou.modules.material.entity.KuaishouResultToken;
 import cn.com.ctop.kuaishou.modules.material.entity.KuaishouResultToken;
-import cn.com.ctop.kuaishou.modules.report.service.IEtlKuaiShouNewlyService;
-import cn.com.ctop.kuaishou.modules.report.service.IKuaiShouDailyAgentService;
-import cn.com.ctop.kuaishou.modules.report.service.IKuaishouAudienceReportDailyService;
-import cn.com.ctop.kuaishou.modules.report.service.IKuaishouReportDailyGroupService;
+import cn.com.ctop.kuaishou.modules.report.service.*;
 import cn.com.ctop.toutiao.modules.link.service.IETLReportBytedanceVideoService;
 import cn.com.ctop.toutiao.modules.link.service.IETLReportBytedanceVideoService;
 import cn.com.ctop.toutiao.modules.material.service.IByteDanceAdvertiserDataService;
 import cn.com.ctop.toutiao.modules.material.service.IByteDanceAdvertiserDataService;
 import cn.com.ctop.toutiao.modules.material.service.IBytedanceEffectVideoInfoService;
 import cn.com.ctop.toutiao.modules.material.service.IBytedanceEffectVideoInfoService;
@@ -125,6 +122,68 @@ public class TestController {
     private IUserAllocationService allocationService;
     private IUserAllocationService allocationService;
     @Autowired
     @Autowired
     private IEtlKuaiShouNewlyService kuaiShouNewlyService;
     private IEtlKuaiShouNewlyService kuaiShouNewlyService;
+    static ExecutorService materialExecutorService = Executors.newFixedThreadPool(8);
+
+    @Autowired
+    private ICtopOauthTokenService tokenService;
+    @Autowired
+    private IEtlKuaiShouGroupDailyReportService etlKuaiShouGroupDailyReportService;
+    @Autowired
+    private IEtlKuaiShouAccountMaterialOverviewService accountMaterialOverviewService;
+
+    @Autowired
+    private IEtlKuaiShouNewlyService newlyService;
+
+
+    @GetMapping("/cleanNewlyData")
+    public void cleanNewlyData() {
+        String nowDate = DateUtils.getNowDate("yyyy-MM-dd");
+        newlyService.cleanNewlyData(nowDate);
+        XxlJobHelper.log("快手清洗上新计划数据完成");
+    }
+
+
+    @GetMapping("/cleanGroupReport")
+    public void cleanGroupReport() throws Exception {
+        List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
+        if (null == tokens || tokens.isEmpty()) {
+            XxlJobHelper.log("定时获取快手数据异常:未获取到可用的token");
+            XxlJobHelper.handleFail();
+            return;
+        }
+        String nowDate = DateUtils.getNowDate("yyyy-MM-dd");
+        for (CtopOauthToken token : tokens) {
+            executorService.submit(new Runnable() {
+                @Override
+                public void run() {
+                    etlKuaiShouGroupDailyReportService.getGroupReport(token.getAccountId(), token.getAccessToken(), nowDate, nowDate, 1);
+                }
+            });
+        }
+    }
+
+
+    @GetMapping("/cleanAccountMaterialOverview")
+    public void cleanAccountMaterialOverview() throws Exception {
+        String nowDate = DateUtils.getNowDate("yyyy-MM-dd");
+        String yesterdayDate = DateUtils.getAnotherDay("yyyy-MM-dd", nowDate, -1);
+        List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
+        if (null == tokens || tokens.isEmpty()) {
+            XxlJobHelper.log("定时获取快手数据异常:未获取到可用的token");
+            XxlJobHelper.handleFail();
+            return;
+        }
+        for (CtopOauthToken token : tokens) {
+            materialExecutorService.submit(new Runnable() {
+                @Override
+                public void run() {
+                    accountMaterialOverviewService.cleanAccountMaterialOverview(token.getAccountId(), yesterdayDate);
+                }
+            });
+        }
+
+
+    }
 
 
 
 
     @GetMapping("/cleanDate")
     @GetMapping("/cleanDate")
@@ -998,10 +1057,7 @@ public class TestController {
     }
     }
 
 
 
 
-    @Autowired
-    private IBytedancePlanDailyReportService planDailyReportService;
-    @Autowired
-    private ICtopOauthTokenService tokenService;
+
 
 
 
 
     @Autowired
     @Autowired

+ 6 - 0
jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java

@@ -75,7 +75,13 @@ public class SampleTest {
     @Test
     @Test
     public void loadBytedanceCreativeData() throws ParseException {
     public void loadBytedanceCreativeData() throws ParseException {
 
 
+        CtopOauthToken token = new CtopOauthToken();
+        token.setAccountId(9901975L);
+        token.setAccessToken("e609f9d3f56f3d6b4653f32c98a6d119");
         kuaiShouAccountMaterialOverviewService.cleanAccountMaterialOverview(9901975L, "2021-03-09");
         kuaiShouAccountMaterialOverviewService.cleanAccountMaterialOverview(9901975L, "2021-03-09");
+        SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
+        Date parse = simpleDateFormat.parse("2021-03-09");
+        reportDailyGroupService.getAdvertiserGroupReportDaily(token, parse, parse);
 
 
         //    1662234704806924
         //    1662234704806924
        /* CtopOauthToken token = new CtopOauthToken();
        /* CtopOauthToken token = new CtopOauthToken();

+ 2 - 1
module-job-kuaishou/src/main/java/cn/com/ctop/job/kuaishou/handler/KuaishouDingPanCleanJob.java

@@ -34,7 +34,8 @@ public class KuaishouDingPanCleanJob {
     @XxlJob("cleanNewlyData")
     @XxlJob("cleanNewlyData")
     public void cleanNewlyData() {
     public void cleanNewlyData() {
         String nowDate = DateUtils.getNowDate("yyyy-MM-dd");
         String nowDate = DateUtils.getNowDate("yyyy-MM-dd");
-        newlyService.cleanNewlyData(nowDate);
+        String yesterdayDate = DateUtils.getAnotherDay("yyyy-MM-dd", nowDate, -1);
+        newlyService.cleanNewlyData(yesterdayDate);
         XxlJobHelper.log("快手清洗上新计划数据完成");
         XxlJobHelper.log("快手清洗上新计划数据完成");
     }
     }
 
 

+ 1 - 1
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/mapper/KuaiShouGroupMapper.java

@@ -36,5 +36,5 @@ public interface KuaiShouGroupMapper extends BaseMapper<KuaiShouGroup> {
 
 
     Integer getNewDataByAccountIdAndDate(Long accountId, String statDate);
     Integer getNewDataByAccountIdAndDate(Long accountId, String statDate);
 
 
-    List<JSONObject> getUnitListByMapAndAccountId(@Param("unitIds") Map<Long, JSONObject> unitMap, @Param("accountId") Long accountId);
+    List<JSONObject> getUnitListByMapAndAccountId(@Param("unitIds") Map<Long, Long> unitMap, @Param("accountId") Long accountId);
 }
 }

+ 1 - 1
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/mapper/xml/KuaiShouGroupMapper.xml

@@ -153,7 +153,7 @@
     <select id="getUnitListByMapAndAccountId" resultType="com.alibaba.fastjson.JSONObject">
     <select id="getUnitListByMapAndAccountId" resultType="com.alibaba.fastjson.JSONObject">
         select
         select
         unit_id as unitId,
         unit_id as unitId,
-        put_status as putStutus,
+        put_status as unitStatus,
         bid_type as bidType,
         bid_type as bidType,
         (case when bid_type = 2
         (case when bid_type = 2
         then bid
         then bid

+ 1 - 1
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/impl/KuaiShouGroupServiceImpl.java

@@ -151,7 +151,7 @@ public class KuaiShouGroupServiceImpl extends ServiceImpl<KuaiShouGroupMapper, K
         if (Check.isNullMap(unitIdMap)) {
         if (Check.isNullMap(unitIdMap)) {
             return unitMap;
             return unitMap;
         }
         }
-        List<JSONObject> unitList = groupMapper.getUnitListByMapAndAccountId(unitMap, accountId);
+        List<JSONObject> unitList = groupMapper.getUnitListByMapAndAccountId(unitIdMap, accountId);
         if (Check.isNull(unitList)) {
         if (Check.isNull(unitList)) {
             return unitMap;
             return unitMap;
         }
         }

+ 2 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/entity/EtlKuaiShouGroupDailyReport.java

@@ -76,6 +76,8 @@ public class EtlKuaiShouGroupDailyReport {
     @Excel(name = "出价", width = 15)
     @Excel(name = "出价", width = 15)
     @ApiModelProperty(value = "出价")
     @ApiModelProperty(value = "出价")
     private Long bid;
     private Long bid;
+    private Integer bidType;
+    private Integer unitStatus;
     /**
     /**
      * 曝光数
      * 曝光数
      */
      */

+ 6 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/xml/EtlKuaiShouGroupDailyReportMapper.xml

@@ -9,6 +9,9 @@
         campaign_name,
         campaign_name,
         unit_id,
         unit_id,
         unit_name,
         unit_name,
+        unit_status,
+        bid_type,
+        bid,
         charge,
         charge,
         photo_show,
         photo_show,
         aclick,
         aclick,
@@ -33,6 +36,9 @@
             #{addList.campaignName},
             #{addList.campaignName},
             #{addList.unitId},
             #{addList.unitId},
             #{addList.unitName},
             #{addList.unitName},
+            #{addList.unitStatus},
+            #{addList.bidType},
+            #{addList.bid},
             #{addList.charge},
             #{addList.charge},
             #{addList.photoShow},
             #{addList.photoShow},
             #{addList.aclick},
             #{addList.aclick},

+ 13 - 3
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/service/impl/EtlKuaiShouGroupDailyReportServiceImpl.java

@@ -140,6 +140,7 @@ public class EtlKuaiShouGroupDailyReportServiceImpl extends ServiceImpl<EtlKuaiS
             addList.add(groupDailyReport);
             addList.add(groupDailyReport);
         }
         }
         if (!Check.isNull(addList)) {
         if (!Check.isNull(addList)) {
+            addList = getBid(accountId, addList);
             groupDailyReportMapper.replaceBatch(addList);
             groupDailyReportMapper.replaceBatch(addList);
         }
         }
     }
     }
@@ -152,10 +153,19 @@ public class EtlKuaiShouGroupDailyReportServiceImpl extends ServiceImpl<EtlKuaiS
         for (EtlKuaiShouGroupDailyReport groupDailyReport : addList) {
         for (EtlKuaiShouGroupDailyReport groupDailyReport : addList) {
             unitIdMap.put(groupDailyReport.getUnitId(), groupDailyReport.getUnitId());
             unitIdMap.put(groupDailyReport.getUnitId(), groupDailyReport.getUnitId());
         }
         }
-
         Map<Long, JSONObject> unitMap = groupService.getBidMapByMap(accountId, unitIdMap);
         Map<Long, JSONObject> unitMap = groupService.getBidMapByMap(accountId, unitIdMap);
-
-
+        if (Check.isNullMap(unitMap)) {
+            return addList;
+        }
+        for (EtlKuaiShouGroupDailyReport groupDailyReport : addList) {
+            JSONObject unitJson = unitMap.get(groupDailyReport.getUnitId());
+            if (Check.isNull(unitJson)) {
+                continue;
+            }
+            groupDailyReport.setUnitStatus(unitJson.getInteger("unitStatus"));
+            groupDailyReport.setBidType(unitJson.getInteger("bidType"));
+            groupDailyReport.setBid(unitJson.getLong("bid"));
+        }
 
 
         return addList;
         return addList;