ソースを参照

修改数据清洗

syh 5 年 前
コミット
0b7c1e5ddb

+ 1 - 1
.gitignore

@@ -31,4 +31,4 @@ gen
 # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
 hs_err_pid*
 **/target/
-#application.yml
+#application.yml

+ 0 - 4
jeecg-boot-module-system/src/main/java/org/jeecg/JeecgApplication.java

@@ -2,7 +2,6 @@ package org.jeecg;
 
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.boot.SpringApplication;
-import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.context.ConfigurableApplicationContext;
 import org.springframework.context.annotation.Bean;
@@ -20,8 +19,6 @@ import java.net.UnknownHostException;
 @SpringBootApplication
 @ComponentScan(basePackages = {"cn.com.ctop", "org.jeecg"})
 @Configuration
-
-
 public class JeecgApplication {
     public static void main(String[] args) throws UnknownHostException {
         ConfigurableApplicationContext application = SpringApplication.run(JeecgApplication.class, args);
@@ -36,7 +33,6 @@ public class JeecgApplication {
                 "swagger-ui: \thttp://" + ip + ":" + port + path + "/swagger-ui.html\n\t" +
                 "Doc: \t\thttp://" + ip + ":" + port + path + "/doc.html\n" +
                 "----------------------------------------------------------");
-
     }
 
     @Bean

+ 147 - 4
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/TestController.java

@@ -1,12 +1,20 @@
 package org.jeecg.modules.ctop.controller;
 
+import cn.com.ctop.bytedance.entity.BytedancePlanDailyReport;
+import cn.com.ctop.bytedance.service.IBytedancePlanDailyReportService;
 import cn.com.ctop.common.module.entity.CtopOauthToken;
+import cn.com.ctop.common.module.entity.UserAllocation;
 import cn.com.ctop.common.module.mapper.CtopOauthTokenMapper;
+import cn.com.ctop.common.module.service.ICtopOauthTokenService;
+import cn.com.ctop.common.module.service.IUserAllocationService;
+import cn.com.ctop.common.module.utils.ResultMapUtils;
+import cn.com.ctop.common.module.utils.StatusCode;
 import cn.com.ctop.kuaishou.modules.batch.service.IKuaishouInterfaceService;
 import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouWebInterfaceService;
 import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
 import org.jeecg.common.api.vo.Result;
 import org.jeecg.common.util.DateUtils;
 import org.jeecg.modules.ctop.service.ICreateInternalService;
@@ -14,10 +22,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.*;
 
 import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
+import java.util.*;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 
@@ -26,6 +31,7 @@ import java.util.concurrent.Executors;
  */
 @RestController
 @RequestMapping("/test")
+@Slf4j
 public class TestController {
     @Autowired
     private ICreateInternalService createInternalService;
@@ -243,4 +249,141 @@ public class TestController {
         thread.start();
     }
 
+    @Autowired
+    private IUserAllocationService userAllocationService;
+
+    @Autowired
+    private IBytedancePlanDailyReportService planDailyReportService;
+    @Autowired
+    private ICtopOauthTokenService tokenService;
+
+    @GetMapping("formatZybData")
+    public Map<String, Object> formatZybData() {
+        //清洗关于作业帮数据
+        //1:查询作业帮相关的用户数据
+        QueryWrapper<UserAllocation> wrapper = new QueryWrapper<>();
+        wrapper.eq("project_id", 215);
+        List<UserAllocation> allocations = userAllocationService.list(wrapper);
+        if (null != allocations && allocations.size() > 0) {
+            allocations.forEach(allocation -> {
+                //根据accountId和时间,查询相关的日报表信息
+                QueryWrapper<BytedancePlanDailyReport> queryWrapper = new QueryWrapper<>();
+                SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
+//                String dateString = dateFormat.format(getDate)+" 00:00:00";
+                queryWrapper.eq("advertiser_id", allocation.getAccountId())
+//                        .eq("stat_datetime",dateString)
+                ;
+                List<BytedancePlanDailyReport> bytedancePlanDailyReports = planDailyReportService.list(queryWrapper);
+                if (null != bytedancePlanDailyReports && bytedancePlanDailyReports.size() > 0) {
+                    bytedancePlanDailyReports.forEach(report -> {
+                        String adName = report.getAdName();
+                        String[] tags = adName.split("-");
+                        report.setAdsense(tags[1]);
+                        report.setPlanCode(tags[2]);
+                        String materialName = tags[3];
+                        report.setMaterialName(materialName);
+                        if (materialName.contains("其他")) {
+                            report.setMaterialType("其他");
+                        } else if (materialName.contains("图片轮播")) {
+                            report.setMaterialType("图片轮播");
+                        } else if (materialName.contains("大字报")) {
+                            report.setMaterialType("大字报");
+                        } else if (materialName.contains("文字动画")) {
+                            report.setMaterialType("文字动画");
+                        } else if (materialName.contains("采访形式")) {
+                            report.setMaterialType("采访形式");
+                        } else if (materialName.contains("剧情")) {
+                            report.setMaterialType("剧情");
+                        } else if (materialName.contains("口播")) {
+                            report.setMaterialType("口播");
+                        } else {
+                            report.setMaterialType("其他");
+                        }
+                        planDailyReportService.updateById(report);
+                    });
+                }
+            });
+        }
+        Map<String, Object> result = new HashMap<>();
+        ResultMapUtils.setResultMap(result, StatusCode.COMMON_SUCCESS);
+        return result;
+    }
+
+    @GetMapping("formatYddData")
+    public Map<String, Object> formatYddData() throws Exception {
+        Map<String, Object> result = new HashMap<>();
+        //清洗关于缘多多数据
+        //1:查询缘多多相关的用户数据
+        QueryWrapper<UserAllocation> yddWrapper = new QueryWrapper<>();
+        yddWrapper.eq("project_id", 113);
+        List<UserAllocation> yddallocations = userAllocationService.list(yddWrapper);
+        if (null != yddallocations && yddallocations.size() > 0) {
+            yddallocations.forEach(allocation -> {
+                //根据accountId和时间,查询相关的日报表信息
+                QueryWrapper<BytedancePlanDailyReport> queryWrapper = new QueryWrapper<>();
+                queryWrapper.eq("advertiser_id", allocation.getAccountId())
+                //.eq("stat_datetime",dateString)
+                ;
+                List<BytedancePlanDailyReport> bytedancePlanDailyReports = planDailyReportService.list(queryWrapper);
+                if (null != bytedancePlanDailyReports && bytedancePlanDailyReports.size() > 0) {
+                    bytedancePlanDailyReports.forEach(report -> {
+                        String adName = report.getAdName();
+                        String[] tags = adName.split("-");
+                        report.setChannelCode(tags[0]);
+                        planDailyReportService.updateById(report);
+                    });
+                }
+            });
+        }
+
+        //清洗关于缘多多数据
+        //1:查询缘多多相关的用户数据
+        QueryWrapper<UserAllocation> dsjyWrapper = new QueryWrapper<>();
+        yddWrapper.eq("project_id", 112);
+        List<UserAllocation> dsjyAllocations = userAllocationService.list(dsjyWrapper);
+        if (null != dsjyAllocations && dsjyAllocations.size() > 0) {
+            dsjyAllocations.forEach(allocation -> {
+                //根据accountId和时间,查询相关的日报表信息
+                QueryWrapper<BytedancePlanDailyReport> queryWrapper = new QueryWrapper<>();
+                queryWrapper.eq("advertiser_id", allocation.getAccountId())
+                //.eq("stat_datetime",dateString)
+                ;
+                List<BytedancePlanDailyReport> bytedancePlanDailyReports = planDailyReportService.list(queryWrapper);
+                if (null != bytedancePlanDailyReports && bytedancePlanDailyReports.size() > 0) {
+                    bytedancePlanDailyReports.forEach(report -> {
+                        String adName = report.getAdName();
+                        String[] tags = adName.split("-");
+                        report.setChannelCode(tags[0]);
+                        planDailyReportService.updateById(report);
+                    });
+                }
+            });
+        }
+
+        //清洗关于缘多多数据
+        //1:查询缘多多相关的用户数据
+        QueryWrapper<UserAllocation> fjyhWrapper = new QueryWrapper<>();
+        yddWrapper.eq("project_id", 222);
+        List<UserAllocation> fjyhAllocations = userAllocationService.list(fjyhWrapper);
+        if (null != fjyhAllocations && fjyhAllocations.size() > 0) {
+            fjyhAllocations.forEach(allocation -> {
+                //根据accountId和时间,查询相关的日报表信息
+                QueryWrapper<BytedancePlanDailyReport> queryWrapper = new QueryWrapper<>();
+                queryWrapper.eq("advertiser_id", allocation.getAccountId())
+                //.eq("stat_datetime",dateString)
+                ;
+                List<BytedancePlanDailyReport> bytedancePlanDailyReports = planDailyReportService.list(queryWrapper);
+                if (null != bytedancePlanDailyReports && bytedancePlanDailyReports.size() > 0) {
+                    bytedancePlanDailyReports.forEach(report -> {
+                        String adName = report.getAdName();
+                        String[] tags = adName.split("-");
+                        report.setChannelCode(tags[0]);
+                        planDailyReportService.updateById(report);
+                    });
+                }
+            });
+        }
+        ResultMapUtils.setResultMap(result, StatusCode.COMMON_SUCCESS);
+        return result;
+    }
 }

+ 57 - 8
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/BytedancePlanDailyReportLoadJob.java

@@ -63,20 +63,21 @@ public class BytedancePlanDailyReportLoadJob implements Job {
                         String[] tags = adName.split("-");
                         report.setAdsense(tags[1]);
                         report.setPlanCode(tags[2]);
-                        String tag = tags[3];
-                        if(tag.contains("其他")){
+                        String materialName = tags[3];
+                        report.setMaterialName(materialName);
+                        if (materialName.contains("其他")) {
                             report.setMaterialType("其他");
-                        }else if (tag.contains("图片轮播")){
+                        } else if (materialName.contains("图片轮播")) {
                             report.setMaterialType("图片轮播");
-                        }else if(tag.contains("大字报")){
+                        } else if (materialName.contains("大字报")) {
                             report.setMaterialType("大字报");
-                        }else if(tag.contains("文字动画")){
+                        } else if (materialName.contains("文字动画")) {
                             report.setMaterialType("文字动画");
-                        }else if(tag.contains("采访形式")){
+                        } else if (materialName.contains("采访形式")) {
                             report.setMaterialType("采访形式");
-                        }else if(tag.contains("剧情")){
+                        } else if (materialName.contains("剧情")) {
                             report.setMaterialType("剧情");
-                        }else if(tag.contains("口播")){
+                        } else if (materialName.contains("口播")) {
                             report.setMaterialType("口播");
                         }else{
                             report.setMaterialType("其他");
@@ -112,6 +113,54 @@ public class BytedancePlanDailyReportLoadJob implements Job {
                 }
             });
         }
+
+        //清洗关于缘多多数据
+        //1:查询缘多多相关的用户数据
+        QueryWrapper<UserAllocation> dsjyWrapper = new QueryWrapper<>();
+        yddWrapper.eq("project_id", 112);
+        List<UserAllocation> dsjyAllocations = userAllocationService.list(dsjyWrapper);
+        if (null != dsjyAllocations && dsjyAllocations.size() > 0) {
+            dsjyAllocations.forEach(allocation -> {
+                //根据accountId和时间,查询相关的日报表信息
+                QueryWrapper<BytedancePlanDailyReport> queryWrapper = new QueryWrapper<>();
+                queryWrapper.eq("advertiser_id", allocation.getAccountId())
+                //.eq("stat_datetime",dateString)
+                ;
+                List<BytedancePlanDailyReport> bytedancePlanDailyReports = planDailyReportService.list(queryWrapper);
+                if (null != bytedancePlanDailyReports && bytedancePlanDailyReports.size() > 0) {
+                    bytedancePlanDailyReports.forEach(report -> {
+                        String adName = report.getAdName();
+                        String[] tags = adName.split("-");
+                        report.setChannelCode(tags[0]);
+                        planDailyReportService.updateById(report);
+                    });
+                }
+            });
+        }
+
+        //清洗关于附近约会数据
+        //1:查询附近约会相关的用户数据
+        QueryWrapper<UserAllocation> fjyhWrapper = new QueryWrapper<>();
+        yddWrapper.eq("project_id", 222);
+        List<UserAllocation> fjyhAllocations = userAllocationService.list(fjyhWrapper);
+        if (null != fjyhAllocations && fjyhAllocations.size() > 0) {
+            fjyhAllocations.forEach(allocation -> {
+                //根据accountId和时间,查询相关的日报表信息
+                QueryWrapper<BytedancePlanDailyReport> queryWrapper = new QueryWrapper<>();
+                queryWrapper.eq("advertiser_id", allocation.getAccountId())
+                //.eq("stat_datetime",dateString)
+                ;
+                List<BytedancePlanDailyReport> bytedancePlanDailyReports = planDailyReportService.list(queryWrapper);
+                if (null != bytedancePlanDailyReports && bytedancePlanDailyReports.size() > 0) {
+                    bytedancePlanDailyReports.forEach(report -> {
+                        String adName = report.getAdName();
+                        String[] tags = adName.split("-");
+                        report.setChannelCode(tags[0]);
+                        planDailyReportService.updateById(report);
+                    });
+                }
+            });
+        }
     }
 
     @Autowired

+ 3 - 3
jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java

@@ -55,9 +55,9 @@ public class SampleTest {
             log.info("定时获取头条数据异常:为获取到可用的token");
             return;
         }
-        tokens.forEach(token -> {
-            reportService.getAdvertiserPlanReport(token,getDate,getDate,CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
-        });
+//        tokens.forEach(token -> {
+//            reportService.getAdvertiserPlanReport(token,getDate,getDate,CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
+//        });
 
         //清洗关于缘多多数据
         //1:查询缘多多相关的用户数据

+ 5 - 0
module-report/src/main/java/cn/com/ctop/bytedance/entity/BytedancePlanDailyReport.java

@@ -415,6 +415,11 @@ public class BytedancePlanDailyReport {
      */
     private String channelCode;
 
+    /**
+     * 素材名称
+     */
+    private String materialName;
+
     public BytedancePlanDailyReport(JSONObject data, Long setAdvertiserId) {
         Long playFeedBreak75 = data.getLong("play_75_feed_break");
         if (null != playFeedBreak75) {

+ 0 - 5
pom.xml

@@ -393,11 +393,6 @@
             <artifactId>javacv-platform</artifactId>
             <version>1.4.2</version>
         </dependency>
-        <!--<dependency>
-            <groupId>org.bytedeco</groupId>
-            <artifactId>javacv-platform</artifactId>
-            <version>1.4.2</version>
-        </dependency>-->
     </dependencies>
 
     <dependencyManagement>