Jelajahi Sumber

添加脚本库接口,优化数据清洗任务代码

syh 5 tahun lalu
induk
melakukan
dac2405aa8
16 mengubah file dengan 508 tambahan dan 79 penghapusan
  1. 248 0
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/ScriptTagInfoController.java
  2. 8 45
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/TestController.java
  3. 63 0
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/entity/ScriptTagInfo.java
  4. 59 0
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/BytedanceCreativeLoadJob.java
  5. 3 29
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/BytedancePlanDailyReportLoadJob.java
  6. 1 2
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/WatermarkVideoJob.java
  7. 15 0
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/mapper/ScriptTagInfoMapper.java
  8. 5 0
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/mapper/xml/ScriptTagInfoMapper.xml
  9. 15 0
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/service/IScriptTagInfoService.java
  10. 19 0
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/service/impl/ScriptTagInfoServiceImpl.java
  11. 4 0
      jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java
  12. 2 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/service/IAppiumJobService.java
  13. 48 3
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/service/impl/AppiumJobServiceImpl.java
  14. 5 0
      module-report/src/main/java/cn/com/ctop/bytedance/mapper/BytedancePlanDailyReportMapper.java
  15. 3 0
      module-report/src/main/java/cn/com/ctop/bytedance/service/IBytedancePlanDailyReportService.java
  16. 10 0
      module-report/src/main/java/cn/com/ctop/bytedance/service/impl/BytedancePlanDailyReportServiceImpl.java

+ 248 - 0
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/ScriptTagInfoController.java

@@ -0,0 +1,248 @@
+package org.jeecg.modules.ctop.controller;
+
+import com.alibaba.fastjson.JSON;
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import com.baomidou.mybatisplus.core.metadata.IPage;
+import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
+import lombok.extern.slf4j.Slf4j;
+import org.jeecg.common.api.vo.Result;
+import org.jeecg.common.aspect.annotation.AutoLog;
+import org.jeecg.common.system.query.QueryGenerator;
+import org.jeecg.common.util.oConvertUtils;
+import org.jeecg.modules.ctop.entity.ScriptTagInfo;
+import org.jeecg.modules.ctop.service.IScriptTagInfoService;
+import org.jeecgframework.poi.excel.ExcelImportUtil;
+import org.jeecgframework.poi.excel.def.NormalExcelConstants;
+import org.jeecgframework.poi.excel.entity.ExportParams;
+import org.jeecgframework.poi.excel.entity.ImportParams;
+import org.jeecgframework.poi.excel.view.JeecgEntityExcelView;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.*;
+import org.springframework.web.multipart.MultipartFile;
+import org.springframework.web.multipart.MultipartHttpServletRequest;
+import org.springframework.web.servlet.ModelAndView;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URLDecoder;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * 脚本库标签信息
+ *
+ * @author jeecg-boot
+ * @version V1.0
+ * @date 2020-02-27
+ */
+@Slf4j
+@Api(tags = "脚本库标签信息")
+@RestController
+@RequestMapping("/ctop/scriptTagInfo")
+public class ScriptTagInfoController {
+    @Autowired
+    private IScriptTagInfoService scriptTagInfoService;
+
+    /**
+     * 分页列表查询
+     *
+     * @param scriptTagInfo
+     * @param pageNo
+     * @param pageSize
+     * @param req
+     * @return
+     */
+    @AutoLog(value = "脚本库标签信息-分页列表查询")
+    @ApiOperation(value = "脚本库标签信息-分页列表查询", notes = "脚本库标签信息-分页列表查询")
+    @GetMapping(value = "/list")
+    public Result<IPage<ScriptTagInfo>> queryPageList(ScriptTagInfo scriptTagInfo,
+                                                      @RequestParam(name = "pageNo", defaultValue = "1") Integer pageNo,
+                                                      @RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize,
+                                                      HttpServletRequest req) {
+        Result<IPage<ScriptTagInfo>> result = new Result<IPage<ScriptTagInfo>>();
+        QueryWrapper<ScriptTagInfo> queryWrapper = QueryGenerator.initQueryWrapper(scriptTagInfo, req.getParameterMap());
+        Page<ScriptTagInfo> page = new Page<ScriptTagInfo>(pageNo, pageSize);
+        IPage<ScriptTagInfo> pageList = scriptTagInfoService.page(page, queryWrapper);
+        result.setSuccess(true);
+        result.setResult(pageList);
+        return result;
+    }
+
+    /**
+     * 添加
+     *
+     * @param scriptTagInfo
+     * @return
+     */
+    @AutoLog(value = "脚本库标签信息-添加")
+    @ApiOperation(value = "脚本库标签信息-添加", notes = "脚本库标签信息-添加")
+    @PostMapping(value = "/add")
+    public Result<ScriptTagInfo> add(@RequestBody ScriptTagInfo scriptTagInfo) {
+        Result<ScriptTagInfo> result = new Result<ScriptTagInfo>();
+        try {
+            scriptTagInfoService.save(scriptTagInfo);
+            result.success("添加成功!");
+        } catch (Exception e) {
+            log.error(e.getMessage(), e);
+            result.error500("操作失败");
+        }
+        return result;
+    }
+
+    /**
+     * 编辑
+     *
+     * @param scriptTagInfo
+     * @return
+     */
+    @AutoLog(value = "脚本库标签信息-编辑")
+    @ApiOperation(value = "脚本库标签信息-编辑", notes = "脚本库标签信息-编辑")
+    @PutMapping(value = "/edit")
+    public Result<ScriptTagInfo> edit(@RequestBody ScriptTagInfo scriptTagInfo) {
+        Result<ScriptTagInfo> result = new Result<ScriptTagInfo>();
+        ScriptTagInfo scriptTagInfoEntity = scriptTagInfoService.getById(scriptTagInfo.getId());
+        if (scriptTagInfoEntity == null) {
+            result.error500("未找到对应实体");
+        } else {
+            boolean ok = scriptTagInfoService.updateById(scriptTagInfo);
+            if (ok) {
+                result.success("修改成功!");
+            }
+        }
+
+        return result;
+    }
+
+    /**
+     * 通过id删除
+     *
+     * @param id
+     * @return
+     */
+    @AutoLog(value = "脚本库标签信息-通过id删除")
+    @ApiOperation(value = "脚本库标签信息-通过id删除", notes = "脚本库标签信息-通过id删除")
+    @DeleteMapping(value = "/delete")
+    public Result<?> delete(@RequestParam(name = "id", required = true) String id) {
+        try {
+            scriptTagInfoService.removeById(id);
+        } catch (Exception e) {
+            log.error("删除失败", e.getMessage());
+            return Result.error("删除失败!");
+        }
+        return Result.ok("删除成功!");
+    }
+
+    /**
+     * 批量删除
+     *
+     * @param ids
+     * @return
+     */
+    @AutoLog(value = "脚本库标签信息-批量删除")
+    @ApiOperation(value = "脚本库标签信息-批量删除", notes = "脚本库标签信息-批量删除")
+    @DeleteMapping(value = "/deleteBatch")
+    public Result<ScriptTagInfo> deleteBatch(@RequestParam(name = "ids", required = true) String ids) {
+        Result<ScriptTagInfo> result = new Result<ScriptTagInfo>();
+        if (ids == null || "".equals(ids.trim())) {
+            result.error500("参数不识别!");
+        } else {
+            this.scriptTagInfoService.removeByIds(Arrays.asList(ids.split(",")));
+            result.success("删除成功!");
+        }
+        return result;
+    }
+
+    /**
+     * 通过id查询
+     *
+     * @param id
+     * @return
+     */
+    @AutoLog(value = "脚本库标签信息-通过id查询")
+    @ApiOperation(value = "脚本库标签信息-通过id查询", notes = "脚本库标签信息-通过id查询")
+    @GetMapping(value = "/queryById")
+    public Result<ScriptTagInfo> queryById(@RequestParam(name = "id", required = true) String id) {
+        Result<ScriptTagInfo> result = new Result<ScriptTagInfo>();
+        ScriptTagInfo scriptTagInfo = scriptTagInfoService.getById(id);
+        if (scriptTagInfo == null) {
+            result.error500("未找到对应实体");
+        } else {
+            result.setResult(scriptTagInfo);
+            result.setSuccess(true);
+        }
+        return result;
+    }
+
+    /**
+     * 导出excel
+     *
+     * @param request
+     * @param response
+     */
+    @RequestMapping(value = "/exportXls")
+    public ModelAndView exportXls(HttpServletRequest request, HttpServletResponse response) {
+        // Step.1 组装查询条件
+        QueryWrapper<ScriptTagInfo> queryWrapper = null;
+        try {
+            String paramsStr = request.getParameter("paramsStr");
+            if (oConvertUtils.isNotEmpty(paramsStr)) {
+                String deString = URLDecoder.decode(paramsStr, "UTF-8");
+                ScriptTagInfo scriptTagInfo = JSON.parseObject(deString, ScriptTagInfo.class);
+                queryWrapper = QueryGenerator.initQueryWrapper(scriptTagInfo, request.getParameterMap());
+            }
+        } catch (UnsupportedEncodingException e) {
+            e.printStackTrace();
+        }
+
+        //Step.2 AutoPoi 导出Excel
+        ModelAndView mv = new ModelAndView(new JeecgEntityExcelView());
+        List<ScriptTagInfo> pageList = scriptTagInfoService.list(queryWrapper);
+        //导出文件名称
+        mv.addObject(NormalExcelConstants.FILE_NAME, "脚本库标签信息列表");
+        mv.addObject(NormalExcelConstants.CLASS, ScriptTagInfo.class);
+        mv.addObject(NormalExcelConstants.PARAMS, new ExportParams("脚本库标签信息列表数据", "导出人:Jeecg", "导出信息"));
+        mv.addObject(NormalExcelConstants.DATA_LIST, pageList);
+        return mv;
+    }
+
+    /**
+     * 通过excel导入数据
+     *
+     * @param request
+     * @param response
+     * @return
+     */
+    @RequestMapping(value = "/importExcel", method = RequestMethod.POST)
+    public Result<?> importExcel(HttpServletRequest request, HttpServletResponse response) {
+        MultipartHttpServletRequest multipartRequest = (MultipartHttpServletRequest) request;
+        Map<String, MultipartFile> fileMap = multipartRequest.getFileMap();
+        for (Map.Entry<String, MultipartFile> entity : fileMap.entrySet()) {
+            MultipartFile file = entity.getValue();
+            ImportParams params = new ImportParams();
+            params.setTitleRows(2);
+            params.setHeadRows(1);
+            params.setNeedSave(true);
+            try {
+                List<ScriptTagInfo> listScriptTagInfos = ExcelImportUtil.importExcel(file.getInputStream(), ScriptTagInfo.class, params);
+                scriptTagInfoService.saveBatch(listScriptTagInfos);
+                return Result.ok("文件导入成功!数据行数:" + listScriptTagInfos.size());
+            } catch (Exception e) {
+                log.error(e.getMessage(), e);
+                return Result.error("文件导入失败:" + e.getMessage());
+            } finally {
+                try {
+                    file.getInputStream().close();
+                } catch (IOException e) {
+                    e.printStackTrace();
+                }
+            }
+        }
+        return Result.ok("文件导入失败!");
+    }
+
+}

+ 8 - 45
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/TestController.java

@@ -1,7 +1,6 @@
 package org.jeecg.modules.ctop.controller;
 
 import cn.com.ctop.bytedance.entity.BytedancePlanDailyReport;
-import cn.com.ctop.bytedance.entity.BytedancePlanHourlyReport;
 import cn.com.ctop.bytedance.service.IBytedancePlanDailyReportService;
 import cn.com.ctop.bytedance.service.IBytedancePlanHourlyReportService;
 import cn.com.ctop.bytedance.service.IReportService;
@@ -93,10 +92,8 @@ public class TestController {
 
             for (int i = 0; i < 7; i++) {
                 Date getStartDate = DateUtils.addDay(endDate, -i);
-
                 //3:获取广告组信息数据
                 kuaishouInterfaceService.getAdvertiserGroupReportHourly(token, getStartDate, getStartDate);
-
             }
             for (int i = 0; i < 6; i++) {
                 Date getStartDate = new Date();
@@ -108,13 +105,9 @@ public class TestController {
                     getStartDate = DateUtils.addDay(endDate, -180 + i * 30);
                     getEndDate = DateUtils.addDay(endDate, -150 + i * 30);
                 }
-
                 //3:获取广告组信息数据
                 kuaishouInterfaceService.getAdvertiserGroupReportDaily(token, getStartDate, getEndDate);
-
             }
-
-
         }
         return "Success";
     }
@@ -473,11 +466,11 @@ public class TestController {
             e.printStackTrace();
         }
         //清洗关于缘多多数据
-        formatData(113, finalGetDate);
+        planDailyReportService.cleanYddData(113, finalGetDate);
         //清洗关于交友数据
-        formatData(112, finalGetDate);
+        planDailyReportService.cleanYddData(112, finalGetDate);
         //清洗关于附近约会数据
-        formatData(222, finalGetDate);
+        planDailyReportService.cleanYddData(222, finalGetDate);
         log.info("头条计划时报数据获取完成");
         Map<String, Object> result = new HashMap<>();
         ResultMapUtils.setResultMap(result, StatusCode.COMMON_SUCCESS);
@@ -487,41 +480,11 @@ public class TestController {
     @GetMapping("formatYzData")
     public Map<String, Object> formatYzData() {
         Map<String, Object> result = new HashMap<>();
-        formatYzData(151, new Date());
+        planDailyReportService.cleanYzData(151);
         ResultMapUtils.setResultMap(result, StatusCode.COMMON_SUCCESS);
         return result;
     }
 
-    private void formatYzData(int projectId, Date getDate) {
-        planDailyReportService.cleanYzData(projectId);
-    }
-
-    private void formatData(Integer projectId, Date finalGetDate) {
-        QueryWrapper<UserAllocation> queryWrapper = new QueryWrapper<>();
-        queryWrapper.eq("project_id", projectId);
-        List<UserAllocation> fjyhAllocations = userAllocationService.list(queryWrapper);
-        if (null != fjyhAllocations && fjyhAllocations.size() > 0) {
-            fjyhAllocations.forEach(allocation -> {
-                //根据accountId和时间,查询相关的时报报表信息
-                QueryWrapper<BytedancePlanHourlyReport> accountWrapper = new QueryWrapper<>();
-                SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
-                String dateString = dateFormat.format(finalGetDate);
-                accountWrapper.eq("advertiser_id", allocation.getAccountId())
-                        .eq("stat_datetime", dateString)
-                ;
-                List<BytedancePlanHourlyReport> bytedancePlanHourlyReports = planHourlyReportService.list(accountWrapper);
-                if (null != bytedancePlanHourlyReports && bytedancePlanHourlyReports.size() > 0) {
-                    bytedancePlanHourlyReports.forEach(report -> {
-                        String adName = report.getAdName();
-                        String[] tags = adName.split("-");
-                        report.setChannelCode(tags[0]);
-                        planHourlyReportService.updateById(report);
-                    });
-                }
-            });
-        }
-    }
-
     @Autowired
     private IBytedancePlanHourlyReportService planHourlyReportService;
 
@@ -542,7 +505,7 @@ public class TestController {
             executorService.submit(new Runnable() {
                 @Override
                 public void run() {
-                    //4: 获取广告创意信息数据
+                    // 获取广告创意信息数据
                     try {
                         advertiserDataService.getAdvertiserCreative(token, null);
                     } catch (Exception e) {
@@ -567,7 +530,7 @@ public class TestController {
     public Map<String, Object> loadCreaticeReportData() {
         Map<String, Object> result = new HashMap<>();
         List<CtopOauthToken> tokens = tokenService.getTokenListByType(CtopAdConstant.PLATFORM_TYPE_BYTEDANCE);
-        for (int i = 47; i < 180; i++) {
+        for (int i = 51; i < 180; i++) {
             Date getDate = DateUtils.addDay(new Date(), -i);
             log.info(DateUtils.formatDate(getDate) + "头条创意日报数据开始获取");
             //查询当日数据
@@ -575,13 +538,13 @@ public class TestController {
                 log.info("定时获取头条数据异常:为获取到可用的token");
                 continue;
             }
-            executorService = Executors.newFixedThreadPool(20);
+            executorService = Executors.newFixedThreadPool(10);
             countDownLatch = new CountDownLatch(tokens.size());
             tokens.forEach(token -> {
                 executorService.submit(new Runnable() {
                     @Override
                     public void run() {
-                        //4: 获取广告创意信息数据
+                        //获取广告创意日报信息数据
                         try {
                             reportService.getAdvertiserCreativeReport(token, getDate, getDate, CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
                         } catch (Exception e) {

+ 63 - 0
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/entity/ScriptTagInfo.java

@@ -0,0 +1,63 @@
+package org.jeecg.modules.ctop.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.experimental.Accessors;
+import org.jeecgframework.poi.excel.annotation.Excel;
+
+import java.util.Date;
+
+/**
+ * 脚本库标签信息
+ *
+ * @author jeecg-boot
+ * @version V1.0
+ * @date 2020-02-27
+ */
+@Data
+@TableName("ctop_script_tag_info")
+@EqualsAndHashCode(callSuper = false)
+@Accessors(chain = true)
+@ApiModel(value = "ctop_script_tag_info对象", description = "脚本库标签信息")
+public class ScriptTagInfo {
+
+    /**
+     * id
+     */
+    @TableId(type = IdType.UUID)
+    @ApiModelProperty(value = "id")
+    private Integer id;
+    /**
+     * 标签名称
+     */
+    @Excel(name = "标签名称", width = 15)
+    @ApiModelProperty(value = "标签名称")
+    private String name;
+    /**
+     * 父id
+     */
+    @Excel(name = "父id", width = 15)
+    @ApiModelProperty(value = "父id")
+    private Integer parentId;
+    /**
+     * enabled
+     */
+    @Excel(name = "enabled", width = 15)
+    @ApiModelProperty(value = "enabled")
+    private Integer enabled;
+    /**
+     * createTime
+     */
+    @ApiModelProperty(value = "createTime")
+    private Date createTime;
+    /**
+     * updateTime
+     */
+    @ApiModelProperty(value = "updateTime")
+    private Date updateTime;
+}

+ 59 - 0
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/BytedanceCreativeLoadJob.java

@@ -0,0 +1,59 @@
+package org.jeecg.modules.ctop.job;
+
+import cn.com.ctop.common.module.entity.CtopOauthToken;
+import cn.com.ctop.common.module.service.ICtopOauthTokenService;
+import cn.com.ctop.common.module.utils.CtopAdConstant;
+import cn.com.ctop.toutiao.service.IByteDanceAdvertiserDataService;
+import lombok.extern.slf4j.Slf4j;
+import org.jeecg.common.util.DateUtils;
+import org.quartz.Job;
+import org.quartz.JobExecutionContext;
+import org.quartz.JobExecutionException;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+@Slf4j
+public class BytedanceCreativeLoadJob implements Job {
+    @Autowired
+    private ICtopOauthTokenService tokenService;
+    static ExecutorService executorService = null;
+    //线程计数器
+    static CountDownLatch countDownLatch = null;
+    @Autowired
+    private IByteDanceAdvertiserDataService advertiserDataService;
+
+    @Override
+    public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
+        Date getDate = DateUtils.addDay(new Date(), -1);
+        List<CtopOauthToken> tokens = tokenService.getTokenListByType(CtopAdConstant.PLATFORM_TYPE_BYTEDANCE);
+        executorService = Executors.newFixedThreadPool(4);
+        countDownLatch = new CountDownLatch(tokens.size());
+        tokens.forEach(token -> {
+            executorService.submit(new Runnable() {
+                @Override
+                public void run() {
+                    try {
+                        String getDateString = DateUtils.formatDate(getDate);
+                        //获取全量视频素材数据
+                        advertiserDataService.getAdvertiserCreative(token, getDateString);
+                    } catch (Exception e) {
+                        e.printStackTrace();
+                    } finally {
+                        countDownLatch.countDown();
+                    }
+                }
+            });
+        });
+        try {
+            countDownLatch.await();
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+        }
+        log.info("创意物料数据同步完成");
+    }
+}

+ 3 - 29
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/BytedancePlanDailyReportLoadJob.java

@@ -46,9 +46,9 @@ public class BytedancePlanDailyReportLoadJob implements Job {
             reportService.getAdvertiserPlanReport(token, getDate, getDate, CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
         });
         formatZybData(215, getDate);
-        formatYddData(113, getDate);
-        formatYddData(112, getDate);
-        formatYddData(222, getDate);
+        planDailyReportService.cleanYddData(112, getDate);
+        planDailyReportService.cleanYddData(113, getDate);
+        planDailyReportService.cleanYddData(222, getDate);
         planDailyReportService.cleanYzData(151);
     }
 
@@ -100,32 +100,6 @@ public class BytedancePlanDailyReportLoadJob implements Job {
         }
     }
 
-    private void formatYddData(Integer projectId, Date getDate) {
-        //清洗关于缘多多数据
-        QueryWrapper<UserAllocation> dsjyWrapper = new QueryWrapper<>();
-        dsjyWrapper.eq("project_id", projectId);
-        List<UserAllocation> dsjyAllocations = userAllocationService.list(dsjyWrapper);
-        if (null != dsjyAllocations && dsjyAllocations.size() > 0) {
-            dsjyAllocations.forEach(allocation -> {
-                //根据accountId和时间,查询相关的日报表信息
-                SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
-                String dateString = dateFormat.format(getDate)+" 00:00:00";
-                QueryWrapper<BytedancePlanDailyReport> queryWrapper = new QueryWrapper<>();
-                queryWrapper.eq("advertiser_id", allocation.getAccountId())
-                        .eq("stat_datetime",dateString);
-                List<BytedancePlanDailyReport> bytedancePlanDailyReports = planDailyReportService.list(queryWrapper);
-                if (null != bytedancePlanDailyReports && bytedancePlanDailyReports.size() > 0) {
-                    bytedancePlanDailyReports.forEach(report -> {
-                        String adName = report.getAdName();
-                        String[] tags = adName.split("-");
-                        report.setChannelCode(tags[0]);
-                        planDailyReportService.updateById(report);
-                    });
-                }
-            });
-        }
-    }
-
     @Autowired
     private IBytedancePlanDailyReportService planDailyReportService;
 }

+ 1 - 2
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/WatermarkVideoJob.java

@@ -34,8 +34,7 @@ public class WatermarkVideoJob implements Job {
                 List<VideoWatermarkTask> list = videoWaterMarkService.list(queryWrapper);
                 if (list != null && list.size() > 0) {
                     for (VideoWatermarkTask videoWatermarkTask : list) {
-                        MpsUtils mpsUtils = new MpsUtils();
-                        QueryJobListResponse.Job job = mpsUtils.getJobStatus(videoWatermarkTask.getJobId());
+                        QueryJobListResponse.Job job = MpsUtils.getJobStatus(videoWatermarkTask.getJobId());
                         if (job.getState().equals("TranscodeSuccess")) {
                             String url = "https://ctop-media.oss-cn-beijing.aliyuncs.com/" + job.getOutput().getOutputFile().getObject();
 

+ 15 - 0
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/mapper/ScriptTagInfoMapper.java

@@ -0,0 +1,15 @@
+package org.jeecg.modules.ctop.mapper;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import org.jeecg.modules.ctop.entity.ScriptTagInfo;
+
+/**
+ * 脚本库标签信息
+ *
+ * @author: jeecg-boot
+ * @date: 2020-02-27
+ * @cersion: V1.0
+ */
+public interface ScriptTagInfoMapper extends BaseMapper<ScriptTagInfo> {
+
+}

+ 5 - 0
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/mapper/xml/ScriptTagInfoMapper.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="org.jeecg.modules.ctop.mapper.ScriptTagInfoMapper">
+
+</mapper>

+ 15 - 0
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/service/IScriptTagInfoService.java

@@ -0,0 +1,15 @@
+package org.jeecg.modules.ctop.service;
+
+import com.baomidou.mybatisplus.extension.service.IService;
+import org.jeecg.modules.ctop.entity.ScriptTagInfo;
+
+/**
+ * 脚本库标签信息
+ *
+ * @author jeecg-boot
+ * @version V1.0
+ * @date 2020-02-27
+ */
+public interface IScriptTagInfoService extends IService<ScriptTagInfo> {
+
+}

+ 19 - 0
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/service/impl/ScriptTagInfoServiceImpl.java

@@ -0,0 +1,19 @@
+package org.jeecg.modules.ctop.service.impl;
+
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.jeecg.modules.ctop.entity.ScriptTagInfo;
+import org.jeecg.modules.ctop.mapper.ScriptTagInfoMapper;
+import org.jeecg.modules.ctop.service.IScriptTagInfoService;
+import org.springframework.stereotype.Service;
+
+/**
+ * 脚本库标签信息
+ *
+ * @author jeecg-boot
+ * @version V1.0
+ * @date 2020-02-27
+ */
+@Service
+public class ScriptTagInfoServiceImpl extends ServiceImpl<ScriptTagInfoMapper, ScriptTagInfo> implements IScriptTagInfoService {
+
+}

+ 4 - 0
jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java

@@ -36,6 +36,10 @@ public class SampleTest {
     private IAppiumJobService jobService;
 
     @Test
+    public void getAgentData() {
+        jobService.loadTantanData();
+    }
+    @Test
     public void testMq() {
         Sender sender = new Sender();
         sender.send();

+ 2 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/service/IAppiumJobService.java

@@ -18,6 +18,8 @@ public interface IAppiumJobService extends IService<AppiumJob> {
 
     boolean bindCreative(String account, String password, String creativeName, String ksId);
 
+    void loadTantanData();
+
     boolean refreshWebPhone();
 
     void updateStatus(AppiumDevice appiumDevice, AppiumTaskLog log, Integer logStatus, Integer deviceStatus);

+ 48 - 3
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/service/impl/AppiumJobServiceImpl.java

@@ -10,12 +10,12 @@ import cn.com.ctop.crawler.modules.appium.mapper.AppiumTaskLogMapper;
 import cn.com.ctop.crawler.modules.appium.mapper.AppiumTaskMapper;
 import cn.com.ctop.crawler.modules.appium.service.IAppiumDeviceService;
 import cn.com.ctop.crawler.modules.appium.service.IAppiumJobService;
-import cn.com.ctop.crawler.modules.appium.service.IAppiumTaskService;
 import cn.com.ctop.crawler.modules.core.util.AppiumUtil;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import io.appium.java_client.android.AndroidDriver;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.http.impl.client.BasicCookieStore;
+import org.jeecg.common.util.DateUtils;
 import org.openqa.selenium.By;
 import org.openqa.selenium.WebDriver;
 import org.openqa.selenium.WebElement;
@@ -25,6 +25,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
+import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -152,8 +153,52 @@ public class AppiumJobServiceImpl extends ServiceImpl<AppiumJobMapper, AppiumJob
         return isBind;
     }
 
-    @Autowired
-    private IAppiumTaskService appiumTaskService;
+    public static final String GET_AGENT_DATA_URL = "https://marketing.tantanapp.com/platform/ad_statistic/channel_report/table?level=agent&interval=hour&";
+
+    @Override
+    public void loadTantanData() {
+        String url = "https://marketing.tantanapp.com/login";
+        System.getProperties().setProperty("webdriver.chrome.driver", chromeDriver);
+        ChromeOptions chromeOptions = new ChromeOptions();
+//        chromeOptions.addArguments("--headless");
+        chromeOptions.addArguments("--incognito");
+        chromeOptions.addArguments("--disable-gpu");
+        chromeOptions.addArguments("--no-sandbox");
+        chromeOptions.addArguments("--window-size=1920,1080");
+        chromeOptions.addArguments("--user-agent=" + HttpUtils2.USER_AGENT);
+        chromeOptions.setAcceptInsecureCerts(true);
+        WebDriver webDriver = new ChromeDriver(chromeOptions);
+        try {
+            HttpUtils2.cookieStore = new BasicCookieStore();
+            webDriver.manage().deleteAllCookies();
+            //获取登录页面
+            webDriver.get(url);
+            Thread.sleep(1000L);
+            WebElement accountElement = webDriver.findElement(By.xpath("//input[@class='form-control user_name']"));
+            accountElement.sendKeys("huichuang");
+            Thread.sleep(1000L);
+            WebElement passwordElement = webDriver.findElement(By.xpath("//input[@class='form-control user_password']"));
+            passwordElement.sendKeys("2PitTUSFG7AgBWVfbnIC");
+            WebElement loginElement = webDriver.findElement(By.xpath("//button[@class='btn btn-primary block full-width m-b user_button']"));
+            Thread.sleep(2000L);
+            //点击登录
+            loginElement.click();
+            Thread.sleep(2000L);
+            String nowDate = DateUtils.formatDate(new Date());
+            String stDate = nowDate + " 00:00:00";
+            String etDate = nowDate + " 23:59:59";
+            webDriver.get(GET_AGENT_DATA_URL + "st=" + stDate + "&et=" + etDate);
+            Thread.sleep(1000L);
+            String getData = webDriver.findElement(By.xpath("//pre[@style='word-wrap: break-word; white-space: pre-wrap;']")).getText();
+            System.out.println(getData);
+        } catch (Exception e) {
+            e.printStackTrace();
+            log.error("爬取代理商数据异常");
+        } finally {
+            webDriver.manage().deleteAllCookies();
+            webDriver.close();
+        }
+    }
 
     @Override
     public boolean refreshWebPhone() {

+ 5 - 0
module-report/src/main/java/cn/com/ctop/bytedance/mapper/BytedancePlanDailyReportMapper.java

@@ -27,4 +27,9 @@ public interface BytedancePlanDailyReportMapper extends BaseMapper<BytedancePlan
             "where advertiser_id in (select account_id from ctop_user_allocation where project_id = #{projectId}) " +
             "  and channel_code = '衍生计划' ")
     void cleanYzDataFuck(@Param(value = "projectId") Integer projectId);
+
+    @Update("update ctop_bytedance_report_plan_daily " +
+            "set channel_code = substring_index(ad_name, '-', 1) " +
+            "where advertiser_id in (select account_id from ctop_user_allocation where project_id = #{projectId}) and stat_datetime =#{date}")
+    void cleanYddData(@Param(value = "projectId") Integer projectId, @Param(value = "date") String date);
 }

+ 3 - 0
module-report/src/main/java/cn/com/ctop/bytedance/service/IBytedancePlanDailyReportService.java

@@ -3,6 +3,7 @@ package cn.com.ctop.bytedance.service;
 import cn.com.ctop.bytedance.entity.BytedancePlanDailyReport;
 import com.baomidou.mybatisplus.extension.service.IService;
 
+import java.util.Date;
 import java.util.List;
 
 /**
@@ -18,4 +19,6 @@ public interface IBytedancePlanDailyReportService extends IService<BytedancePlan
     void replaceBatch(List<BytedancePlanDailyReport> dailyReports);
 
     void cleanYzData(Integer projectId);
+
+    void cleanYddData(Integer projectId, Date getDate);
 }

+ 10 - 0
module-report/src/main/java/cn/com/ctop/bytedance/service/impl/BytedancePlanDailyReportServiceImpl.java

@@ -6,9 +6,11 @@ import cn.com.ctop.bytedance.service.IBytedancePlanDailyReportService;
 import cn.com.ctop.common.module.utils.StringUtils;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.jeecg.common.util.DateUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
+import java.util.Date;
 import java.util.List;
 
 /**
@@ -54,4 +56,12 @@ public class BytedancePlanDailyReportServiceImpl extends ServiceImpl<BytedancePl
         dailyReportMapper.cleanYzData(projectId);
         dailyReportMapper.cleanYzDataFuck(projectId);
     }
+
+    @Override
+    public void cleanYddData(Integer projectId, Date getDate) {
+        String dateString = DateUtils.formatDate(getDate) + " 00:00:00";
+        dailyReportMapper.cleanYddData(projectId, dateString);
+        dailyReportMapper.cleanYddData(projectId, dateString);
+        dailyReportMapper.cleanYddData(projectId, dateString);
+    }
 }