Browse Source

Merge branch 'master' into test_performance2

sunzhen 5 years ago
parent
commit
a182ac997a
60 changed files with 1335 additions and 113 deletions
  1. 20 4
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/AdvertiserController.java
  2. 164 0
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/BytedanceFundDailyController.java
  3. 55 0
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/CreativeTitleController.java
  4. 135 0
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/entity/BytedanceFundDaily.java
  5. 70 3
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/DataeyeJob.java
  6. 15 0
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/mapper/BytedanceFundDailyMapper.java
  7. 5 0
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/mapper/xml/BytedanceFundDailyMapper.xml
  8. 2 1
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/service/IAdvertiserService.java
  9. 16 0
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/service/IBytedanceFundDailyService.java
  10. 11 0
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/service/ICreativeTitleService.java
  11. 11 1
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/service/impl/AdvertiserServiceImpl.java
  12. 60 0
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/service/impl/BytedanceFundDailyServiceImpl.java
  13. 81 0
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/service/impl/CreativeTitleServiceImpl.java
  14. 3 3
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/excelutil/entity/KuaishouXxlEntity.java
  15. 11 13
      jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java
  16. 11 7
      module-common/src/main/java/cn/com/ctop/common/module/utils/HttpUtils.java
  17. 8 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeAdArtStyle.java
  18. 1 1
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeCompany.java
  19. 15 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeCompanyProduct.java
  20. 12 10
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeMedia.java
  21. 15 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeMediaCompany.java
  22. 15 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeMediaProduct.java
  23. 8 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyePlayRules.java
  24. 10 4
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeProduct.java
  25. 15 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeProductAdArtStyle.java
  26. 15 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeProductPlayRule.java
  27. 15 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeProductTheme.java
  28. 8 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeThemes.java
  29. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyeAdArtStyleMapper.java
  30. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyeCompanyProductMapper.java
  31. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyeMediaMapper.java
  32. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyePlayRulesMapper.java
  33. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyeProductAdArtStyleMapper.java
  34. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyeProductMapper.java
  35. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyeProductPlayRuleMapper.java
  36. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyeProductThemeMapper.java
  37. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyeThemesMapper.java
  38. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeAdArtStyleService.java
  39. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeCompanyProductService.java
  40. 4 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeLabelService.java
  41. 8 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeMediaService.java
  42. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyePlayRulesService.java
  43. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeProductAdArtStyleService.java
  44. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeProductPlayRuleService.java
  45. 5 1
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeProductService.java
  46. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeProductThemeService.java
  47. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeThemesService.java
  48. 11 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeAdArtStyleServiceImpl.java
  49. 11 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeCompanyProductServiceImpl.java
  50. 30 15
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeCompanyServiceImpl.java
  51. 62 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeLabelServiceImpl.java
  52. 73 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeMediaServiceImpl.java
  53. 11 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyePlayRulesServiceImpl.java
  54. 13 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeProductAdArtStyleServiceImpl.java
  55. 11 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeProductPlayRuleServiceImpl.java
  56. 107 2
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeProductServiceImpl.java
  57. 11 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeProductThemeServiceImpl.java
  58. 12 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeThemesServiceImpl.java
  59. 1 1
      module-report/src/main/java/cn/com/ctop/bytedance/mapper/xml/KuaiShouYiCheReportMapper.xml
  60. 46 47
      module-report/src/main/java/cn/com/ctop/bytedance/mapper/xml/MaterialReportMapper.xml

+ 20 - 4
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/AdvertiserController.java

@@ -416,12 +416,20 @@ public class AdvertiserController {
         if (null == systemType || systemType.trim().equals("")) {
             systemType = null;
         }
-        SheetInfoVo totalSheetVo = advertiserService.getXxlStatisticInfo(projectId, systemType, null, null, "汇总表");
+        SheetInfoVo totalSheetVo = advertiserService.getXxlStatisticInfo(projectId, systemType, null, "汇总表");
         sheets.add(totalSheetVo);
+        if(null == systemType){
+            List<SheetInfoVo>vos = advertiserService.getSystemTypeXxlStatisticInfo(projectId);
+            if(null!=vos&&vos.size()>0){
+                for(SheetInfoVo vo:vos){
+                    sheets.add(vo);
+                }
+            }
+        }
         List<UserAllocation> allocations = userAllocationService.getByProjectId(projectId, systemType);
         if (null != allocations && allocations.size() > 0) {
             for (UserAllocation allocation : allocations) {
-                SheetInfoVo vo = advertiserService.getXxlStatisticInfo(projectId, systemType, allocation.getAccountId(), "state_date", allocation.getAuthName());
+                SheetInfoVo vo = advertiserService.getXxlStatisticInfo(projectId, systemType, allocation.getAccountId(), allocation.getAuthName());
                 sheets.add(vo);
             }
         }
@@ -443,12 +451,20 @@ public class AdvertiserController {
             systemType = null;
         }
         //查询总的消耗信息
-        SheetInfoVo totalSheetVo = advertiserService.getXxlStatisticInfo(projectId, systemType, null, null, "汇总表");
+        SheetInfoVo totalSheetVo = advertiserService.getXxlStatisticInfo(projectId, systemType, null, "汇总表");
         sheets.add(totalSheetVo);
+        if(null == systemType){
+            List<SheetInfoVo>vos = advertiserService.getSystemTypeXxlStatisticInfo(projectId);
+            if(null!=vos&&vos.size()>0){
+                for(SheetInfoVo vo:vos){
+                    sheets.add(vo);
+                }
+            }
+        }
         List<UserAllocation> allocations = userAllocationService.getByProjectId(projectId, systemType);
         if (null != allocations && allocations.size() > 0) {
             for (UserAllocation allocation : allocations) {
-                SheetInfoVo vo = advertiserService.getXxlStatisticInfo(projectId, systemType, allocation.getAccountId(), "state_date", allocation.getAuthName());
+                SheetInfoVo vo = advertiserService.getXxlStatisticInfo(projectId, systemType, allocation.getAccountId(), allocation.getAuthName());
                 sheets.add(vo);
             }
         }

+ 164 - 0
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/BytedanceFundDailyController.java

@@ -0,0 +1,164 @@
+package org.jeecg.modules.ctop.controller;
+
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import com.baomidou.mybatisplus.core.metadata.IPage;
+import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
+import lombok.extern.slf4j.Slf4j;
+import org.jeecg.common.api.vo.Result;
+import org.jeecg.common.aspect.annotation.AutoLog;
+import org.jeecg.common.system.query.QueryGenerator;
+import org.jeecg.modules.ctop.entity.BytedanceFundDaily;
+import org.jeecg.modules.ctop.service.IBytedanceFundDailyService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.*;
+
+import javax.servlet.http.HttpServletRequest;
+import java.util.Arrays;
+
+/**
+ * 账号日流水信息
+ *
+ * @author jeecg-boot
+ * @version V1.0
+ * @date 2020-04-08
+ */
+@Slf4j
+@Api(tags = "账号日流水信息")
+@RestController
+@RequestMapping("/ctop/bytedanceFundDaily")
+public class BytedanceFundDailyController {
+    @Autowired
+    private IBytedanceFundDailyService bytedanceFundDailyService;
+
+    /**
+     * 分页列表查询
+     *
+     * @param bytedanceFundDaily
+     * @param pageNo
+     * @param pageSize
+     * @param req
+     * @return
+     */
+    @AutoLog(value = "账号日流水信息-分页列表查询")
+    @ApiOperation(value = "账号日流水信息-分页列表查询", notes = "账号日流水信息-分页列表查询")
+    @GetMapping(value = "/list")
+    public Result<IPage<BytedanceFundDaily>> queryPageList(BytedanceFundDaily bytedanceFundDaily,
+                                                           @RequestParam(name = "pageNo", defaultValue = "1") Integer pageNo,
+                                                           @RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize,
+                                                           HttpServletRequest req) {
+        Result<IPage<BytedanceFundDaily>> result = new Result<IPage<BytedanceFundDaily>>();
+        QueryWrapper<BytedanceFundDaily> queryWrapper = QueryGenerator.initQueryWrapper(bytedanceFundDaily, req.getParameterMap());
+        Page<BytedanceFundDaily> page = new Page<BytedanceFundDaily>(pageNo, pageSize);
+        IPage<BytedanceFundDaily> pageList = bytedanceFundDailyService.page(page, queryWrapper);
+        result.setSuccess(true);
+        result.setResult(pageList);
+        return result;
+    }
+
+    /**
+     * 添加
+     *
+     * @param bytedanceFundDaily
+     * @return
+     */
+    @AutoLog(value = "账号日流水信息-添加")
+    @ApiOperation(value = "账号日流水信息-添加", notes = "账号日流水信息-添加")
+    @PostMapping(value = "/add")
+    public Result<BytedanceFundDaily> add(@RequestBody BytedanceFundDaily bytedanceFundDaily) {
+        Result<BytedanceFundDaily> result = new Result<BytedanceFundDaily>();
+        try {
+            bytedanceFundDailyService.save(bytedanceFundDaily);
+            result.success("添加成功!");
+        } catch (Exception e) {
+            log.error(e.getMessage(), e);
+            result.error500("操作失败");
+        }
+        return result;
+    }
+
+    /**
+     * 编辑
+     *
+     * @param bytedanceFundDaily
+     * @return
+     */
+    @AutoLog(value = "账号日流水信息-编辑")
+    @ApiOperation(value = "账号日流水信息-编辑", notes = "账号日流水信息-编辑")
+    @PutMapping(value = "/edit")
+    public Result<BytedanceFundDaily> edit(@RequestBody BytedanceFundDaily bytedanceFundDaily) {
+        Result<BytedanceFundDaily> result = new Result<BytedanceFundDaily>();
+        BytedanceFundDaily bytedanceFundDailyEntity = bytedanceFundDailyService.getById(bytedanceFundDaily.getId());
+        if (bytedanceFundDailyEntity == null) {
+            result.error500("未找到对应实体");
+        } else {
+            boolean ok = bytedanceFundDailyService.updateById(bytedanceFundDaily);
+            if (ok) {
+                result.success("修改成功!");
+            }
+        }
+
+        return result;
+    }
+
+    /**
+     * 通过id删除
+     *
+     * @param id
+     * @return
+     */
+    @AutoLog(value = "账号日流水信息-通过id删除")
+    @ApiOperation(value = "账号日流水信息-通过id删除", notes = "账号日流水信息-通过id删除")
+    @DeleteMapping(value = "/delete")
+    public Result<?> delete(@RequestParam(name = "id", required = true) String id) {
+        try {
+            bytedanceFundDailyService.removeById(id);
+        } catch (Exception e) {
+            log.error("删除失败", e.getMessage());
+            return Result.error("删除失败!");
+        }
+        return Result.ok("删除成功!");
+    }
+
+    /**
+     * 批量删除
+     *
+     * @param ids
+     * @return
+     */
+    @AutoLog(value = "账号日流水信息-批量删除")
+    @ApiOperation(value = "账号日流水信息-批量删除", notes = "账号日流水信息-批量删除")
+    @DeleteMapping(value = "/deleteBatch")
+    public Result<BytedanceFundDaily> deleteBatch(@RequestParam(name = "ids", required = true) String ids) {
+        Result<BytedanceFundDaily> result = new Result<BytedanceFundDaily>();
+        if (ids == null || "".equals(ids.trim())) {
+            result.error500("参数不识别!");
+        } else {
+            this.bytedanceFundDailyService.removeByIds(Arrays.asList(ids.split(",")));
+            result.success("删除成功!");
+        }
+        return result;
+    }
+
+    /**
+     * 通过id查询
+     *
+     * @param id
+     * @return
+     */
+    @AutoLog(value = "账号日流水信息-通过id查询")
+    @ApiOperation(value = "账号日流水信息-通过id查询", notes = "账号日流水信息-通过id查询")
+    @GetMapping(value = "/queryById")
+    public Result<BytedanceFundDaily> queryById(@RequestParam(name = "id", required = true) String id) {
+        Result<BytedanceFundDaily> result = new Result<BytedanceFundDaily>();
+        BytedanceFundDaily bytedanceFundDaily = bytedanceFundDailyService.getById(id);
+        if (bytedanceFundDaily == null) {
+            result.error500("未找到对应实体");
+        } else {
+            result.setResult(bytedanceFundDaily);
+            result.setSuccess(true);
+        }
+        return result;
+    }
+}

+ 55 - 0
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/CreativeTitleController.java

@@ -0,0 +1,55 @@
+package org.jeecg.modules.ctop.controller;
+
+import com.alibaba.fastjson.JSONArray;
+import org.jeecg.common.api.vo.Result;
+import org.jeecg.modules.ctop.service.ICreativeTitleService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+@RestController
+@RequestMapping("/ctop/creativeTitle/")
+public class CreativeTitleController {
+    @Autowired
+    private ICreativeTitleService creativeTitleService;
+
+
+    @GetMapping("getTitleLabel")
+    public Result<JSONArray> getTitleLabel() {
+        Result<JSONArray> result = new Result<>();
+        JSONArray labelArr = creativeTitleService.getTitleLabel();
+        result.setSuccess(true);
+        result.setResult(labelArr);
+        return result;
+
+
+    }
+
+
+    @GetMapping("titleSuggestion")
+    public Result<JSONArray> titleSuggestion(String firstCategory, String secondCategory, String keyword) {
+        Result<JSONArray> result = new Result<>();
+        JSONArray labelArr = creativeTitleService.titleSuggestion(firstCategory, secondCategory, keyword);
+        result.setSuccess(true);
+        result.setResult(labelArr);
+        return result;
+
+
+    }
+
+
+
+    @GetMapping("titleSearch")
+    public Result<JSONArray> titleSearch(String firstCategory, String secondCategory, String keywords) {
+        Result<JSONArray> result = new Result<>();
+        JSONArray labelArr = creativeTitleService.titleSearch(firstCategory, secondCategory, keywords);
+        result.setSuccess(true);
+        result.setResult(labelArr);
+        return result;
+
+
+    }
+
+
+}

+ 135 - 0
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/entity/BytedanceFundDaily.java

@@ -0,0 +1,135 @@
+package org.jeecg.modules.ctop.entity;
+
+import cn.com.ctop.common.module.entity.CtopOauthToken;
+import com.alibaba.fastjson.JSONObject;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.experimental.Accessors;
+import org.jeecgframework.poi.excel.annotation.Excel;
+
+import java.util.Date;
+
+/**
+ * 账号日流水信息
+ *
+ * @author jeecg-boot
+ * @version V1.0
+ * @date 2020-04-08
+ */
+@Data
+@TableName("ctop_bytedance_fund_daily")
+@EqualsAndHashCode(callSuper = false)
+@Accessors(chain = true)
+@ApiModel(value = "ctop_bytedance_fund_daily对象", description = "账号日流水信息")
+public class BytedanceFundDaily {
+
+    /**
+     * id
+     */
+    @TableId
+    @ApiModelProperty(value = "id")
+    private String id;
+    /**
+     * 广告主id
+     */
+    @Excel(name = "广告主id", width = 15)
+    @ApiModelProperty(value = "广告主id")
+    private Long advertiserId;
+    /**
+     * 日期
+     */
+    @Excel(name = "日期", width = 15)
+    @ApiModelProperty(value = "日期")
+    private String date;
+    /**
+     * 日终结于
+     */
+    @Excel(name = "日终结余", width = 15)
+    @ApiModelProperty(value = "日终结于")
+    private java.math.BigDecimal balance;
+    /**
+     * 现金支出
+     */
+    @Excel(name = "现金支出", width = 15)
+    @ApiModelProperty(value = "现金支出")
+    private java.math.BigDecimal cashCost;
+    /**
+     * 总支出
+     */
+    @Excel(name = "总支出", width = 15)
+    @ApiModelProperty(value = "总支出")
+    private java.math.BigDecimal cost;
+    /**
+     * 冻结
+     */
+    @Excel(name = "冻结", width = 15)
+    @ApiModelProperty(value = "冻结")
+    private java.math.BigDecimal frozen;
+    /**
+     * 总存入
+     */
+    @Excel(name = "总存入", width = 15)
+    @ApiModelProperty(value = "总存入")
+    private java.math.BigDecimal income;
+    /**
+     * 赠款支出
+     */
+    @Excel(name = "赠款支出", width = 15)
+    @ApiModelProperty(value = "赠款支出")
+    private java.math.BigDecimal rewardCost;
+    /**
+     * 返货支出
+     */
+    @Excel(name = "返货支出", width = 15)
+    @ApiModelProperty(value = "返货支出")
+    private java.math.BigDecimal returnGoodsCos;
+    /**
+     * 总转入
+     */
+    @Excel(name = "总转入", width = 15)
+    @ApiModelProperty(value = "总转入")
+    private java.math.BigDecimal transferIn;
+    /**
+     * 总转出
+     */
+    @Excel(name = "总转出", width = 15)
+    @ApiModelProperty(value = "总转出")
+    private java.math.BigDecimal transferOut;
+    /**
+     * createTime
+     */
+    @ApiModelProperty(value = "createTime")
+    private Date createTime;
+    /**
+     * updateTime
+     */
+    @ApiModelProperty(value = "updateTime")
+    private Date updateTime;
+
+    public BytedanceFundDaily(JSONObject data, CtopOauthToken token) {
+        System.out.println(data.toJSONString());
+        this.advertiserId = token.getAccountId();
+        this.date = data.getString("date");
+        this.id = this.advertiserId + "_" + this.date;
+        this.balance = data.getBigDecimal("balance");
+        this.cashCost = data.getBigDecimal("cash_cost");
+        this.cost = data.getBigDecimal("cost");
+        this.frozen = data.getBigDecimal("frozen");
+        this.income = data.getBigDecimal("income");
+        this.rewardCost = data.getBigDecimal("reward_cost");
+        this.returnGoodsCos = data.getBigDecimal("return_goods_cost");
+        this.transferIn = data.getBigDecimal("transfer_in");
+        this.transferOut = data.getBigDecimal("transfer_out");
+        this.createTime = new Date();
+        this.updateTime = new Date();
+    }
+
+    public BytedanceFundDaily() {
+        this.createTime = new Date();
+        this.updateTime = new Date();
+    }
+}

+ 70 - 3
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/DataeyeJob.java

@@ -1,20 +1,87 @@
 package org.jeecg.modules.ctop.job;
 
+import cn.com.ctop.common.module.utils.HttpUtils;
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeCompany;
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeProduct;
 import cn.com.ctop.crawler.modules.dataeye.service.DataeyeCompanyService;
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyeMediaService;
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyeProductService;
+import cn.com.ctop.crawler.modules.dataeye.service.impl.DataeyeProductServiceImpl;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import lombok.extern.slf4j.Slf4j;
 import org.quartz.Job;
 import org.quartz.JobExecutionContext;
 import org.quartz.JobExecutionException;
 import org.springframework.beans.factory.annotation.Autowired;
 
+import java.util.Iterator;
+import java.util.List;
+
 @Slf4j
 public class DataeyeJob implements Job {
     @Autowired
     private DataeyeCompanyService dataeyeCompanyService;
+    @Autowired
+    private DataeyeProductService dataeyeProductService;
+    @Autowired
+    private DataeyeMediaService dataeyeMediaService;
     @Override
     public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
-        for (int i = 1; i < 1000; i++){
-            dataeyeCompanyService.getCompanyInfo(i);
-        }
+        dataeyeMediaService.getMediaList();
+//        List<DataeyeCompany> list = dataeyeCompanyService.list();
+//        if (list != null && list.size() > 0){
+//            for(DataeyeCompany dataeyeCompany : list){
+//                try {
+//                    String result = HttpUtils.httpPostFormRequest("https://blzs.dataeye.com/company/listProductDistribution?companyId="+dataeyeCompany.getCompanyId(),null,null);
+//                    ObjectMapper mapper = new ObjectMapper();
+//                    mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
+//                    JsonNode rootNode = mapper.readTree(result);
+//                    int statusCode = rootNode.get("statusCode").asInt();
+//                    if (statusCode == 200) {
+//                        Iterator<JsonNode> nodes = rootNode.get("content").elements();
+//                        while (nodes.hasNext()){
+//                            JsonNode node = nodes.next();
+//                            Long productId = node.get("id").asLong();
+//                            dataeyeProductService.getProductInfo(dataeyeCompany.getCompanyId(),productId);
+//                        }
+//                    }
+//                }catch (Exception e){
+//                    e.printStackTrace();
+//                }
+//            }
+//        }
+//        int pageId=1;
+//        while(pageId <= 100){
+//            try {
+//                System.out.println("pageId:"+pageId);
+//                String result = HttpUtils.httpPostFormRequest("https://blzs.dataeye.com/ranking/listCompanyRanking?type=all&pageId="+pageId+"&pageSize=10&mediaId&playRuleId&themeId&searchKey&days=90&top=500",null,null);
+//                ObjectMapper mapper = new ObjectMapper();
+//                mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
+//                JsonNode rootNode = mapper.readTree(result);
+//                int statusCode = rootNode.get("statusCode").asInt();
+//                if (statusCode == 200){
+//                    Iterator<JsonNode> nodes = rootNode.get("content").elements();
+//                    while (nodes.hasNext()){
+//                        JsonNode node = nodes.next();
+//                        int companyId = node.get("companyId").asInt();
+//                        System.out.println(companyId);
+//                        dataeyeCompanyService.getCompanyInfo(companyId);
+//                    }
+//
+//                }
+//            }catch (Exception e){
+//
+//            }
+//            pageId++;
+//        }
+//        for (int i = 3528; i < 1000000; i++){
+//            try {
+//                dataeyeProductService.getProductInfo(i);
+//            } catch (Exception e) {
+//                e.printStackTrace();
+//            }
+//        }
     }
 }

+ 15 - 0
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/mapper/BytedanceFundDailyMapper.java

@@ -0,0 +1,15 @@
+package org.jeecg.modules.ctop.mapper;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import org.jeecg.modules.ctop.entity.BytedanceFundDaily;
+
+/**
+ * 账号日流水信息
+ *
+ * @author: jeecg-boot
+ * @date: 2020-04-08
+ * @cersion: V1.0
+ */
+public interface BytedanceFundDailyMapper extends BaseMapper<BytedanceFundDaily> {
+
+}

+ 5 - 0
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/mapper/xml/BytedanceFundDailyMapper.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="org.jeecg.modules.ctop.mapper.BytedanceFundDailyMapper">
+
+</mapper>

+ 2 - 1
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/service/IAdvertiserService.java

@@ -16,6 +16,7 @@ public interface IAdvertiserService extends IService<Advertiser> {
 
     List<Advertiser> getBySaleId(String saleId);
 
-    SheetInfoVo getXxlStatisticInfo(Long projectId, String systemType, Long accountId, String groupBy, String sheetName);
+    SheetInfoVo getXxlStatisticInfo(Long projectId, String systemType, Long accountId, String sheetName);
 
+    List<SheetInfoVo> getSystemTypeXxlStatisticInfo(Long projectId);
 }

+ 16 - 0
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/service/IBytedanceFundDailyService.java

@@ -0,0 +1,16 @@
+package org.jeecg.modules.ctop.service;
+
+import cn.com.ctop.common.module.entity.CtopOauthToken;
+import com.baomidou.mybatisplus.extension.service.IService;
+import org.jeecg.modules.ctop.entity.BytedanceFundDaily;
+
+/**
+ * 账号日流水信息
+ *
+ * @author jeecg-boot
+ * @version V1.0
+ * @date 2020-04-08
+ */
+public interface IBytedanceFundDailyService extends IService<BytedanceFundDaily> {
+    void loadFundDataByPage(CtopOauthToken token, String startDate, String endDate, Integer pageNum);
+}

+ 11 - 0
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/service/ICreativeTitleService.java

@@ -0,0 +1,11 @@
+package org.jeecg.modules.ctop.service;
+
+import com.alibaba.fastjson.JSONArray;
+
+public interface ICreativeTitleService {
+    JSONArray getTitleLabel();
+
+    JSONArray titleSuggestion(String firstCategory, String secondCategory, String keyword);
+
+    JSONArray titleSearch(String firstCategory, String secondCategory, String keywords);
+}

+ 11 - 1
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/service/impl/AdvertiserServiceImpl.java

@@ -32,7 +32,7 @@ public class AdvertiserServiceImpl extends ServiceImpl<AdvertiserMapper, Adverti
     }
 
     @Override
-    public SheetInfoVo getXxlStatisticInfo(Long projectId, String systemType, Long accountId, String groupBy, String sheetName) {
+    public SheetInfoVo getXxlStatisticInfo(Long projectId, String systemType, Long accountId, String sheetName) {
         SheetInfoVo vo = new SheetInfoVo();
         List<KuaishouXxlEntity> details = new ArrayList<>();
         vo.setSheetName(sheetName);
@@ -44,4 +44,14 @@ public class AdvertiserServiceImpl extends ServiceImpl<AdvertiserMapper, Adverti
         vo.setDetails(details);
         return vo;
     }
+
+    @Override
+    public List<SheetInfoVo> getSystemTypeXxlStatisticInfo(Long projectId) {
+        List<SheetInfoVo>vos = new ArrayList<>();
+        SheetInfoVo android = getXxlStatisticInfo(projectId,"android",null,"android汇总表");
+        vos.add(android);
+        SheetInfoVo ios = getXxlStatisticInfo(projectId,"ios",null,"ios汇总表");
+        vos.add(ios);
+        return vos;
+    }
 }

+ 60 - 0
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/service/impl/BytedanceFundDailyServiceImpl.java

@@ -0,0 +1,60 @@
+package org.jeecg.modules.ctop.service.impl;
+
+import cn.com.ctop.common.module.entity.CtopOauthToken;
+import cn.com.ctop.common.module.utils.HttpUtils;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import lombok.extern.slf4j.Slf4j;
+import lombok.var;
+import org.jeecg.modules.ctop.entity.BytedanceFundDaily;
+import org.jeecg.modules.ctop.mapper.BytedanceFundDailyMapper;
+import org.jeecg.modules.ctop.service.IBytedanceFundDailyService;
+import org.springframework.stereotype.Service;
+
+/**
+ * 账号日流水信息
+ *
+ * @author jeecg-boot
+ * @version V1.0
+ * @date 2020-04-08
+ */
+@Slf4j
+@Service
+public class BytedanceFundDailyServiceImpl extends ServiceImpl<BytedanceFundDailyMapper, BytedanceFundDaily> implements IBytedanceFundDailyService {
+    @Override
+    public void loadFundDataByPage(CtopOauthToken token, String startDate, String endDate, Integer pageNum) {
+        Integer pageSize = 100;
+        JSONObject result = getAdvertiserFundDailyStat(token, startDate, endDate, pageNum, pageSize);
+        System.out.println(result);
+        var code = result.getInteger("code");
+        var message = result.getString("message");
+        if (null == code || code != 0) {
+            log.error("账号日流水数据获取异常:{};accountId:{}", message, token.getAccountId());
+            return;
+        }
+        JSONArray dataArray = result.getJSONObject("data").getJSONArray("list");
+        if (null == dataArray || dataArray.size() <= 0) {
+            return;
+        }
+        for (var i = 0; i < dataArray.size(); i++) {
+            var data = dataArray.getJSONObject(i);
+            var fundDaily = new BytedanceFundDaily(data, token);
+            this.saveOrUpdate(fundDaily);
+        }
+        loadFundDataByPage(token, startDate, endDate, pageNum + 1);
+    }
+
+    public static JSONObject getAdvertiserFundDailyStat(CtopOauthToken token, String startDate, String endDate, Integer pageNum, Integer pageSize) {
+        // 请求地址
+        String url = "https://ad.oceanengine.com/open_api/2/advertiser/fund/daily_stat/";
+        // 请求参数
+        JSONObject params = new JSONObject();
+        params.put("advertiser_id", token.getAccountId());
+        params.put("start_date", startDate);
+        params.put("end_date", endDate);
+        params.put("page", pageNum);
+        params.put("page_size", pageSize);
+        return HttpUtils.bytedanceGetRequest(token.getAccessToken(), url, params);
+    }
+}

+ 81 - 0
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/service/impl/CreativeTitleServiceImpl.java

@@ -0,0 +1,81 @@
+package org.jeecg.modules.ctop.service.impl;
+
+import cn.com.ctop.common.module.utils.Check;
+import cn.com.ctop.common.module.utils.HttpUtils;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import org.jeecg.modules.ctop.service.ICreativeTitleService;
+import org.springframework.stereotype.Service;
+
+import java.util.HashMap;
+import java.util.Map;
+
+@Service
+public class CreativeTitleServiceImpl implements ICreativeTitleService {
+    @Override
+    public JSONArray getTitleLabel() {
+        Map<String, String> headers = new HashMap<>();
+        headers.put("Content-Type", "application/json");
+        String result = HttpUtils.httpGet("https://cc.oceanengine.com/creative_center_server/api/title/new_title_label", null, headers);
+        JSONObject resultJson = JSONObject.parseObject(result);
+        if (resultJson.getInteger("code") == 0) {
+            JSONObject dataJson = resultJson.getJSONObject("data");
+            if (!Check.isNull(dataJson)) {
+                return dataJson.getJSONArray("title_label");
+            }
+
+        }
+        return null;
+    }
+
+
+    @Override
+    public JSONArray titleSuggestion(String firstCategory, String secondCategory, String keyword) {
+        if (Check.isNull(keyword)) {
+            return null;
+        }
+
+        Map<String, String> headers = new HashMap<>();
+        headers.put("Content-Type", "application/json");
+        Map<String, Object> paramsMap = new HashMap<>();
+        paramsMap.put("first_category", firstCategory);
+        paramsMap.put("second_category", secondCategory);
+        paramsMap.put("keyword", keyword);
+
+        String result = HttpUtils.httpGet("https://cc.oceanengine.com/creative_center_server/api/title/title_suggestion", paramsMap, headers);
+        JSONObject resultJson = JSONObject.parseObject(result);
+        if (resultJson.getInteger("code") == 0) {
+            JSONObject dataJson = resultJson.getJSONObject("data");
+            if (!Check.isNull(dataJson)) {
+                return dataJson.getJSONArray("titles");
+            }
+
+        }
+        return null;
+    }
+
+    @Override
+    public JSONArray titleSearch(String firstCategory, String secondCategory, String keywords) {
+        if (Check.isNull(keywords)) {
+            return null;
+        }
+        Map<String, String> headers = new HashMap<>();
+        headers.put("Content-Type", "application/json");
+        Map<String, Object> paramsMap = new HashMap<>();
+        paramsMap.put("first_category", firstCategory);
+        paramsMap.put("second_category", secondCategory);
+        JSONObject json = new JSONObject();
+        json.put("keywords", keywords);
+        paramsMap.put("params", json);
+        String result = HttpUtils.httpGet("https://cc.oceanengine.com/creative_center_server/api/title/title_search", paramsMap, headers);
+        JSONObject resultJson = JSONObject.parseObject(result);
+        if (resultJson.getInteger("code") == 0) {
+            JSONObject dataJson = resultJson.getJSONObject("data");
+            if (!Check.isNull(dataJson)) {
+                return dataJson.getJSONArray("titles");
+            }
+
+        }
+        return null;
+    }
+}

+ 3 - 3
jeecg-boot-module-system/src/main/java/org/jeecg/modules/excelutil/entity/KuaishouXxlEntity.java

@@ -38,7 +38,7 @@ public class KuaishouXxlEntity extends BaseRowModel {
      * 激活成本
      */
     @ExcelProperty(index = 6, value = "激活成本")
-    private int activeCost;
+    private BigDecimal activeCost;
     @ExcelProperty(index = 7, value = "每日计划(组)")
     private int planNum;
     @ExcelProperty(index = 8, value = "每日素材安排")
@@ -84,11 +84,11 @@ public class KuaishouXxlEntity extends BaseRowModel {
         this.active = active;
     }
 
-    public int getActiveCost() {
+    public BigDecimal getActiveCost() {
         return activeCost;
     }
 
-    public void setActiveCost(int activeCost) {
+    public void setActiveCost(BigDecimal activeCost) {
         this.activeCost = activeCost;
     }
 

+ 11 - 13
jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java

@@ -5,16 +5,15 @@ import cn.com.ctop.common.module.entity.CtopOauthToken;
 import cn.com.ctop.common.module.mapper.CtopOauthTokenMapper;
 import cn.com.ctop.common.module.service.ICtopOauthTokenService;
 import cn.com.ctop.common.module.service.IUserAllocationService;
-import cn.com.ctop.common.module.utils.CtopAdConstant;
 import cn.com.ctop.common.module.utils.HttpUtils;
 import cn.com.ctop.kuaishou.modules.batch.service.IKuaiShouCreativeService;
 import cn.com.ctop.kuaishou.modules.batch.service.IKuaiShouHistoryReportTaskService;
 import cn.com.ctop.kuaishou.modules.batch.service.IKuaishouInterfaceService;
-import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouWebInterfaceService;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import lombok.extern.slf4j.Slf4j;
+import org.jeecg.modules.ctop.service.IBytedanceFundDailyService;
 import org.jeecg.modules.ctop.service.IPerformanceSaleService;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -23,6 +22,7 @@ import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.test.context.junit4.SpringRunner;
 
 import java.text.SimpleDateFormat;
+import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
@@ -134,26 +134,24 @@ public class SampleTest {
     private static CountDownLatch countDownLatch = null;
 
     @Autowired
-    private IKuaishouWebInterfaceService kuaishouWebInterfaceService;
+    private IBytedanceFundDailyService fundDailyService;
     @Test
     public void testLoadJob() {
-//        try {
-//            kuaishouWebInterfaceService.adkuaishouWebLogin("18606113995", "SsjX666!@");
-//            kuaishouWebInterfaceService.deleteAllComment(new HashMap<>());
-//        } catch (IOException e) {
-//            e.printStackTrace();
-//        }
-
-        List<CtopOauthToken> tokens = tokenService.getTokenListByType(CtopAdConstant.PLATFORM_TYPE_BYTEDANCE);
+        String endDate = "2020-04-07";
+        String startDate = "2019-09-01";
+        CtopOauthToken token = tokenService.getTokenByAccountId(1654676647522317L);
+//        List<CtopOauthToken> tokens = tokenService.getTokenListByType(CtopAdConstant.PLATFORM_TYPE_BYTEDANCE);
+        List<CtopOauthToken> tokens = new ArrayList<>();
+        tokens.add(token);
         if (null != tokens && tokens.size() > 0) {
             countDownLatch = new CountDownLatch(tokens.size());
-            tokens.forEach(token -> {
+            tokens.forEach(getToken -> {
                 //拉取往期数据
                 executorService.submit(new Runnable() {
                     @Override
                     public void run() {
                         try {
-                            reportService.loadBytedanceHistoryData(token);
+                            fundDailyService.loadFundDataByPage(getToken, startDate, endDate, 1);
                             countDownLatch.countDown();
                         } catch (Exception e) {
                             e.printStackTrace();

+ 11 - 7
module-common/src/main/java/cn/com/ctop/common/module/utils/HttpUtils.java

@@ -173,15 +173,19 @@ public class HttpUtils {
                 }
             }
             List<NameValuePair> paramList = new ArrayList<NameValuePair>();
-            for (String key : param.keySet()) {
-                BasicNameValuePair basicNameValuePair = new BasicNameValuePair(key, String.valueOf(param.get(key)));
-                paramList.add(basicNameValuePair);
+            if (param != null){
+                for (String key : param.keySet()) {
+                    BasicNameValuePair basicNameValuePair = new BasicNameValuePair(key, String.valueOf(param.get(key)));
+                    paramList.add(basicNameValuePair);
+                }
+                // 第二步:我们发现Entity是一个接口,所以只能找实现类,发现实现类又需要一个集合,集合的泛型是NameValuePair类型
+                UrlEncodedFormEntity formEntity = new UrlEncodedFormEntity(paramList);
+                // 第一步:通过setEntity 将我们的entity对象传递过去
+                httppost.setEntity(formEntity);
             }
 
-            // 第二步:我们发现Entity是一个接口,所以只能找实现类,发现实现类又需要一个集合,集合的泛型是NameValuePair类型
-            UrlEncodedFormEntity formEntity = new UrlEncodedFormEntity(paramList);
-            // 第一步:通过setEntity 将我们的entity对象传递过去
-            httppost.setEntity(formEntity);
+
+
             HttpEntity respentity;
 
             HttpResponse response = httpClient.execute(httppost);

+ 8 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeAdArtStyle.java

@@ -1,6 +1,14 @@
 package cn.com.ctop.crawler.modules.dataeye.entity;
 
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+@Data
+@TableName("ctop_dataeye_ad_art_style")
 public class DataeyeAdArtStyle {
+    @TableId(value = "id", type = IdType.INPUT)
     private Integer id;
     private String artStyle;
     private Integer orderId;

+ 1 - 1
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeCompany.java

@@ -11,7 +11,7 @@ import java.util.Date;
 @TableName("ctop_dataeye_company")
 public class DataeyeCompany {
     @TableId(value = "id", type = IdType.INPUT)
-    private Integer companyId;
+    private Long companyId;
     private String companyName;
     private Date firstSeen;
     private Date lastSeen;

+ 15 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeCompanyProduct.java

@@ -0,0 +1,15 @@
+package cn.com.ctop.crawler.modules.dataeye.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+@Data
+@TableName("ctop_dataeye_company_product")
+public class DataeyeCompanyProduct {
+    @TableId(value = "id", type = IdType.INPUT)
+    private Long id;
+    private Long companyId;
+    private Long productId;
+}

+ 12 - 10
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeMedia.java

@@ -1,18 +1,20 @@
 package cn.com.ctop.crawler.modules.dataeye.entity;
 
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
 import io.swagger.models.auth.In;
+import lombok.Data;
 
 import java.util.Date;
-
+@Data
+@TableName("ctop_dataeye_media")
 public class DataeyeMedia {
-    private Integer id;
+    @TableId(value = "id", type = IdType.INPUT)
+    private Long mediaId;
     private String mediaName;
-    private String logoUrl;
-    private String supportIos;
-    private String supportAndroid;
-    private String positionList;
-    private Integer isUnion;
-    private Integer isShow;
-    private Date createTime;
-    private Date updateTime;
+    private String logo;
+    private Integer companyNum;
+    private Integer productNum;
+    private Integer materialNum;
 }

+ 15 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeMediaCompany.java

@@ -0,0 +1,15 @@
+package cn.com.ctop.crawler.modules.dataeye.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+@Data
+@TableName("ctop_dataeye_media_company")
+public class DataeyeMediaCompany {
+    @TableId(value = "id", type = IdType.INPUT)
+    private Long id;
+    private Long mediaId;
+    private Long companyId;
+}

+ 15 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeMediaProduct.java

@@ -0,0 +1,15 @@
+package cn.com.ctop.crawler.modules.dataeye.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+@Data
+@TableName("ctop_dataeye_media_product")
+public class DataeyeMediaProduct {
+    @TableId(value = "id", type = IdType.INPUT)
+    private Long id;
+    private Long MediaId;
+    private Long productId;
+}

+ 8 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyePlayRules.java

@@ -1,6 +1,14 @@
 package cn.com.ctop.crawler.modules.dataeye.entity;
 
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+@Data
+@TableName("ctop_dataeye_play_rule")
 public class DataeyePlayRules {
+    @TableId(value = "id", type = IdType.INPUT)
     private Integer id;
     private String playRule;
     private Integer orderId;

+ 10 - 4
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeProduct.java

@@ -1,12 +1,20 @@
 package cn.com.ctop.crawler.modules.dataeye.entity;
 
-import java.util.Date;
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
 
+import java.util.Date;
+import java.util.List;
+@Data
+@TableName("ctop_dataeye_product")
 public class DataeyeProduct {
+    @TableId(value = "id", type = IdType.INPUT)
     private Integer productId;
     private String icon;
     private String productName;
-    private Date firstSceen;
+    private Date firstSeen;
     private Date lastSeen;
     private Integer days;
     private Integer type;
@@ -14,6 +22,4 @@ public class DataeyeProduct {
     private Integer mediaNum;
     private Integer positionNum;
     private Integer materialNum;
-    private String wxQrCode;
-
 }

+ 15 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeProductAdArtStyle.java

@@ -0,0 +1,15 @@
+package cn.com.ctop.crawler.modules.dataeye.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+@Data
+@TableName("ctop_dataeye_product_ad_art_style")
+public class DataeyeProductAdArtStyle {
+    @TableId(value = "id", type = IdType.INPUT)
+    private Long id;
+    private Long productId;
+    private Long styleId;
+}

+ 15 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeProductPlayRule.java

@@ -0,0 +1,15 @@
+package cn.com.ctop.crawler.modules.dataeye.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+@Data
+@TableName("ctop_dataeye_product_play_rule")
+public class DataeyeProductPlayRule {
+    @TableId(value = "id", type = IdType.INPUT)
+    private Long id;
+    private Long productId;
+    private Long ruleId;
+}

+ 15 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeProductTheme.java

@@ -0,0 +1,15 @@
+package cn.com.ctop.crawler.modules.dataeye.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+@Data
+@TableName("ctop_dataeye_product_theme")
+public class DataeyeProductTheme {
+    @TableId(value = "id", type = IdType.INPUT)
+    private Long id;
+    private Long productId;
+    private Long themeId;
+}

+ 8 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeThemes.java

@@ -1,6 +1,14 @@
 package cn.com.ctop.crawler.modules.dataeye.entity;
 
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+@Data
+@TableName("ctop_dataeye_theme")
 public class DataeyeThemes {
+    @TableId(value = "id", type = IdType.INPUT)
     private Integer id;
     private String theme;
     private Integer orderId;

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyeAdArtStyleMapper.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.dataeye.mapper;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeAdArtStyle;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+public interface DataeyeAdArtStyleMapper extends BaseMapper<DataeyeAdArtStyle> {
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyeCompanyProductMapper.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.dataeye.mapper;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeCompanyProduct;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+public interface DataeyeCompanyProductMapper extends BaseMapper<DataeyeCompanyProduct> {
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyeMediaMapper.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.dataeye.mapper;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeMedia;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+public interface DataeyeMediaMapper extends BaseMapper<DataeyeMedia> {
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyePlayRulesMapper.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.dataeye.mapper;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyePlayRules;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+public interface DataeyePlayRulesMapper extends BaseMapper<DataeyePlayRules> {
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyeProductAdArtStyleMapper.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.dataeye.mapper;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeProductAdArtStyle;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+public interface DataeyeProductAdArtStyleMapper extends BaseMapper<DataeyeProductAdArtStyle> {
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyeProductMapper.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.dataeye.mapper;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeProduct;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+public interface DataeyeProductMapper extends BaseMapper<DataeyeProduct> {
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyeProductPlayRuleMapper.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.dataeye.mapper;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeProductPlayRule;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+public interface DataeyeProductPlayRuleMapper extends BaseMapper<DataeyeProductPlayRule> {
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyeProductThemeMapper.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.dataeye.mapper;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeProductTheme;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+public interface DataeyeProductThemeMapper extends BaseMapper<DataeyeProductTheme> {
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyeThemesMapper.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.dataeye.mapper;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeThemes;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+public interface DataeyeThemesMapper extends BaseMapper<DataeyeThemes> {
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeAdArtStyleService.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.dataeye.service;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeAdArtStyle;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+public interface DataeyeAdArtStyleService extends IService<DataeyeAdArtStyle> {
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeCompanyProductService.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.dataeye.service;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeCompanyProduct;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+public interface DataeyeCompanyProductService extends IService<DataeyeCompanyProduct> {
+}

+ 4 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeLabelService.java

@@ -0,0 +1,4 @@
+package cn.com.ctop.crawler.modules.dataeye.service;
+
+public interface DataeyeLabelService {
+}

+ 8 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeMediaService.java

@@ -0,0 +1,8 @@
+package cn.com.ctop.crawler.modules.dataeye.service;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeMedia;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+public interface DataeyeMediaService extends IService<DataeyeMedia> {
+    public void getMediaList();
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyePlayRulesService.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.dataeye.service;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyePlayRules;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+public interface DataeyePlayRulesService extends IService<DataeyePlayRules> {
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeProductAdArtStyleService.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.dataeye.service;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeProductAdArtStyle;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+public interface DataeyeProductAdArtStyleService extends IService<DataeyeProductAdArtStyle> {
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeProductPlayRuleService.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.dataeye.service;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeProductPlayRule;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+public interface DataeyeProductPlayRuleService extends IService<DataeyeProductPlayRule> {
+}

+ 5 - 1
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeProductService.java

@@ -1,4 +1,8 @@
 package cn.com.ctop.crawler.modules.dataeye.service;
 
-public interface DataeyeProductService {
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeProduct;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+public interface DataeyeProductService extends IService<DataeyeProduct> {
+    public void getProductInfo(Long companyId,Long productId);
 }

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeProductThemeService.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.dataeye.service;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeProductTheme;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+public interface DataeyeProductThemeService extends IService<DataeyeProductTheme> {
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeThemesService.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.dataeye.service;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeThemes;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+public interface DataeyeThemesService extends IService<DataeyeThemes> {
+}

+ 11 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeAdArtStyleServiceImpl.java

@@ -0,0 +1,11 @@
+package cn.com.ctop.crawler.modules.dataeye.service.impl;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeAdArtStyle;
+import cn.com.ctop.crawler.modules.dataeye.mapper.DataeyeAdArtStyleMapper;
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyeAdArtStyleService;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.springframework.stereotype.Service;
+
+@Service
+public class DataeyeAdArtStyleServiceImpl extends ServiceImpl<DataeyeAdArtStyleMapper, DataeyeAdArtStyle> implements DataeyeAdArtStyleService {
+}

+ 11 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeCompanyProductServiceImpl.java

@@ -0,0 +1,11 @@
+package cn.com.ctop.crawler.modules.dataeye.service.impl;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeCompanyProduct;
+import cn.com.ctop.crawler.modules.dataeye.mapper.DataeyeCompanyProductMapper;
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyeCompanyProductService;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.springframework.stereotype.Service;
+
+@Service
+public class DataeyeCompanyProductServiceImpl extends ServiceImpl<DataeyeCompanyProductMapper, DataeyeCompanyProduct> implements DataeyeCompanyProductService {
+}

+ 30 - 15
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeCompanyServiceImpl.java

@@ -5,6 +5,7 @@ import cn.com.ctop.crawler.modules.core.util.SignUtil;
 import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeCompany;
 import cn.com.ctop.crawler.modules.dataeye.mapper.DataeyeCompanyMapper;
 import cn.com.ctop.crawler.modules.dataeye.service.DataeyeCompanyService;
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyeProductService;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.JsonNode;
@@ -13,6 +14,7 @@ import com.google.gson.Gson;
 import org.apache.http.cookie.Cookie;
 import org.apache.http.impl.cookie.BasicClientCookie;
 import org.jeecg.common.util.DateUtils;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import sun.misc.BASE64Encoder;
 
@@ -22,26 +24,17 @@ import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.Map;
 @Service
 public class DataeyeCompanyServiceImpl extends ServiceImpl<DataeyeCompanyMapper, DataeyeCompany> implements DataeyeCompanyService {
-
+    @Autowired
+    private DataeyeProductService dataeyeProductService;
     public void getCompanyInfo(Integer companyId){
-        Map<String,String> headerMap = new HashMap<>();
-        headerMap.put("Host","adx.dataeye.com");
-        headerMap.put("Origin","https://adx.dataeye.com");
-        headerMap.put("Referer","https://adx.dataeye.com/");
-        BasicClientCookie ck = new BasicClientCookie("tokenglobal", "721501ec4ec04a50bbffc0c0240cbbdd63a0d5456205b9a5ea1581ce6c6554ca");
-        ck.setDomain("adx.dataeye.com");
-        ck.setExpiryDate(DateUtils.addDay(new Date(),365));
-        ck.setPath("/");
-        HttpUtils.COOKIESTORE.addCookie(ck);
         Map<String,Object> paramMap = new HashMap<>();
         paramMap.put("companyId",companyId);
-        paramMap.put("thisTimes",(long)(new Date().getTime()/100));
-        String sign = SignUtil.getDataeyeSign(paramMap);
-        paramMap.put("sign",sign);
-        String response = HttpUtils.httpPostFormRequest("https://adx.dataeye.com/company/getCompanyInfo",paramMap,headerMap);
+        String response = HttpUtils.httpPostFormRequest("https://blzs.dataeye.com/company/getCompanyInfo",paramMap,null);
+        System.out.println(response);
         ObjectMapper mapper = new ObjectMapper();
         mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
         try {
@@ -50,7 +43,29 @@ public class DataeyeCompanyServiceImpl extends ServiceImpl<DataeyeCompanyMapper,
             if (statusCode == 200){
                 String content = rootNode.get("content").toString();
                 DataeyeCompany company = mapper.readValue(content,DataeyeCompany.class);
-                this.save(company);
+                if(company != null && company.getCompanyName() != null){
+                    this.saveOrUpdate(company);
+                    String result = HttpUtils.httpPostFormRequest("https://blzs.dataeye.com/company/listProductDistribution?companyId="+companyId,null,null);
+                    System.out.println(result);
+                    rootNode = mapper.readTree(result);
+                    statusCode = rootNode.get("statusCode").asInt();
+                    if (statusCode == 200) {
+                        Iterator<JsonNode> nodes = rootNode.get("content").elements();
+                        while (nodes.hasNext()){
+                            JsonNode node = nodes.next();
+                            Long productId = node.get("id").asLong();
+//                            Map<String,Long> map = new HashMap<>();
+//                            map.put("companyId",companyId.longValue());
+//                            map.put("productId",productId);
+//                            try {
+//                                DataeyeProductServiceImpl.taskQueue.put(map);
+//                            } catch (InterruptedException e) {
+//                                e.printStackTrace();
+//                            }
+                            dataeyeProductService.getProductInfo(companyId.longValue(),productId);
+                        }
+                    }
+                }
             }
         } catch (IOException e) {
             e.printStackTrace();

+ 62 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeLabelServiceImpl.java

@@ -0,0 +1,62 @@
+package cn.com.ctop.crawler.modules.dataeye.service.impl;
+
+import cn.com.ctop.common.module.utils.HttpUtils;
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeAdArtStyle;
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeCompany;
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyePlayRules;
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeThemes;
+import cn.com.ctop.crawler.modules.dataeye.mapper.DataeyeAdArtStyleMapper;
+import cn.com.ctop.crawler.modules.dataeye.mapper.DataeyePlayRulesMapper;
+import cn.com.ctop.crawler.modules.dataeye.mapper.DataeyeThemesMapper;
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyeAdArtStyleService;
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyeLabelService;
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyePlayRulesService;
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyeThemesService;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.io.IOException;
+import java.util.List;
+
+@Service
+public class DataeyeLabelServiceImpl implements DataeyeLabelService {
+    @Autowired
+    private DataeyeAdArtStyleService dataeyeAdArtStyleService;
+    @Autowired
+    private DataeyeThemesService dataeyeThemesService;
+    @Autowired
+    private DataeyePlayRulesService dataeyePlayRulesService;
+
+    public void getLabelList(){
+        String response = HttpUtils.httpPostFormRequest("https://blzs.dataeye.com/common/getNewLabelMenu",null,null);
+        System.out.println(response);
+        ObjectMapper mapper = new ObjectMapper();
+        mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
+        try {
+            JsonNode rootNode = mapper.readTree(response);
+            int statusCode = rootNode.get("statusCode").asInt();
+            if (statusCode == 200){
+                String adThemeList = rootNode.get("content").get("adThemeList").toString();
+                String adPlayRuleList = rootNode.get("content").get("adPlayRuleList").toString();
+                String adArtStyleList = rootNode.get("content").get("adArtStyleList").toString();
+                List<DataeyeAdArtStyle> adArtStyles = mapper.readValue(adArtStyleList, new TypeReference<List<DataeyeAdArtStyle>>(){});
+                List<DataeyeThemes> themes = mapper.readValue(adThemeList,new TypeReference<List<DataeyeThemes>>(){});
+                List<DataeyePlayRules> playRules = mapper.readValue(adPlayRuleList,new TypeReference<List<DataeyePlayRules>>(){});
+                dataeyeAdArtStyleService.saveOrUpdateBatch(adArtStyles);
+                dataeyeThemesService.saveOrUpdateBatch(themes);
+                dataeyePlayRulesService.saveOrUpdateBatch(playRules);
+//                DataeyeCompany company = mapper.readValue(content,DataeyeCompany.class);
+//                if(company != null && company.getCompanyName() != null){
+//                    this.save(company);
+//                }
+
+            }
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+}

+ 73 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeMediaServiceImpl.java

@@ -0,0 +1,73 @@
+package cn.com.ctop.crawler.modules.dataeye.service.impl;
+
+import cn.com.ctop.common.module.utils.HttpUtils;
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeCompany;
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeMedia;
+import cn.com.ctop.crawler.modules.dataeye.mapper.DataeyeMediaMapper;
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyeCompanyService;
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyeMediaService;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.swagger.models.auth.In;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.io.IOException;
+import java.util.*;
+
+@Service
+public class DataeyeMediaServiceImpl extends ServiceImpl<DataeyeMediaMapper, DataeyeMedia> implements DataeyeMediaService {
+    @Autowired
+    private DataeyeCompanyService dataeyeCompanyService;
+    public void getMediaList(){
+        String response = HttpUtils.httpPostFormRequest("https://blzs.dataeye.com/common/listMedia",null,null);
+        System.out.println(response);
+        ObjectMapper mapper = new ObjectMapper();
+        mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
+        try {
+            JsonNode rootNode = mapper.readTree(response);
+            int statusCode = rootNode.get("statusCode").asInt();
+            if (statusCode == 200){
+                Iterator<JsonNode> nodes = rootNode.get("content").elements();
+                List<Integer> ids = new ArrayList<>();
+                while (nodes.hasNext()){
+                    JsonNode node =  nodes.next();
+                    Integer mediaId = node.get("k").asInt();
+                    ids.add(mediaId);
+                    response = HttpUtils.httpPostFormRequest("https://blzs.dataeye.com/media/getMediaInfo?mediaId="+mediaId,null,null);
+                    System.out.println(response);
+                    rootNode = mapper.readTree(response);
+                    statusCode = rootNode.get("statusCode").asInt();
+                    if (statusCode == 200){
+                        String content = rootNode.get("content").toString();
+                        DataeyeMedia media = mapper.readValue(content,DataeyeMedia.class);
+                        if(media != null && media.getMediaId() != null){
+                            this.saveOrUpdate(media);
+
+                        }
+                    }
+                }
+                if(ids != null){
+                    for (Integer mediaId :ids){
+                        response = HttpUtils.httpPostFormRequest("https://blzs.dataeye.com/media/listCompanyDistribution?mediaId="+mediaId,null,null);
+                        System.out.println(response);
+                        rootNode = mapper.readTree(response);
+                        statusCode = rootNode.get("statusCode").asInt();
+                        if (statusCode == 200){
+                            Iterator<JsonNode> companyNodes = rootNode.get("content").elements();
+                            while (companyNodes.hasNext()){
+                                JsonNode companyNode = companyNodes.next();
+                                Integer companyId = companyNode.get("id").asInt();
+                                dataeyeCompanyService.getCompanyInfo(companyId);
+                            }
+                        }
+                    }
+                }
+            }
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+}

+ 11 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyePlayRulesServiceImpl.java

@@ -0,0 +1,11 @@
+package cn.com.ctop.crawler.modules.dataeye.service.impl;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyePlayRules;
+import cn.com.ctop.crawler.modules.dataeye.mapper.DataeyePlayRulesMapper;
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyePlayRulesService;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.springframework.stereotype.Service;
+
+@Service
+public class DataeyePlayRulesServiceImpl extends ServiceImpl<DataeyePlayRulesMapper, DataeyePlayRules> implements DataeyePlayRulesService{
+}

+ 13 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeProductAdArtStyleServiceImpl.java

@@ -0,0 +1,13 @@
+package cn.com.ctop.crawler.modules.dataeye.service.impl;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeProductAdArtStyle;
+import cn.com.ctop.crawler.modules.dataeye.mapper.DataeyeProductAdArtStyleMapper;
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyeAdArtStyleService;
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyePlayRulesService;
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyeProductAdArtStyleService;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.springframework.stereotype.Service;
+
+@Service
+public class DataeyeProductAdArtStyleServiceImpl extends ServiceImpl<DataeyeProductAdArtStyleMapper, DataeyeProductAdArtStyle> implements DataeyeProductAdArtStyleService {
+}

+ 11 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeProductPlayRuleServiceImpl.java

@@ -0,0 +1,11 @@
+package cn.com.ctop.crawler.modules.dataeye.service.impl;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeProductPlayRule;
+import cn.com.ctop.crawler.modules.dataeye.mapper.DataeyeProductPlayRuleMapper;
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyeProductPlayRuleService;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.springframework.stereotype.Service;
+
+@Service
+public class DataeyeProductPlayRuleServiceImpl extends ServiceImpl<DataeyeProductPlayRuleMapper, DataeyeProductPlayRule> implements DataeyeProductPlayRuleService{
+}

+ 107 - 2
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeProductServiceImpl.java

@@ -1,8 +1,113 @@
 package cn.com.ctop.crawler.modules.dataeye.service.impl;
 
-import cn.com.ctop.crawler.modules.dataeye.service.DataeyeProductService;
+import cn.com.ctop.common.module.utils.HttpUtils;
+import cn.com.ctop.crawler.modules.dataeye.entity.*;
+import cn.com.ctop.crawler.modules.dataeye.mapper.DataeyeProductMapper;
+import cn.com.ctop.crawler.modules.dataeye.service.*;
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
+import us.codecraft.webmagic.thread.CountableThreadPool;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.logging.Logger;
 
 @Service
-public class DataeyeProductServiceImpl implements DataeyeProductService {
+public class DataeyeProductServiceImpl extends ServiceImpl<DataeyeProductMapper, DataeyeProduct> implements DataeyeProductService {
+    @Autowired
+    private DataeyeProductThemeService dataeyeProductThemeService;
+    @Autowired
+    private DataeyeProductPlayRuleService dataeyeProductPlayRuleService;
+    @Autowired
+    private DataeyeProductAdArtStyleService dataeyeProductAdArtStyleService;
+    @Autowired
+    private DataeyeCompanyProductService dataeyeCompanyProductService;
+
+    public void getProductInfo(Long companyId,Long productId){
+        Map<String,Object> paramMap = new HashMap<>();
+        paramMap.put("productId",productId);
+        String response = HttpUtils.httpPostFormRequest("https://blzs.dataeye.com/product/getProductInfo",paramMap,null);
+        System.out.println(response);
+        ObjectMapper mapper = new ObjectMapper();
+        mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
+        try {
+            JsonNode rootNode = mapper.readTree(response);
+            int statusCode = rootNode.get("statusCode").asInt();
+            if (statusCode == 200){
+                String content = rootNode.get("content").toString();
+                DataeyeProduct product = mapper.readValue(content,DataeyeProduct.class);
+                if(product != null && product.getProductName() != null && !product.getProductName().equals("")){
+                    this.saveOrUpdate(product);
+                }
+                QueryWrapper<DataeyeCompanyProduct> queryWrapper1 = new QueryWrapper<>();
+                queryWrapper1.eq("company_id",companyId);
+                queryWrapper1.eq("product_id",productId);
+                DataeyeCompanyProduct dataeyeCompanyProduct = dataeyeCompanyProductService.getOne(queryWrapper1);
+                if(dataeyeCompanyProduct == null){
+                    dataeyeCompanyProduct = new DataeyeCompanyProduct();
+                    dataeyeCompanyProduct.setCompanyId(companyId);
+                    dataeyeCompanyProduct.setProductId(productId);
+                    dataeyeCompanyProductService.save(dataeyeCompanyProduct);
+                }
+                Iterator<JsonNode> themeNodes = rootNode.get("content").get("themes").elements();
+                Iterator<JsonNode> ruleNodes = rootNode.get("content").get("playRules").elements();
+                Iterator<JsonNode> styleNodes = rootNode.get("content").get("adArtStyles").elements();
+                while (themeNodes.hasNext()){
+                    JsonNode node = themeNodes.next();
+                    Long themeId = node.get("id").asLong();
+                    QueryWrapper<DataeyeProductTheme> queryWrapper = new QueryWrapper<>();
+                    queryWrapper.eq("product_id",productId);
+                    queryWrapper.eq("theme_id",themeId);
+                    DataeyeProductTheme theme = dataeyeProductThemeService.getOne(queryWrapper);
+                    if (theme == null){
+                        theme = new DataeyeProductTheme();
+                        theme.setProductId(productId.longValue());
+                        theme.setThemeId(themeId);
+                        dataeyeProductThemeService.save(theme);
+                    }
+                }
+                while (ruleNodes.hasNext()){
+                    JsonNode node = ruleNodes.next();
+                    Long ruleId = node.get("id").asLong();
+                    QueryWrapper<DataeyeProductPlayRule> queryWrapper = new QueryWrapper<>();
+                    queryWrapper.eq("product_id",productId);
+                    queryWrapper.eq("rule_id",ruleId);
+                    DataeyeProductPlayRule rule = dataeyeProductPlayRuleService.getOne(queryWrapper);
+                    if (rule == null){
+                        rule = new DataeyeProductPlayRule();
+                        rule.setProductId(productId.longValue());
+                        rule.setRuleId(ruleId);
+                        dataeyeProductPlayRuleService.save(rule);
+                    }
+                }
+                while (styleNodes.hasNext()){
+                    JsonNode node = styleNodes.next();
+                    Long styleId = node.get("id").asLong();
+                    QueryWrapper<DataeyeProductAdArtStyle> queryWrapper = new QueryWrapper<>();
+                    queryWrapper.eq("product_id",productId);
+                    queryWrapper.eq("style_id",styleId);
+                    DataeyeProductAdArtStyle style = dataeyeProductAdArtStyleService.getOne(queryWrapper);
+                    if (style == null){
+                        style = new DataeyeProductAdArtStyle();
+                        style.setProductId(productId.longValue());
+                        style.setStyleId(styleId);
+                        dataeyeProductAdArtStyleService.save(style);
+                    }
+                }
+            }
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
 }

+ 11 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeProductThemeServiceImpl.java

@@ -0,0 +1,11 @@
+package cn.com.ctop.crawler.modules.dataeye.service.impl;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeProductTheme;
+import cn.com.ctop.crawler.modules.dataeye.mapper.DataeyeProductThemeMapper;
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyeProductThemeService;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.springframework.stereotype.Service;
+
+@Service
+public class DataeyeProductThemeServiceImpl extends ServiceImpl<DataeyeProductThemeMapper, DataeyeProductTheme> implements DataeyeProductThemeService {
+}

+ 12 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeThemesServiceImpl.java

@@ -0,0 +1,12 @@
+package cn.com.ctop.crawler.modules.dataeye.service.impl;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeThemes;
+import cn.com.ctop.crawler.modules.dataeye.mapper.DataeyeCompanyMapper;
+import cn.com.ctop.crawler.modules.dataeye.mapper.DataeyeThemesMapper;
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyeThemesService;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.springframework.stereotype.Service;
+
+@Service
+public class DataeyeThemesServiceImpl extends ServiceImpl<DataeyeThemesMapper, DataeyeThemes> implements DataeyeThemesService {
+}

+ 1 - 1
module-report/src/main/java/cn/com/ctop/bytedance/mapper/xml/KuaiShouYiCheReportMapper.xml

@@ -446,7 +446,7 @@
             and t1.stat_date = #{statDate}
         </if>
         ) t2
-        WHERE t2.campaignName in(845,846,847,848,849,850,1071,1072)
+        WHERE t2.campaignName in(845,846,847,848,849,850,1071,1072,1066)
 
     </select>
 

+ 46 - 47
module-report/src/main/java/cn/com/ctop/bytedance/mapper/xml/MaterialReportMapper.xml

@@ -92,31 +92,34 @@
         END
         ) conversionPrice, -- 转化单价
 
-        t2.url url, -- 视频url
+        --  t2.url url, -- 视频url
+
+        (SELECT  url
+        FROM ctop_kuaishou_video_get
+        WHERE signature = t2.signature
+        LIMIT 1
+        ) url,
         t2.signature signature -- 视频md5
         from
         ctop_user_allocation t1
         left join
-        ctop_kuaishou_report_hourly_creative_statistic t2
+        ctop_kuaishou_report_daily_creative t2
         on t1.account_id = t2.account_id
+
         <where>
             <if test="userId != null">
                 and t1.user_id = #{userId,jdbcType=VARCHAR}
             </if>
-            and t1.media_id = '2'
-            and t1.account_id != ''
-            and t2.signature != ''
+
             <if test="startDate != null">
                 and t2.stat_date = #{startDate}
             </if>
-            <if test="statHour != null">
-                and t2.stat_hour &lt;= #{statHour}
-            </if>
+            and t2.signature is not null
         </where>
-
         group by t2.signature
         order by sum(t2.charge) desc
-        limit 100
+        limit 100;
+
 
 
     </select>
@@ -207,8 +210,6 @@
 
     <select id="selectDailyByMap" parameterType="Map" resultType="com.alibaba.fastjson.JSONObject">
 
-
-
         SELECT
         sum(t2.charge) cost, -- 花费
         sum(t2.photo_show) photoShow, -- 封面曝光数
@@ -471,87 +472,85 @@
 
     <select id="selectDayMaterialList" parameterType="Map" resultType="com.alibaba.fastjson.JSONObject">
         SELECT
-        sum(charge) cost, -- 花费
-        sum(photo_show) photoShow, -- 封面曝光数
-        sum(photo_click) photoClick, -- 封面点击数
-        sum(aclick) aclick, -- 素材曝光数
-        sum(bclick) bclick, -- 行为数
+        sum(t1.charge) cost, -- 花费
+        sum(t1.photo_show) photoShow, -- 封面曝光数
+        sum(t1.photo_click) photoClick, -- 封面点击数
+        sum(t1.aclick) aclick, -- 素材曝光数
+        sum(t1.bclick) bclick, -- 行为数
         (case
-        when sum(photo_show) != 0
-        then (sum(photo_click) / sum(photo_show))
+        when sum(t1.photo_show) != 0
+        then (sum(t1.photo_click) / sum(t1.photo_show))
         else 0
         end
         ) photoClickRatio, -- 封面点击率
 
         (case
-        when sum(aclick) != 0
-        then (sum(bclick) / sum(aclick))
+        when sum(t1.aclick) != 0
+        then (sum(t1.bclick) / sum(t1.aclick))
         else 0
         end
         ) actionRatio, -- 行为率
 
         (case
-        when sum(photo_show) != 0
-        then ((sum(charge) / sum(photo_show)) * 1000)
+        when sum(t1.photo_show) != 0
+        then ((sum(t1.charge) / sum(t1.photo_show)) * 1000)
         else 0
         end
         ) impression1kCost, -- 均千次封面曝光花费
 
         (case
-        when sum(photo_click) != 0
-        then (sum(charge) / sum(photo_click))
+        when sum(t1.photo_click) != 0
+        then (sum(t1.charge) / sum(t1.photo_click))
         else 0
         end
         ) photoClickCost, -- 平均封面点击单价
 
         (CASE
-        WHEN sum(photo_click) != 0
-        THEN (sum(charge) / sum(photo_click))
+        WHEN sum(t1.photo_click) != 0
+        THEN (sum(t1.charge) / sum(t1.photo_click))
         ELSE 0
         end
         ) actionCost, -- 平均行为单价
 
         (case
-        WHEN sum(form_count) != 0
-        THEN sum(form_count)
-        WHEN sum(activation) != 0
-        THEN sum(activation)
+        WHEN sum(t1.form_count) != 0
+        THEN sum(t1.form_count)
+        WHEN sum(t1.activation) != 0
+        THEN sum(t1.activation)
         ELSE 0
         END
         ) conversions, -- 转化数
 
         (
         CASE
-        WHEN sum(form_count) != 0
-        THEN (sum(charge) / sum(form_count))
-        WHEN sum(activation) != 0
-        THEN (sum(charge) / sum(activation))
+        WHEN sum(t1.form_count) != 0
+        THEN (sum(t1.charge) / sum(t1.form_count))
+        WHEN sum(t1.activation) != 0
+        THEN (sum(t1.charge) / sum(t1.activation))
         ELSE 0
         END
         ) conversionPrice, -- 转化单价
 
-        ( SELECT url from ctop_kuaishou_video_get WHERE signature = signature limit 1) url,
-        signature signature, -- 视频md5
-        account_id accountId
+        ( SELECT url from ctop_kuaishou_video_get WHERE signature = t1.signature limit 1) url,
+        t1.signature signature, -- 视频md5
+        t1.account_id accountId
         from
-        ctop_kuaishou_report_daily_creative
-        where  account_id in
+        ctop_kuaishou_report_daily_creative t1
+        where  t1.account_id in
         <foreach item="item" index="index" collection="accountIds"
                  open="(" separator="," close=")">
             #{item}
         </foreach>
-
-
         <if test="startDate != null">
-            and stat_date &gt;= #{startDate}
+            and t1.stat_date &gt;= #{startDate}
         </if>
 
         <if test="endDate != null">
-            and stat_date &lt;= #{endDate}
+            and t1.stat_date &lt;= #{endDate}
         </if>
-        and signature is not null
-        GROUP BY signature
-        order by sum(charge) desc
+        and t1.signature is not null
+        GROUP BY t1.signature
+        order by sum(t1.charge) desc
         limit 100
     </select>