瀏覽代碼

dataeye数据接口

xuzuoyun 5 年之前
父節點
當前提交
5e17421b12
共有 44 個文件被更改,包括 699 次插入43 次删除
  1. 70 3
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/DataeyeJob.java
  2. 11 7
      module-common/src/main/java/cn/com/ctop/common/module/utils/HttpUtils.java
  3. 8 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeAdArtStyle.java
  4. 1 1
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeCompany.java
  5. 15 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeCompanyProduct.java
  6. 12 10
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeMedia.java
  7. 15 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeMediaCompany.java
  8. 15 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeMediaProduct.java
  9. 8 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyePlayRules.java
  10. 10 4
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeProduct.java
  11. 15 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeProductAdArtStyle.java
  12. 15 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeProductPlayRule.java
  13. 15 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeProductTheme.java
  14. 8 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeThemes.java
  15. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyeAdArtStyleMapper.java
  16. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyeCompanyProductMapper.java
  17. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyeMediaMapper.java
  18. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyePlayRulesMapper.java
  19. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyeProductAdArtStyleMapper.java
  20. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyeProductMapper.java
  21. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyeProductPlayRuleMapper.java
  22. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyeProductThemeMapper.java
  23. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyeThemesMapper.java
  24. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeAdArtStyleService.java
  25. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeCompanyProductService.java
  26. 4 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeLabelService.java
  27. 8 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeMediaService.java
  28. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyePlayRulesService.java
  29. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeProductAdArtStyleService.java
  30. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeProductPlayRuleService.java
  31. 5 1
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeProductService.java
  32. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeProductThemeService.java
  33. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeThemesService.java
  34. 11 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeAdArtStyleServiceImpl.java
  35. 11 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeCompanyProductServiceImpl.java
  36. 30 15
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeCompanyServiceImpl.java
  37. 62 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeLabelServiceImpl.java
  38. 73 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeMediaServiceImpl.java
  39. 11 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyePlayRulesServiceImpl.java
  40. 13 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeProductAdArtStyleServiceImpl.java
  41. 11 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeProductPlayRuleServiceImpl.java
  42. 107 2
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeProductServiceImpl.java
  43. 11 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeProductThemeServiceImpl.java
  44. 12 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeThemesServiceImpl.java

+ 70 - 3
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/DataeyeJob.java

@@ -1,20 +1,87 @@
 package org.jeecg.modules.ctop.job;
 package org.jeecg.modules.ctop.job;
 
 
+import cn.com.ctop.common.module.utils.HttpUtils;
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeCompany;
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeProduct;
 import cn.com.ctop.crawler.modules.dataeye.service.DataeyeCompanyService;
 import cn.com.ctop.crawler.modules.dataeye.service.DataeyeCompanyService;
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyeMediaService;
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyeProductService;
+import cn.com.ctop.crawler.modules.dataeye.service.impl.DataeyeProductServiceImpl;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import lombok.extern.slf4j.Slf4j;
 import lombok.extern.slf4j.Slf4j;
 import org.quartz.Job;
 import org.quartz.Job;
 import org.quartz.JobExecutionContext;
 import org.quartz.JobExecutionContext;
 import org.quartz.JobExecutionException;
 import org.quartz.JobExecutionException;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Autowired;
 
 
+import java.util.Iterator;
+import java.util.List;
+
 @Slf4j
 @Slf4j
 public class DataeyeJob implements Job {
 public class DataeyeJob implements Job {
     @Autowired
     @Autowired
     private DataeyeCompanyService dataeyeCompanyService;
     private DataeyeCompanyService dataeyeCompanyService;
+    @Autowired
+    private DataeyeProductService dataeyeProductService;
+    @Autowired
+    private DataeyeMediaService dataeyeMediaService;
     @Override
     @Override
     public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
     public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
-        for (int i = 1; i < 1000; i++){
-            dataeyeCompanyService.getCompanyInfo(i);
-        }
+        dataeyeMediaService.getMediaList();
+//        List<DataeyeCompany> list = dataeyeCompanyService.list();
+//        if (list != null && list.size() > 0){
+//            for(DataeyeCompany dataeyeCompany : list){
+//                try {
+//                    String result = HttpUtils.httpPostFormRequest("https://blzs.dataeye.com/company/listProductDistribution?companyId="+dataeyeCompany.getCompanyId(),null,null);
+//                    ObjectMapper mapper = new ObjectMapper();
+//                    mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
+//                    JsonNode rootNode = mapper.readTree(result);
+//                    int statusCode = rootNode.get("statusCode").asInt();
+//                    if (statusCode == 200) {
+//                        Iterator<JsonNode> nodes = rootNode.get("content").elements();
+//                        while (nodes.hasNext()){
+//                            JsonNode node = nodes.next();
+//                            Long productId = node.get("id").asLong();
+//                            dataeyeProductService.getProductInfo(dataeyeCompany.getCompanyId(),productId);
+//                        }
+//                    }
+//                }catch (Exception e){
+//                    e.printStackTrace();
+//                }
+//            }
+//        }
+//        int pageId=1;
+//        while(pageId <= 100){
+//            try {
+//                System.out.println("pageId:"+pageId);
+//                String result = HttpUtils.httpPostFormRequest("https://blzs.dataeye.com/ranking/listCompanyRanking?type=all&pageId="+pageId+"&pageSize=10&mediaId&playRuleId&themeId&searchKey&days=90&top=500",null,null);
+//                ObjectMapper mapper = new ObjectMapper();
+//                mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
+//                JsonNode rootNode = mapper.readTree(result);
+//                int statusCode = rootNode.get("statusCode").asInt();
+//                if (statusCode == 200){
+//                    Iterator<JsonNode> nodes = rootNode.get("content").elements();
+//                    while (nodes.hasNext()){
+//                        JsonNode node = nodes.next();
+//                        int companyId = node.get("companyId").asInt();
+//                        System.out.println(companyId);
+//                        dataeyeCompanyService.getCompanyInfo(companyId);
+//                    }
+//
+//                }
+//            }catch (Exception e){
+//
+//            }
+//            pageId++;
+//        }
+//        for (int i = 3528; i < 1000000; i++){
+//            try {
+//                dataeyeProductService.getProductInfo(i);
+//            } catch (Exception e) {
+//                e.printStackTrace();
+//            }
+//        }
     }
     }
 }
 }

+ 11 - 7
module-common/src/main/java/cn/com/ctop/common/module/utils/HttpUtils.java

@@ -173,15 +173,19 @@ public class HttpUtils {
                 }
                 }
             }
             }
             List<NameValuePair> paramList = new ArrayList<NameValuePair>();
             List<NameValuePair> paramList = new ArrayList<NameValuePair>();
-            for (String key : param.keySet()) {
-                BasicNameValuePair basicNameValuePair = new BasicNameValuePair(key, String.valueOf(param.get(key)));
-                paramList.add(basicNameValuePair);
+            if (param != null){
+                for (String key : param.keySet()) {
+                    BasicNameValuePair basicNameValuePair = new BasicNameValuePair(key, String.valueOf(param.get(key)));
+                    paramList.add(basicNameValuePair);
+                }
+                // 第二步:我们发现Entity是一个接口,所以只能找实现类,发现实现类又需要一个集合,集合的泛型是NameValuePair类型
+                UrlEncodedFormEntity formEntity = new UrlEncodedFormEntity(paramList);
+                // 第一步:通过setEntity 将我们的entity对象传递过去
+                httppost.setEntity(formEntity);
             }
             }
 
 
-            // 第二步:我们发现Entity是一个接口,所以只能找实现类,发现实现类又需要一个集合,集合的泛型是NameValuePair类型
-            UrlEncodedFormEntity formEntity = new UrlEncodedFormEntity(paramList);
-            // 第一步:通过setEntity 将我们的entity对象传递过去
-            httppost.setEntity(formEntity);
+
+
             HttpEntity respentity;
             HttpEntity respentity;
 
 
             HttpResponse response = httpClient.execute(httppost);
             HttpResponse response = httpClient.execute(httppost);

+ 8 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeAdArtStyle.java

@@ -1,6 +1,14 @@
 package cn.com.ctop.crawler.modules.dataeye.entity;
 package cn.com.ctop.crawler.modules.dataeye.entity;
 
 
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+@Data
+@TableName("ctop_dataeye_ad_art_style")
 public class DataeyeAdArtStyle {
 public class DataeyeAdArtStyle {
+    @TableId(value = "id", type = IdType.INPUT)
     private Integer id;
     private Integer id;
     private String artStyle;
     private String artStyle;
     private Integer orderId;
     private Integer orderId;

+ 1 - 1
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeCompany.java

@@ -11,7 +11,7 @@ import java.util.Date;
 @TableName("ctop_dataeye_company")
 @TableName("ctop_dataeye_company")
 public class DataeyeCompany {
 public class DataeyeCompany {
     @TableId(value = "id", type = IdType.INPUT)
     @TableId(value = "id", type = IdType.INPUT)
-    private Integer companyId;
+    private Long companyId;
     private String companyName;
     private String companyName;
     private Date firstSeen;
     private Date firstSeen;
     private Date lastSeen;
     private Date lastSeen;

+ 15 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeCompanyProduct.java

@@ -0,0 +1,15 @@
+package cn.com.ctop.crawler.modules.dataeye.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+@Data
+@TableName("ctop_dataeye_company_product")
+public class DataeyeCompanyProduct {
+    @TableId(value = "id", type = IdType.INPUT)
+    private Long id;
+    private Long companyId;
+    private Long productId;
+}

+ 12 - 10
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeMedia.java

@@ -1,18 +1,20 @@
 package cn.com.ctop.crawler.modules.dataeye.entity;
 package cn.com.ctop.crawler.modules.dataeye.entity;
 
 
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
 import io.swagger.models.auth.In;
 import io.swagger.models.auth.In;
+import lombok.Data;
 
 
 import java.util.Date;
 import java.util.Date;
-
+@Data
+@TableName("ctop_dataeye_media")
 public class DataeyeMedia {
 public class DataeyeMedia {
-    private Integer id;
+    @TableId(value = "id", type = IdType.INPUT)
+    private Long mediaId;
     private String mediaName;
     private String mediaName;
-    private String logoUrl;
-    private String supportIos;
-    private String supportAndroid;
-    private String positionList;
-    private Integer isUnion;
-    private Integer isShow;
-    private Date createTime;
-    private Date updateTime;
+    private String logo;
+    private Integer companyNum;
+    private Integer productNum;
+    private Integer materialNum;
 }
 }

+ 15 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeMediaCompany.java

@@ -0,0 +1,15 @@
+package cn.com.ctop.crawler.modules.dataeye.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+@Data
+@TableName("ctop_dataeye_media_company")
+public class DataeyeMediaCompany {
+    @TableId(value = "id", type = IdType.INPUT)
+    private Long id;
+    private Long mediaId;
+    private Long companyId;
+}

+ 15 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeMediaProduct.java

@@ -0,0 +1,15 @@
+package cn.com.ctop.crawler.modules.dataeye.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+@Data
+@TableName("ctop_dataeye_media_product")
+public class DataeyeMediaProduct {
+    @TableId(value = "id", type = IdType.INPUT)
+    private Long id;
+    private Long MediaId;
+    private Long productId;
+}

+ 8 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyePlayRules.java

@@ -1,6 +1,14 @@
 package cn.com.ctop.crawler.modules.dataeye.entity;
 package cn.com.ctop.crawler.modules.dataeye.entity;
 
 
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+@Data
+@TableName("ctop_dataeye_play_rule")
 public class DataeyePlayRules {
 public class DataeyePlayRules {
+    @TableId(value = "id", type = IdType.INPUT)
     private Integer id;
     private Integer id;
     private String playRule;
     private String playRule;
     private Integer orderId;
     private Integer orderId;

+ 10 - 4
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeProduct.java

@@ -1,12 +1,20 @@
 package cn.com.ctop.crawler.modules.dataeye.entity;
 package cn.com.ctop.crawler.modules.dataeye.entity;
 
 
-import java.util.Date;
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
 
 
+import java.util.Date;
+import java.util.List;
+@Data
+@TableName("ctop_dataeye_product")
 public class DataeyeProduct {
 public class DataeyeProduct {
+    @TableId(value = "id", type = IdType.INPUT)
     private Integer productId;
     private Integer productId;
     private String icon;
     private String icon;
     private String productName;
     private String productName;
-    private Date firstSceen;
+    private Date firstSeen;
     private Date lastSeen;
     private Date lastSeen;
     private Integer days;
     private Integer days;
     private Integer type;
     private Integer type;
@@ -14,6 +22,4 @@ public class DataeyeProduct {
     private Integer mediaNum;
     private Integer mediaNum;
     private Integer positionNum;
     private Integer positionNum;
     private Integer materialNum;
     private Integer materialNum;
-    private String wxQrCode;
-
 }
 }

+ 15 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeProductAdArtStyle.java

@@ -0,0 +1,15 @@
+package cn.com.ctop.crawler.modules.dataeye.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+@Data
+@TableName("ctop_dataeye_product_ad_art_style")
+public class DataeyeProductAdArtStyle {
+    @TableId(value = "id", type = IdType.INPUT)
+    private Long id;
+    private Long productId;
+    private Long styleId;
+}

+ 15 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeProductPlayRule.java

@@ -0,0 +1,15 @@
+package cn.com.ctop.crawler.modules.dataeye.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+@Data
+@TableName("ctop_dataeye_product_play_rule")
+public class DataeyeProductPlayRule {
+    @TableId(value = "id", type = IdType.INPUT)
+    private Long id;
+    private Long productId;
+    private Long ruleId;
+}

+ 15 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeProductTheme.java

@@ -0,0 +1,15 @@
+package cn.com.ctop.crawler.modules.dataeye.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+@Data
+@TableName("ctop_dataeye_product_theme")
+public class DataeyeProductTheme {
+    @TableId(value = "id", type = IdType.INPUT)
+    private Long id;
+    private Long productId;
+    private Long themeId;
+}

+ 8 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeThemes.java

@@ -1,6 +1,14 @@
 package cn.com.ctop.crawler.modules.dataeye.entity;
 package cn.com.ctop.crawler.modules.dataeye.entity;
 
 
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+@Data
+@TableName("ctop_dataeye_theme")
 public class DataeyeThemes {
 public class DataeyeThemes {
+    @TableId(value = "id", type = IdType.INPUT)
     private Integer id;
     private Integer id;
     private String theme;
     private String theme;
     private Integer orderId;
     private Integer orderId;

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyeAdArtStyleMapper.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.dataeye.mapper;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeAdArtStyle;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+public interface DataeyeAdArtStyleMapper extends BaseMapper<DataeyeAdArtStyle> {
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyeCompanyProductMapper.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.dataeye.mapper;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeCompanyProduct;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+public interface DataeyeCompanyProductMapper extends BaseMapper<DataeyeCompanyProduct> {
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyeMediaMapper.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.dataeye.mapper;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeMedia;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+public interface DataeyeMediaMapper extends BaseMapper<DataeyeMedia> {
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyePlayRulesMapper.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.dataeye.mapper;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyePlayRules;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+public interface DataeyePlayRulesMapper extends BaseMapper<DataeyePlayRules> {
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyeProductAdArtStyleMapper.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.dataeye.mapper;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeProductAdArtStyle;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+public interface DataeyeProductAdArtStyleMapper extends BaseMapper<DataeyeProductAdArtStyle> {
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyeProductMapper.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.dataeye.mapper;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeProduct;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+public interface DataeyeProductMapper extends BaseMapper<DataeyeProduct> {
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyeProductPlayRuleMapper.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.dataeye.mapper;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeProductPlayRule;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+public interface DataeyeProductPlayRuleMapper extends BaseMapper<DataeyeProductPlayRule> {
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyeProductThemeMapper.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.dataeye.mapper;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeProductTheme;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+public interface DataeyeProductThemeMapper extends BaseMapper<DataeyeProductTheme> {
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyeThemesMapper.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.dataeye.mapper;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeThemes;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+public interface DataeyeThemesMapper extends BaseMapper<DataeyeThemes> {
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeAdArtStyleService.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.dataeye.service;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeAdArtStyle;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+public interface DataeyeAdArtStyleService extends IService<DataeyeAdArtStyle> {
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeCompanyProductService.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.dataeye.service;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeCompanyProduct;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+public interface DataeyeCompanyProductService extends IService<DataeyeCompanyProduct> {
+}

+ 4 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeLabelService.java

@@ -0,0 +1,4 @@
+package cn.com.ctop.crawler.modules.dataeye.service;
+
+public interface DataeyeLabelService {
+}

+ 8 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeMediaService.java

@@ -0,0 +1,8 @@
+package cn.com.ctop.crawler.modules.dataeye.service;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeMedia;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+public interface DataeyeMediaService extends IService<DataeyeMedia> {
+    public void getMediaList();
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyePlayRulesService.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.dataeye.service;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyePlayRules;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+public interface DataeyePlayRulesService extends IService<DataeyePlayRules> {
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeProductAdArtStyleService.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.dataeye.service;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeProductAdArtStyle;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+public interface DataeyeProductAdArtStyleService extends IService<DataeyeProductAdArtStyle> {
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeProductPlayRuleService.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.dataeye.service;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeProductPlayRule;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+public interface DataeyeProductPlayRuleService extends IService<DataeyeProductPlayRule> {
+}

+ 5 - 1
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeProductService.java

@@ -1,4 +1,8 @@
 package cn.com.ctop.crawler.modules.dataeye.service;
 package cn.com.ctop.crawler.modules.dataeye.service;
 
 
-public interface DataeyeProductService {
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeProduct;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+public interface DataeyeProductService extends IService<DataeyeProduct> {
+    public void getProductInfo(Long companyId,Long productId);
 }
 }

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeProductThemeService.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.dataeye.service;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeProductTheme;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+public interface DataeyeProductThemeService extends IService<DataeyeProductTheme> {
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeThemesService.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.dataeye.service;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeThemes;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+public interface DataeyeThemesService extends IService<DataeyeThemes> {
+}

+ 11 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeAdArtStyleServiceImpl.java

@@ -0,0 +1,11 @@
+package cn.com.ctop.crawler.modules.dataeye.service.impl;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeAdArtStyle;
+import cn.com.ctop.crawler.modules.dataeye.mapper.DataeyeAdArtStyleMapper;
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyeAdArtStyleService;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.springframework.stereotype.Service;
+
+@Service
+public class DataeyeAdArtStyleServiceImpl extends ServiceImpl<DataeyeAdArtStyleMapper, DataeyeAdArtStyle> implements DataeyeAdArtStyleService {
+}

+ 11 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeCompanyProductServiceImpl.java

@@ -0,0 +1,11 @@
+package cn.com.ctop.crawler.modules.dataeye.service.impl;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeCompanyProduct;
+import cn.com.ctop.crawler.modules.dataeye.mapper.DataeyeCompanyProductMapper;
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyeCompanyProductService;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.springframework.stereotype.Service;
+
+@Service
+public class DataeyeCompanyProductServiceImpl extends ServiceImpl<DataeyeCompanyProductMapper, DataeyeCompanyProduct> implements DataeyeCompanyProductService {
+}

+ 30 - 15
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeCompanyServiceImpl.java

@@ -5,6 +5,7 @@ import cn.com.ctop.crawler.modules.core.util.SignUtil;
 import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeCompany;
 import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeCompany;
 import cn.com.ctop.crawler.modules.dataeye.mapper.DataeyeCompanyMapper;
 import cn.com.ctop.crawler.modules.dataeye.mapper.DataeyeCompanyMapper;
 import cn.com.ctop.crawler.modules.dataeye.service.DataeyeCompanyService;
 import cn.com.ctop.crawler.modules.dataeye.service.DataeyeCompanyService;
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyeProductService;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.JsonNode;
@@ -13,6 +14,7 @@ import com.google.gson.Gson;
 import org.apache.http.cookie.Cookie;
 import org.apache.http.cookie.Cookie;
 import org.apache.http.impl.cookie.BasicClientCookie;
 import org.apache.http.impl.cookie.BasicClientCookie;
 import org.jeecg.common.util.DateUtils;
 import org.jeecg.common.util.DateUtils;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.springframework.stereotype.Service;
 import sun.misc.BASE64Encoder;
 import sun.misc.BASE64Encoder;
 
 
@@ -22,26 +24,17 @@ import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.security.NoSuchAlgorithmException;
 import java.util.Date;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.Map;
 import java.util.Map;
 @Service
 @Service
 public class DataeyeCompanyServiceImpl extends ServiceImpl<DataeyeCompanyMapper, DataeyeCompany> implements DataeyeCompanyService {
 public class DataeyeCompanyServiceImpl extends ServiceImpl<DataeyeCompanyMapper, DataeyeCompany> implements DataeyeCompanyService {
-
+    @Autowired
+    private DataeyeProductService dataeyeProductService;
     public void getCompanyInfo(Integer companyId){
     public void getCompanyInfo(Integer companyId){
-        Map<String,String> headerMap = new HashMap<>();
-        headerMap.put("Host","adx.dataeye.com");
-        headerMap.put("Origin","https://adx.dataeye.com");
-        headerMap.put("Referer","https://adx.dataeye.com/");
-        BasicClientCookie ck = new BasicClientCookie("tokenglobal", "721501ec4ec04a50bbffc0c0240cbbdd63a0d5456205b9a5ea1581ce6c6554ca");
-        ck.setDomain("adx.dataeye.com");
-        ck.setExpiryDate(DateUtils.addDay(new Date(),365));
-        ck.setPath("/");
-        HttpUtils.COOKIESTORE.addCookie(ck);
         Map<String,Object> paramMap = new HashMap<>();
         Map<String,Object> paramMap = new HashMap<>();
         paramMap.put("companyId",companyId);
         paramMap.put("companyId",companyId);
-        paramMap.put("thisTimes",(long)(new Date().getTime()/100));
-        String sign = SignUtil.getDataeyeSign(paramMap);
-        paramMap.put("sign",sign);
-        String response = HttpUtils.httpPostFormRequest("https://adx.dataeye.com/company/getCompanyInfo",paramMap,headerMap);
+        String response = HttpUtils.httpPostFormRequest("https://blzs.dataeye.com/company/getCompanyInfo",paramMap,null);
+        System.out.println(response);
         ObjectMapper mapper = new ObjectMapper();
         ObjectMapper mapper = new ObjectMapper();
         mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
         mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
         try {
         try {
@@ -50,7 +43,29 @@ public class DataeyeCompanyServiceImpl extends ServiceImpl<DataeyeCompanyMapper,
             if (statusCode == 200){
             if (statusCode == 200){
                 String content = rootNode.get("content").toString();
                 String content = rootNode.get("content").toString();
                 DataeyeCompany company = mapper.readValue(content,DataeyeCompany.class);
                 DataeyeCompany company = mapper.readValue(content,DataeyeCompany.class);
-                this.save(company);
+                if(company != null && company.getCompanyName() != null){
+                    this.saveOrUpdate(company);
+                    String result = HttpUtils.httpPostFormRequest("https://blzs.dataeye.com/company/listProductDistribution?companyId="+companyId,null,null);
+                    System.out.println(result);
+                    rootNode = mapper.readTree(result);
+                    statusCode = rootNode.get("statusCode").asInt();
+                    if (statusCode == 200) {
+                        Iterator<JsonNode> nodes = rootNode.get("content").elements();
+                        while (nodes.hasNext()){
+                            JsonNode node = nodes.next();
+                            Long productId = node.get("id").asLong();
+//                            Map<String,Long> map = new HashMap<>();
+//                            map.put("companyId",companyId.longValue());
+//                            map.put("productId",productId);
+//                            try {
+//                                DataeyeProductServiceImpl.taskQueue.put(map);
+//                            } catch (InterruptedException e) {
+//                                e.printStackTrace();
+//                            }
+                            dataeyeProductService.getProductInfo(companyId.longValue(),productId);
+                        }
+                    }
+                }
             }
             }
         } catch (IOException e) {
         } catch (IOException e) {
             e.printStackTrace();
             e.printStackTrace();

+ 62 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeLabelServiceImpl.java

@@ -0,0 +1,62 @@
+package cn.com.ctop.crawler.modules.dataeye.service.impl;
+
+import cn.com.ctop.common.module.utils.HttpUtils;
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeAdArtStyle;
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeCompany;
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyePlayRules;
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeThemes;
+import cn.com.ctop.crawler.modules.dataeye.mapper.DataeyeAdArtStyleMapper;
+import cn.com.ctop.crawler.modules.dataeye.mapper.DataeyePlayRulesMapper;
+import cn.com.ctop.crawler.modules.dataeye.mapper.DataeyeThemesMapper;
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyeAdArtStyleService;
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyeLabelService;
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyePlayRulesService;
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyeThemesService;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.io.IOException;
+import java.util.List;
+
+@Service
+public class DataeyeLabelServiceImpl implements DataeyeLabelService {
+    @Autowired
+    private DataeyeAdArtStyleService dataeyeAdArtStyleService;
+    @Autowired
+    private DataeyeThemesService dataeyeThemesService;
+    @Autowired
+    private DataeyePlayRulesService dataeyePlayRulesService;
+
+    public void getLabelList(){
+        String response = HttpUtils.httpPostFormRequest("https://blzs.dataeye.com/common/getNewLabelMenu",null,null);
+        System.out.println(response);
+        ObjectMapper mapper = new ObjectMapper();
+        mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
+        try {
+            JsonNode rootNode = mapper.readTree(response);
+            int statusCode = rootNode.get("statusCode").asInt();
+            if (statusCode == 200){
+                String adThemeList = rootNode.get("content").get("adThemeList").toString();
+                String adPlayRuleList = rootNode.get("content").get("adPlayRuleList").toString();
+                String adArtStyleList = rootNode.get("content").get("adArtStyleList").toString();
+                List<DataeyeAdArtStyle> adArtStyles = mapper.readValue(adArtStyleList, new TypeReference<List<DataeyeAdArtStyle>>(){});
+                List<DataeyeThemes> themes = mapper.readValue(adThemeList,new TypeReference<List<DataeyeThemes>>(){});
+                List<DataeyePlayRules> playRules = mapper.readValue(adPlayRuleList,new TypeReference<List<DataeyePlayRules>>(){});
+                dataeyeAdArtStyleService.saveOrUpdateBatch(adArtStyles);
+                dataeyeThemesService.saveOrUpdateBatch(themes);
+                dataeyePlayRulesService.saveOrUpdateBatch(playRules);
+//                DataeyeCompany company = mapper.readValue(content,DataeyeCompany.class);
+//                if(company != null && company.getCompanyName() != null){
+//                    this.save(company);
+//                }
+
+            }
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+}

+ 73 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeMediaServiceImpl.java

@@ -0,0 +1,73 @@
+package cn.com.ctop.crawler.modules.dataeye.service.impl;
+
+import cn.com.ctop.common.module.utils.HttpUtils;
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeCompany;
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeMedia;
+import cn.com.ctop.crawler.modules.dataeye.mapper.DataeyeMediaMapper;
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyeCompanyService;
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyeMediaService;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.swagger.models.auth.In;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.io.IOException;
+import java.util.*;
+
+@Service
+public class DataeyeMediaServiceImpl extends ServiceImpl<DataeyeMediaMapper, DataeyeMedia> implements DataeyeMediaService {
+    @Autowired
+    private DataeyeCompanyService dataeyeCompanyService;
+    public void getMediaList(){
+        String response = HttpUtils.httpPostFormRequest("https://blzs.dataeye.com/common/listMedia",null,null);
+        System.out.println(response);
+        ObjectMapper mapper = new ObjectMapper();
+        mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
+        try {
+            JsonNode rootNode = mapper.readTree(response);
+            int statusCode = rootNode.get("statusCode").asInt();
+            if (statusCode == 200){
+                Iterator<JsonNode> nodes = rootNode.get("content").elements();
+                List<Integer> ids = new ArrayList<>();
+                while (nodes.hasNext()){
+                    JsonNode node =  nodes.next();
+                    Integer mediaId = node.get("k").asInt();
+                    ids.add(mediaId);
+                    response = HttpUtils.httpPostFormRequest("https://blzs.dataeye.com/media/getMediaInfo?mediaId="+mediaId,null,null);
+                    System.out.println(response);
+                    rootNode = mapper.readTree(response);
+                    statusCode = rootNode.get("statusCode").asInt();
+                    if (statusCode == 200){
+                        String content = rootNode.get("content").toString();
+                        DataeyeMedia media = mapper.readValue(content,DataeyeMedia.class);
+                        if(media != null && media.getMediaId() != null){
+                            this.saveOrUpdate(media);
+
+                        }
+                    }
+                }
+                if(ids != null){
+                    for (Integer mediaId :ids){
+                        response = HttpUtils.httpPostFormRequest("https://blzs.dataeye.com/media/listCompanyDistribution?mediaId="+mediaId,null,null);
+                        System.out.println(response);
+                        rootNode = mapper.readTree(response);
+                        statusCode = rootNode.get("statusCode").asInt();
+                        if (statusCode == 200){
+                            Iterator<JsonNode> companyNodes = rootNode.get("content").elements();
+                            while (companyNodes.hasNext()){
+                                JsonNode companyNode = companyNodes.next();
+                                Integer companyId = companyNode.get("id").asInt();
+                                dataeyeCompanyService.getCompanyInfo(companyId);
+                            }
+                        }
+                    }
+                }
+            }
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+}

+ 11 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyePlayRulesServiceImpl.java

@@ -0,0 +1,11 @@
+package cn.com.ctop.crawler.modules.dataeye.service.impl;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyePlayRules;
+import cn.com.ctop.crawler.modules.dataeye.mapper.DataeyePlayRulesMapper;
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyePlayRulesService;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.springframework.stereotype.Service;
+
+@Service
+public class DataeyePlayRulesServiceImpl extends ServiceImpl<DataeyePlayRulesMapper, DataeyePlayRules> implements DataeyePlayRulesService{
+}

+ 13 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeProductAdArtStyleServiceImpl.java

@@ -0,0 +1,13 @@
+package cn.com.ctop.crawler.modules.dataeye.service.impl;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeProductAdArtStyle;
+import cn.com.ctop.crawler.modules.dataeye.mapper.DataeyeProductAdArtStyleMapper;
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyeAdArtStyleService;
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyePlayRulesService;
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyeProductAdArtStyleService;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.springframework.stereotype.Service;
+
+@Service
+public class DataeyeProductAdArtStyleServiceImpl extends ServiceImpl<DataeyeProductAdArtStyleMapper, DataeyeProductAdArtStyle> implements DataeyeProductAdArtStyleService {
+}

+ 11 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeProductPlayRuleServiceImpl.java

@@ -0,0 +1,11 @@
+package cn.com.ctop.crawler.modules.dataeye.service.impl;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeProductPlayRule;
+import cn.com.ctop.crawler.modules.dataeye.mapper.DataeyeProductPlayRuleMapper;
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyeProductPlayRuleService;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.springframework.stereotype.Service;
+
+@Service
+public class DataeyeProductPlayRuleServiceImpl extends ServiceImpl<DataeyeProductPlayRuleMapper, DataeyeProductPlayRule> implements DataeyeProductPlayRuleService{
+}

+ 107 - 2
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeProductServiceImpl.java

@@ -1,8 +1,113 @@
 package cn.com.ctop.crawler.modules.dataeye.service.impl;
 package cn.com.ctop.crawler.modules.dataeye.service.impl;
 
 
-import cn.com.ctop.crawler.modules.dataeye.service.DataeyeProductService;
+import cn.com.ctop.common.module.utils.HttpUtils;
+import cn.com.ctop.crawler.modules.dataeye.entity.*;
+import cn.com.ctop.crawler.modules.dataeye.mapper.DataeyeProductMapper;
+import cn.com.ctop.crawler.modules.dataeye.service.*;
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.springframework.stereotype.Service;
+import us.codecraft.webmagic.thread.CountableThreadPool;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.logging.Logger;
 
 
 @Service
 @Service
-public class DataeyeProductServiceImpl implements DataeyeProductService {
+public class DataeyeProductServiceImpl extends ServiceImpl<DataeyeProductMapper, DataeyeProduct> implements DataeyeProductService {
+    @Autowired
+    private DataeyeProductThemeService dataeyeProductThemeService;
+    @Autowired
+    private DataeyeProductPlayRuleService dataeyeProductPlayRuleService;
+    @Autowired
+    private DataeyeProductAdArtStyleService dataeyeProductAdArtStyleService;
+    @Autowired
+    private DataeyeCompanyProductService dataeyeCompanyProductService;
+
+    public void getProductInfo(Long companyId,Long productId){
+        Map<String,Object> paramMap = new HashMap<>();
+        paramMap.put("productId",productId);
+        String response = HttpUtils.httpPostFormRequest("https://blzs.dataeye.com/product/getProductInfo",paramMap,null);
+        System.out.println(response);
+        ObjectMapper mapper = new ObjectMapper();
+        mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
+        try {
+            JsonNode rootNode = mapper.readTree(response);
+            int statusCode = rootNode.get("statusCode").asInt();
+            if (statusCode == 200){
+                String content = rootNode.get("content").toString();
+                DataeyeProduct product = mapper.readValue(content,DataeyeProduct.class);
+                if(product != null && product.getProductName() != null && !product.getProductName().equals("")){
+                    this.saveOrUpdate(product);
+                }
+                QueryWrapper<DataeyeCompanyProduct> queryWrapper1 = new QueryWrapper<>();
+                queryWrapper1.eq("company_id",companyId);
+                queryWrapper1.eq("product_id",productId);
+                DataeyeCompanyProduct dataeyeCompanyProduct = dataeyeCompanyProductService.getOne(queryWrapper1);
+                if(dataeyeCompanyProduct == null){
+                    dataeyeCompanyProduct = new DataeyeCompanyProduct();
+                    dataeyeCompanyProduct.setCompanyId(companyId);
+                    dataeyeCompanyProduct.setProductId(productId);
+                    dataeyeCompanyProductService.save(dataeyeCompanyProduct);
+                }
+                Iterator<JsonNode> themeNodes = rootNode.get("content").get("themes").elements();
+                Iterator<JsonNode> ruleNodes = rootNode.get("content").get("playRules").elements();
+                Iterator<JsonNode> styleNodes = rootNode.get("content").get("adArtStyles").elements();
+                while (themeNodes.hasNext()){
+                    JsonNode node = themeNodes.next();
+                    Long themeId = node.get("id").asLong();
+                    QueryWrapper<DataeyeProductTheme> queryWrapper = new QueryWrapper<>();
+                    queryWrapper.eq("product_id",productId);
+                    queryWrapper.eq("theme_id",themeId);
+                    DataeyeProductTheme theme = dataeyeProductThemeService.getOne(queryWrapper);
+                    if (theme == null){
+                        theme = new DataeyeProductTheme();
+                        theme.setProductId(productId.longValue());
+                        theme.setThemeId(themeId);
+                        dataeyeProductThemeService.save(theme);
+                    }
+                }
+                while (ruleNodes.hasNext()){
+                    JsonNode node = ruleNodes.next();
+                    Long ruleId = node.get("id").asLong();
+                    QueryWrapper<DataeyeProductPlayRule> queryWrapper = new QueryWrapper<>();
+                    queryWrapper.eq("product_id",productId);
+                    queryWrapper.eq("rule_id",ruleId);
+                    DataeyeProductPlayRule rule = dataeyeProductPlayRuleService.getOne(queryWrapper);
+                    if (rule == null){
+                        rule = new DataeyeProductPlayRule();
+                        rule.setProductId(productId.longValue());
+                        rule.setRuleId(ruleId);
+                        dataeyeProductPlayRuleService.save(rule);
+                    }
+                }
+                while (styleNodes.hasNext()){
+                    JsonNode node = styleNodes.next();
+                    Long styleId = node.get("id").asLong();
+                    QueryWrapper<DataeyeProductAdArtStyle> queryWrapper = new QueryWrapper<>();
+                    queryWrapper.eq("product_id",productId);
+                    queryWrapper.eq("style_id",styleId);
+                    DataeyeProductAdArtStyle style = dataeyeProductAdArtStyleService.getOne(queryWrapper);
+                    if (style == null){
+                        style = new DataeyeProductAdArtStyle();
+                        style.setProductId(productId.longValue());
+                        style.setStyleId(styleId);
+                        dataeyeProductAdArtStyleService.save(style);
+                    }
+                }
+            }
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
 }
 }

+ 11 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeProductThemeServiceImpl.java

@@ -0,0 +1,11 @@
+package cn.com.ctop.crawler.modules.dataeye.service.impl;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeProductTheme;
+import cn.com.ctop.crawler.modules.dataeye.mapper.DataeyeProductThemeMapper;
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyeProductThemeService;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.springframework.stereotype.Service;
+
+@Service
+public class DataeyeProductThemeServiceImpl extends ServiceImpl<DataeyeProductThemeMapper, DataeyeProductTheme> implements DataeyeProductThemeService {
+}

+ 12 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeThemesServiceImpl.java

@@ -0,0 +1,12 @@
+package cn.com.ctop.crawler.modules.dataeye.service.impl;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeThemes;
+import cn.com.ctop.crawler.modules.dataeye.mapper.DataeyeCompanyMapper;
+import cn.com.ctop.crawler.modules.dataeye.mapper.DataeyeThemesMapper;
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyeThemesService;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.springframework.stereotype.Service;
+
+@Service
+public class DataeyeThemesServiceImpl extends ServiceImpl<DataeyeThemesMapper, DataeyeThemes> implements DataeyeThemesService {
+}