瀏覽代碼

变色龙数据获取

xuzuoyun 5 年之前
父節點
當前提交
a2ed6e3607

+ 13 - 5
module-common/src/main/java/cn/com/ctop/common/module/utils/HttpUtils2.java

@@ -144,9 +144,12 @@ public class HttpUtils2 {
             HttpPost httppost = new HttpPost(url);
             httppost.setHeader("User-Agent", USER_AGENT);
             httppost.addHeader("Content-Type", "application/json");
-            for (String key : headers.keySet()) {
-                httppost.setHeader(key, headers.get(key));
+            if(headers != null){
+                for (String key : headers.keySet()) {
+                    httppost.setHeader(key, headers.get(key));
+                }
             }
+
             httppost.setEntity(new StringEntity(new Gson().toJson(param), Charset.forName("UTF-8")));
             HttpEntity respentity;
 
@@ -237,10 +240,15 @@ public class HttpUtils2 {
             HttpPost httppost = new HttpPost(url);
             httppost.setHeader("User-Agent", USER_AGENT);
             httppost.addHeader("Content-Type", "application/json");
-            for (String key : headers.keySet()) {
-                httppost.setHeader(key, headers.get(key));
+            if(headers != null){
+                for (String key : headers.keySet()) {
+                    httppost.setHeader(key, headers.get(key));
+                }
+            }
+            if(params != null){
+                httppost.setEntity(new StringEntity(params.toJSONString(), Charset.forName("UTF-8")));
             }
-            httppost.setEntity(new StringEntity(params.toJSONString(), Charset.forName("UTF-8")));
+
             HttpEntity respentity;
 
             HttpResponse response = httpClient.execute(httppost);

+ 6 - 0
module-crawler/pom.xml

@@ -22,6 +22,12 @@
             <artifactId>jeecg-boot-base-common</artifactId>
             <version>2.0.2</version>
         </dependency>
+        <dependency>
+            <groupId>org.codehaus.jackson</groupId>
+            <artifactId>jackson-core-asl</artifactId>
+            <version>1.9.11</version>
+            <scope>compile</scope>
+        </dependency>
     </dependencies>
     <build>
         <resources>

+ 1 - 1
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/entity/PangolinReportAccountDaily.java

@@ -1,6 +1,7 @@
 package cn.com.ctop.crawler.modules.pangolin.entity;
 
 import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableField;
 import com.baomidou.mybatisplus.annotation.TableId;
 import com.baomidou.mybatisplus.annotation.TableName;
 import lombok.Data;
@@ -12,7 +13,6 @@ import java.util.Date;
 @TableName("ctop_pangolin_report_account_daily")
 public class PangolinReportAccountDaily {
     private Date date;
-    @TableId(value = "id", type = IdType.INPUT)
     private String accountName;
     private String appName;
     private Integer externalActivation;

+ 20 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/entity/PangolinReportAppHour.java

@@ -0,0 +1,20 @@
+package cn.com.ctop.crawler.modules.pangolin.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+import java.math.BigDecimal;
+import java.util.Date;
+
+@Data
+@TableName("ctop_pangolin_report_app_hour")
+public class PangolinReportAppHour {
+    private Date date;
+    private Integer hour;
+    private Long appId;
+    private String type;
+    private BigDecimal num;
+}

+ 2 - 2
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/entity/PangolinReportBillMonth.java

@@ -1,6 +1,7 @@
 package cn.com.ctop.crawler.modules.pangolin.entity;
 
 import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableField;
 import com.baomidou.mybatisplus.annotation.TableId;
 import com.baomidou.mybatisplus.annotation.TableName;
 import lombok.Data;
@@ -12,8 +13,7 @@ import java.math.BigDecimal;
 public class PangolinReportBillMonth {
     private Integer year;
     private Integer month;
-    @TableId(value = "id", type = IdType.INPUT)
-    private Integer appId;
+    private Long appId;
     private String os;
     private Integer sumInternalActivations;
     private BigDecimal actualTotalPrice;

+ 1 - 1
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/entity/PangolinReportChannelDaily.java

@@ -1,6 +1,7 @@
 package cn.com.ctop.crawler.modules.pangolin.entity;
 
 import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableField;
 import com.baomidou.mybatisplus.annotation.TableId;
 import com.baomidou.mybatisplus.annotation.TableName;
 import lombok.Data;
@@ -12,7 +13,6 @@ import java.util.Date;
 @TableName("ctop_pangolin_report_channel_daily")
 public class PangolinReportChannelDaily {
     private Date date;
-    @TableId(value = "id", type = IdType.INPUT)
     private String channelName;
     private String appName;
     private String os;

+ 46 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/job/PangolinDataJob.java

@@ -0,0 +1,46 @@
+package cn.com.ctop.crawler.modules.pangolin.job;
+
+import cn.com.ctop.common.module.entity.BindAccountLogin;
+import cn.com.ctop.common.module.service.IBindAccountLoginService;
+import cn.com.ctop.crawler.modules.pangolin.entity.PangolinApp;
+import cn.com.ctop.crawler.modules.pangolin.service.*;
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import org.jeecg.common.util.DateUtils;
+import org.quartz.Job;
+import org.quartz.JobExecutionContext;
+import org.quartz.JobExecutionException;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.util.List;
+
+public class PangolinDataJob implements Job {
+    @Autowired
+    private IBindAccountLoginService bindAccountLoginService;
+    @Autowired
+    private PangolinAppService pangolinAppService;
+    @Autowired
+    private PangolinCrawlerService pangolinCrawlerService;
+    @Override
+    public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
+        QueryWrapper<BindAccountLogin> queryWrapper = new QueryWrapper<>();
+        queryWrapper.eq("login_type","pangolin");
+        queryWrapper.eq("status",1);
+        queryWrapper.isNotNull("cookie");
+        List<BindAccountLogin> list = bindAccountLoginService.list(queryWrapper);
+        if (list != null && list.size() > 0){
+            for (BindAccountLogin bindAccountLogin:list){
+                pangolinCrawlerService.getChannelList(bindAccountLogin);
+                pangolinCrawlerService.getActivationList(bindAccountLogin, DateUtils.getNowDate("yyyy-MM-dd"),1);
+                pangolinCrawlerService.getCheckList(bindAccountLogin);
+                QueryWrapper<PangolinApp> queryWrapper1 = new QueryWrapper<>();
+                queryWrapper1.eq("account_name",bindAccountLogin.getAccountName());
+                List<PangolinApp> appList = pangolinAppService.list(queryWrapper1);
+                if(appList != null && appList.size() > 0){
+                    for (PangolinApp app : appList){
+                        pangolinCrawlerService.getRealTimeList(bindAccountLogin,app.getId());
+                    }
+                }
+            }
+        }
+    }
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/PangolinReportAppHourMapper.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.pangolin.mapper;
+
+import cn.com.ctop.crawler.modules.pangolin.entity.PangolinReportAppHour;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+public interface PangolinReportAppHourMapper extends BaseMapper<PangolinReportAppHour> {
+}

+ 11 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/PangolinCrawlerService.java

@@ -0,0 +1,11 @@
+package cn.com.ctop.crawler.modules.pangolin.service;
+
+import cn.com.ctop.common.module.entity.BindAccountLogin;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+public interface PangolinCrawlerService {
+    public void getCheckList(BindAccountLogin bindAccountLogin);
+    public void getChannelList(BindAccountLogin bindAccountLogin);
+    public void getActivationList(BindAccountLogin bindAccountLogin,String date,int page);
+    public void getRealTimeList(BindAccountLogin bindAccountLogin,Long appId);
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/PangolinReportAppHourService.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.pangolin.service;
+
+import cn.com.ctop.crawler.modules.pangolin.entity.PangolinReportAppHour;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+public interface PangolinReportAppHourService extends IService<PangolinReportAppHour> {
+}

+ 353 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinCrawlerServiceImpl.java

@@ -0,0 +1,353 @@
+package cn.com.ctop.crawler.modules.pangolin.service.impl;
+
+import cn.com.ctop.common.module.entity.BindAccountLogin;
+import cn.com.ctop.common.module.service.IBindAccountLoginService;
+import cn.com.ctop.common.module.utils.HttpUtils2;
+import cn.com.ctop.crawler.modules.pangolin.entity.*;
+import cn.com.ctop.crawler.modules.pangolin.service.*;
+import cn.hutool.core.date.DateUtil;
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.gson.Gson;
+import com.google.gson.JsonObject;
+import org.apache.http.impl.cookie.BasicClientCookie;
+import org.jeecg.common.util.DateUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.util.*;
+
+@Service
+public class PangolinCrawlerServiceImpl implements PangolinCrawlerService {
+    @Autowired
+    private IBindAccountLoginService bindAccountLoginService;
+    @Autowired
+    private PangolinAppService pangolinAppService;
+    @Autowired
+    private PangolinChannelService pangolinChannelService;
+    @Autowired
+    private PangolinReportBillMonthService pangolinReportBillMonthService;
+    @Autowired
+    private PangolinReportChannelDailyService pangolinReportChannelDailyService;
+    @Autowired
+    private PangolinReportAppHourService pangolinReportAppHourService;
+
+    public void getCheckList(BindAccountLogin bindAccountLogin){
+        String url = "https://pangolin.bytedance.com/api/check_apps/list";
+        ObjectMapper mapper = new ObjectMapper();
+        try {
+            List<String> cookieList = mapper.readValue(bindAccountLogin.getCookie(), new TypeReference<List<String>>() {});
+            if(cookieList != null && cookieList.size() > 0){
+                for (String cookieStr : cookieList){
+                    String[] cookieArray = cookieStr.split(";");
+                    Map<String, String> cookieMap = new HashMap<String, String>();
+                    for (String cookie : cookieArray) {
+                        String[] kv = cookie.split("=");
+                        if (kv.length > 1) {
+                            cookieMap.put(kv[0].trim(), kv[1].trim());
+                        }
+                    }
+                    BasicClientCookie clientCookie = new BasicClientCookie(cookieArray[0].split("=")[0],
+                            cookieArray[0].split("=")[1]);
+                    clientCookie.setDomain("pangolin.bytedance.com");
+                    clientCookie.setPath(cookieMap.get("path"));
+                    HttpUtils2.cookieStore.addCookie(clientCookie);
+                }
+            }
+            String result = HttpUtils2.httpPostRequest(url,null,null);
+            System.out.println(result);
+            if(checkLogin(bindAccountLogin,result)){
+                JsonNode resultNode = mapper.readTree(result);
+                Iterator<JsonNode> appIterator = resultNode.get("data").get("List").elements();
+                while (appIterator.hasNext()){
+                    JsonNode checkNode = appIterator.next();
+                    Long appId = checkNode.get("ApplicationId").asLong();
+                    String date = checkNode.get("Date").asText();
+                    String os = checkNode.get("Os").asText();
+                    Integer sumInternalActivations = checkNode.get("SumInternalActivations").asInt();
+                    BigDecimal actualTotalPrice = checkNode.get("ActualTotalPrice").decimalValue();
+                    BigDecimal estimatedTotalPrice = checkNode.get("EstimatedTotalPrice").decimalValue();
+                    BigDecimal accountQualityScore = checkNode.get("AccountQualityScore").decimalValue();
+                    String appName = checkNode.get("AppName").asText();
+                    PangolinReportBillMonth pangolinReportBillMonth = new PangolinReportBillMonth();
+                    pangolinReportBillMonth.setAccountQualityScore(accountQualityScore);
+                    pangolinReportBillMonth.setActualTotalPrice(actualTotalPrice);
+                    pangolinReportBillMonth.setAppId(appId);
+                    pangolinReportBillMonth.setEstimatedTotalPrice(estimatedTotalPrice);
+                    pangolinReportBillMonth.setOs(os);
+                    pangolinReportBillMonth.setSumInternalActivations(sumInternalActivations);
+                    pangolinReportBillMonth.setYear(Integer.parseInt(date.split("-")[0]));
+                    pangolinReportBillMonth.setMonth(Integer.parseInt(date.split("-")[1]));
+                    QueryWrapper<PangolinReportBillMonth> queryWrapper = new QueryWrapper<>();
+                    queryWrapper.eq("os",pangolinReportBillMonth.getOs());
+                    queryWrapper.eq("app_id",pangolinReportBillMonth.getAppId());
+                    queryWrapper.eq("month",pangolinReportBillMonth.getMonth());
+                    queryWrapper.eq("year",pangolinReportBillMonth.getYear());
+                    if(pangolinReportBillMonthService.getOne(queryWrapper) == null){
+                        pangolinReportBillMonthService.save(pangolinReportBillMonth);
+                    }else {
+                        pangolinReportBillMonthService.update(pangolinReportBillMonth,queryWrapper);
+                    }
+                }
+            }
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+
+    public void getChannelList(BindAccountLogin bindAccountLogin){
+        String url = "https://pangolin.bytedance.com/api/search/external_default";
+        ObjectMapper mapper = new ObjectMapper();
+        try {
+            List<String> cookieList = mapper.readValue(bindAccountLogin.getCookie(), new TypeReference<List<String>>() {});
+            if(cookieList != null && cookieList.size() > 0){
+                for (String cookieStr : cookieList){
+                    String[] cookieArray = cookieStr.split(";");
+                    Map<String, String> cookieMap = new HashMap<String, String>();
+                    for (String cookie : cookieArray) {
+                        String[] kv = cookie.split("=");
+                        if (kv.length > 1) {
+                            cookieMap.put(kv[0].trim(), kv[1].trim());
+                        }
+                    }
+                    BasicClientCookie clientCookie = new BasicClientCookie(cookieArray[0].split("=")[0],
+                            cookieArray[0].split("=")[1]);
+                    clientCookie.setDomain("pangolin.bytedance.com");
+                    clientCookie.setPath(cookieMap.get("path"));
+                    HttpUtils2.cookieStore.addCookie(clientCookie);
+                }
+            }
+            String result = HttpUtils2.httpGetRequest(url);
+            System.out.println(result);
+            if(checkLogin(bindAccountLogin,result)){
+                JsonNode resultNode = mapper.readTree(result);
+                Iterator<JsonNode> appIterator = resultNode.get("data").get("AppNameList").elements();
+                while (appIterator.hasNext()){
+                    JsonNode appNode = appIterator.next();
+                    Long id = appNode.get("Id").asLong();
+                    String name = appNode.get("Name").asText();
+                    PangolinApp app = new PangolinApp();
+                    app.setAccountName(bindAccountLogin.getAccountName());
+                    app.setId(id);
+                    app.setName(name);
+                    pangolinAppService.saveOrUpdate(app);
+                }
+                Iterator<JsonNode> channelIterator = resultNode.get("data").get("ChannelNameList").elements();
+                while (channelIterator.hasNext()){
+                    JsonNode channelNode = channelIterator.next();
+                    Long id = channelNode.get("Id").asLong();
+                    String name = channelNode.get("Name").asText();
+                    PangolinChannel channel = new PangolinChannel();
+                    channel.setAccountName(bindAccountLogin.getAccountName());
+                    channel.setId(id);
+                    channel.setName(name);
+                    pangolinChannelService.saveOrUpdate(channel);
+                }
+            }
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+
+    public void getActivationList(BindAccountLogin bindAccountLogin,String date,int page){
+        String url = "https://pangolin.bytedance.com/api/activation/list";
+        ObjectMapper mapper = new ObjectMapper();
+        try {
+            List<String> cookieList = mapper.readValue(bindAccountLogin.getCookie(), new TypeReference<List<String>>() {});
+            if(cookieList != null && cookieList.size() > 0){
+                for (String cookieStr : cookieList){
+                    String[] cookieArray = cookieStr.split(";");
+                    Map<String, String> cookieMap = new HashMap<String, String>();
+                    for (String cookie : cookieArray) {
+                        String[] kv = cookie.split("=");
+                        if (kv.length > 1) {
+                            cookieMap.put(kv[0].trim(), kv[1].trim());
+                        }
+                    }
+                    BasicClientCookie clientCookie = new BasicClientCookie(cookieArray[0].split("=")[0],
+                            cookieArray[0].split("=")[1]);
+                    clientCookie.setDomain("pangolin.bytedance.com");
+                    clientCookie.setPath(cookieMap.get("path"));
+                    HttpUtils2.cookieStore.addCookie(clientCookie);
+                }
+            }
+            Map<String,Object> paramMap = new HashMap<>();
+            Map<String,Object> searchMap = new HashMap<>();
+            searchMap.put("DateStart",DateUtils.addDay(date,-2));
+            searchMap.put("DateEnd",date);
+            searchMap.put("AppId","");
+            searchMap.put("ChannelName","");
+            searchMap.put("OS","");
+            Map<String,Object> pageMap = new HashMap<>();
+            pageMap.put("p",page);
+            paramMap.put("search",searchMap);
+            paramMap.put("page",pageMap);
+            String result = HttpUtils2.httpPostRequest(url,paramMap,null);
+            System.out.println(result);
+            if(checkLogin(bindAccountLogin,result)){
+                JsonNode resultNode = mapper.readTree(result);
+                Iterator<JsonNode> listIterator = resultNode.get("data").get("list").elements();
+                while (listIterator.hasNext()){
+                    JsonNode dataNode = listIterator.next();
+                    String startDate = dataNode.get("StatsDate").asText();
+                    String channelName = dataNode.get("ChannelName").asText();
+                    String appName = dataNode.get("AppVerboseName").asText();
+                    String os = dataNode.get("OS").asText();
+                    Integer externalActivation = dataNode.get("ExternalActivation").asInt();
+                    BigDecimal retentionRatio = dataNode.get("RetentionRatio").decimalValue();
+                    BigDecimal totalPrice = dataNode.get("TotalPrice").decimalValue();
+                    PangolinReportChannelDaily pangolinReportChannelDaily = new PangolinReportChannelDaily();
+                    pangolinReportChannelDaily.setAppName(appName);
+                    pangolinReportChannelDaily.setChannelName(channelName);
+                    pangolinReportChannelDaily.setDate(DateUtils.parseDate(startDate,"yyyy-MM-dd"));
+                    pangolinReportChannelDaily.setExternalActivation(externalActivation);
+                    pangolinReportChannelDaily.setOs(os);
+                    pangolinReportChannelDaily.setRetentionRatio(retentionRatio);
+                    pangolinReportChannelDaily.setTotalPrice(totalPrice);
+                    QueryWrapper<PangolinReportChannelDaily> queryWrapper = new QueryWrapper<>();
+                    queryWrapper.eq("date",pangolinReportChannelDaily.getDate());
+                    queryWrapper.eq("channel_name",pangolinReportChannelDaily.getChannelName());
+                    if(pangolinReportChannelDailyService.getOne(queryWrapper) == null){
+                        pangolinReportChannelDailyService.save(pangolinReportChannelDaily);
+                    }else {
+                        pangolinReportChannelDailyService.update(pangolinReportChannelDaily,queryWrapper);
+                    }
+                }
+                Integer maxPage = resultNode.get("data").get("page").get("MaxPage").asInt();
+                if(page < maxPage){
+                    page++;
+                    getActivationList(bindAccountLogin,date,page);
+                }
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    public void getRealTimeList(BindAccountLogin bindAccountLogin,Long appId){
+        String url = "https://pangolin.bytedance.com/api/real_time/";
+        ObjectMapper mapper = new ObjectMapper();
+        try {
+            List<String> cookieList = mapper.readValue(bindAccountLogin.getCookie(), new TypeReference<List<String>>() {});
+            if(cookieList != null && cookieList.size() > 0){
+                for (String cookieStr : cookieList){
+                    String[] cookieArray = cookieStr.split(";");
+                    Map<String, String> cookieMap = new HashMap<String, String>();
+                    for (String cookie : cookieArray) {
+                        String[] kv = cookie.split("=");
+                        if (kv.length > 1) {
+                            cookieMap.put(kv[0].trim(), kv[1].trim());
+                        }
+                    }
+                    BasicClientCookie clientCookie = new BasicClientCookie(cookieArray[0].split("=")[0],
+                            cookieArray[0].split("=")[1]);
+                    clientCookie.setDomain("pangolin.bytedance.com");
+                    clientCookie.setPath(cookieMap.get("path"));
+                    HttpUtils2.cookieStore.addCookie(clientCookie);
+                }
+            }
+            Map<String,Object> paramMap = new HashMap<>();
+            List<Long> appIds = new ArrayList<>();
+            appIds.add(appId);
+            paramMap.put("isInternal",0);
+            paramMap.put("channelName","");
+            paramMap.put("appId",appIds);
+            String result = HttpUtils2.httpPostRequest(url,paramMap,null);
+            System.out.println(result);
+            if(checkLogin(bindAccountLogin,result)){
+                JsonNode resultNode = mapper.readTree(result);
+                Iterator<String> clickIterator = resultNode.get("data").get("ChartData").get("click_num").fieldNames();
+                while (clickIterator.hasNext()){
+                    String date = clickIterator.next();
+                    JsonNode clickNode = resultNode.get("data").get("ChartData").get("click_num").get(date);
+                    int size = clickNode.size();
+                    for(int i = 0; i < size; i++){
+                        BigDecimal clickNum = clickNode.get(i).decimalValue();
+                        PangolinReportAppHour pangolinReportAppHour = new PangolinReportAppHour();
+                        pangolinReportAppHour.setAppId(appId);
+                        pangolinReportAppHour.setDate(DateUtils.parseDate(date,"yyyy-MM-dd"));
+                        pangolinReportAppHour.setNum(clickNum);
+                        pangolinReportAppHour.setType("click");
+                        pangolinReportAppHour.setHour(i);
+                        QueryWrapper<PangolinReportAppHour> queryWrapper = new QueryWrapper<>();
+                        queryWrapper.eq("date",pangolinReportAppHour.getDate());
+                        queryWrapper.eq("app_id",pangolinReportAppHour.getAppId());
+                        queryWrapper.eq("type",pangolinReportAppHour.getType());
+                        queryWrapper.eq("hour",pangolinReportAppHour.getHour());
+                        if(pangolinReportAppHourService.getOne(queryWrapper) == null){
+                            pangolinReportAppHourService.save(pangolinReportAppHour);
+                        }else {
+                            pangolinReportAppHourService.update(pangolinReportAppHour,queryWrapper);
+                        }
+//                        pangolinReportAppHourService.saveOrUpdate(pangolinReportAppHour);
+                    }
+                }
+                Iterator<String> activeIterator = resultNode.get("data").get("ChartData").get("active_num").fieldNames();
+                while (activeIterator.hasNext()){
+                    String date = activeIterator.next();
+                    JsonNode activeNode = resultNode.get("data").get("ChartData").get("active_num").get(date);
+                    int size = activeNode.size();
+                    for(int i = 0; i < size; i++){
+                        BigDecimal activeNum = activeNode.get(i).decimalValue();
+                        PangolinReportAppHour pangolinReportAppHour = new PangolinReportAppHour();
+                        pangolinReportAppHour.setAppId(appId);
+                        pangolinReportAppHour.setDate(DateUtils.parseDate(date,"yyyy-MM-dd"));
+                        pangolinReportAppHour.setNum(activeNum);
+                        pangolinReportAppHour.setType("active");
+                        pangolinReportAppHour.setHour(i);
+                        QueryWrapper<PangolinReportAppHour> queryWrapper = new QueryWrapper<>();
+                        queryWrapper.eq("date",pangolinReportAppHour.getDate());
+                        queryWrapper.eq("app_id",pangolinReportAppHour.getAppId());
+                        queryWrapper.eq("type",pangolinReportAppHour.getType());
+                        queryWrapper.eq("hour",pangolinReportAppHour.getHour());
+                        if(pangolinReportAppHourService.getOne(queryWrapper) == null){
+                            pangolinReportAppHourService.save(pangolinReportAppHour);
+                        }else {
+                            pangolinReportAppHourService.update(pangolinReportAppHour,queryWrapper);
+                        }
+                    }
+                }
+                Iterator<String> rateIterator = resultNode.get("data").get("ChartData").get("ratio").fieldNames();
+                while (rateIterator.hasNext()){
+                    String date = rateIterator.next();
+                    JsonNode rateNode = resultNode.get("data").get("ChartData").get("ratio").get(date);
+                    int size = rateNode.size();
+                    for(int i = 0; i < size; i++){
+                        BigDecimal clickNum = rateNode.get(i).decimalValue();
+                        PangolinReportAppHour pangolinReportAppHour = new PangolinReportAppHour();
+                        pangolinReportAppHour.setAppId(appId);
+                        pangolinReportAppHour.setDate(DateUtils.parseDate(date,"yyyy-MM-dd"));
+                        pangolinReportAppHour.setNum(clickNum);
+                        pangolinReportAppHour.setType("rate");
+                        pangolinReportAppHour.setHour(i);
+                        QueryWrapper<PangolinReportAppHour> queryWrapper = new QueryWrapper<>();
+                        queryWrapper.eq("date",pangolinReportAppHour.getDate());
+                        queryWrapper.eq("app_id",pangolinReportAppHour.getAppId());
+                        queryWrapper.eq("type",pangolinReportAppHour.getType());
+                        queryWrapper.eq("hour",pangolinReportAppHour.getHour());
+                        if(pangolinReportAppHourService.getOne(queryWrapper) == null){
+                            pangolinReportAppHourService.save(pangolinReportAppHour);
+                        }else {
+                            pangolinReportAppHourService.update(pangolinReportAppHour,queryWrapper);
+                        }
+                    }
+                }
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    public boolean checkLogin(BindAccountLogin bindAccountLogin,String response){
+        if(response.contains("<html>")){
+            bindAccountLogin.setCookie(null);
+            bindAccountLoginService.updateById(bindAccountLogin);
+            return false;
+        }
+        return true;
+    }
+}

+ 11 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinReportAppHourServiceImpl.java

@@ -0,0 +1,11 @@
+package cn.com.ctop.crawler.modules.pangolin.service.impl;
+
+import cn.com.ctop.crawler.modules.pangolin.entity.PangolinReportAppHour;
+import cn.com.ctop.crawler.modules.pangolin.mapper.PangolinReportAppHourMapper;
+import cn.com.ctop.crawler.modules.pangolin.service.PangolinReportAppHourService;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.springframework.stereotype.Service;
+
+@Service
+public class PangolinReportAppHourServiceImpl extends ServiceImpl<PangolinReportAppHourMapper, PangolinReportAppHour> implements PangolinReportAppHourService {
+}