xuzuoyun 5 лет назад
Родитель
Сommit
ba4064411e

+ 349 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinCrawlerServiceImpl.java

@@ -0,0 +1,349 @@
+package cn.com.ctop.crawler.modules.pangolin.service.impl;
+
+import cn.com.ctop.common.module.entity.BindAccountLogin;
+import cn.com.ctop.common.module.service.IBindAccountLoginService;
+import cn.com.ctop.common.module.utils.HttpUtils2;
+import cn.com.ctop.crawler.modules.pangolin.entity.*;
+import cn.com.ctop.crawler.modules.pangolin.service.*;
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.http.impl.cookie.BasicClientCookie;
+import org.jeecg.common.util.DateUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.util.*;
+
+@Service
+public class PangolinCrawlerServiceImpl implements PangolinCrawlerService {
+    @Autowired
+    private IBindAccountLoginService bindAccountLoginService;
+    @Autowired
+    private PangolinAppService pangolinAppService;
+    @Autowired
+    private PangolinChannelService pangolinChannelService;
+    @Autowired
+    private PangolinReportBillMonthService pangolinReportBillMonthService;
+    @Autowired
+    private PangolinReportChannelDailyService pangolinReportChannelDailyService;
+    @Autowired
+    private PangolinReportAppHourService pangolinReportAppHourService;
+
+    public void getCheckList(BindAccountLogin bindAccountLogin){
+        String url = "https://pangolin.bytedance.com/api/check_apps/list";
+        ObjectMapper mapper = new ObjectMapper();
+        try {
+            List<String> cookieList = mapper.readValue(bindAccountLogin.getCookie(), new TypeReference<List<String>>() {});
+            if(cookieList != null && cookieList.size() > 0){
+                for (String cookieStr : cookieList){
+                    String[] cookieArray = cookieStr.split(";");
+                    Map<String, String> cookieMap = new HashMap<String, String>();
+                    for (String cookie : cookieArray) {
+                        String[] kv = cookie.split("=");
+                        if (kv.length > 1) {
+                            cookieMap.put(kv[0].trim(), kv[1].trim());
+                        }
+                    }
+                    BasicClientCookie clientCookie = new BasicClientCookie(cookieArray[0].split("=")[0],
+                            cookieArray[0].split("=")[1]);
+                    clientCookie.setDomain("pangolin.bytedance.com");
+                    clientCookie.setPath(cookieMap.get("path"));
+                    HttpUtils2.cookieStore.addCookie(clientCookie);
+                }
+            }
+            String result = HttpUtils2.httpPostRequest(url,null,null);
+            System.out.println(result);
+            if(checkLogin(bindAccountLogin,result)){
+                JsonNode resultNode = mapper.readTree(result);
+                Iterator<JsonNode> appIterator = resultNode.get("data").get("List").elements();
+                while (appIterator.hasNext()){
+                    JsonNode checkNode = appIterator.next();
+                    Long appId = checkNode.get("ApplicationId").asLong();
+                    String date = checkNode.get("Date").asText();
+                    String os = checkNode.get("Os").asText();
+                    Integer sumInternalActivations = checkNode.get("SumInternalActivations").asInt();
+                    BigDecimal actualTotalPrice = checkNode.get("ActualTotalPrice").decimalValue();
+                    BigDecimal estimatedTotalPrice = checkNode.get("EstimatedTotalPrice").decimalValue();
+                    BigDecimal accountQualityScore = checkNode.get("AccountQualityScore").decimalValue();
+                    String appName = checkNode.get("AppName").asText();
+                    PangolinReportBillMonth pangolinReportBillMonth = new PangolinReportBillMonth();
+                    pangolinReportBillMonth.setAccountQualityScore(accountQualityScore);
+                    pangolinReportBillMonth.setActualTotalPrice(actualTotalPrice);
+                    pangolinReportBillMonth.setAppId(appId);
+                    pangolinReportBillMonth.setEstimatedTotalPrice(estimatedTotalPrice);
+                    pangolinReportBillMonth.setOs(os);
+                    pangolinReportBillMonth.setSumInternalActivations(sumInternalActivations);
+                    pangolinReportBillMonth.setYear(Integer.parseInt(date.split("-")[0]));
+                    pangolinReportBillMonth.setMonth(Integer.parseInt(date.split("-")[1]));
+                    QueryWrapper<PangolinReportBillMonth> queryWrapper = new QueryWrapper<>();
+                    queryWrapper.eq("os",pangolinReportBillMonth.getOs());
+                    queryWrapper.eq("app_id",pangolinReportBillMonth.getAppId());
+                    queryWrapper.eq("month",pangolinReportBillMonth.getMonth());
+                    queryWrapper.eq("year",pangolinReportBillMonth.getYear());
+                    if(pangolinReportBillMonthService.getOne(queryWrapper) == null){
+                        pangolinReportBillMonthService.save(pangolinReportBillMonth);
+                    }else {
+                        pangolinReportBillMonthService.update(pangolinReportBillMonth,queryWrapper);
+                    }
+                }
+            }
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+
+    public void getChannelList(BindAccountLogin bindAccountLogin){
+        String url = "https://pangolin.bytedance.com/api/search/external_default";
+        ObjectMapper mapper = new ObjectMapper();
+        try {
+            List<String> cookieList = mapper.readValue(bindAccountLogin.getCookie(), new TypeReference<List<String>>() {});
+            if(cookieList != null && cookieList.size() > 0){
+                for (String cookieStr : cookieList){
+                    String[] cookieArray = cookieStr.split(";");
+                    Map<String, String> cookieMap = new HashMap<String, String>();
+                    for (String cookie : cookieArray) {
+                        String[] kv = cookie.split("=");
+                        if (kv.length > 1) {
+                            cookieMap.put(kv[0].trim(), kv[1].trim());
+                        }
+                    }
+                    BasicClientCookie clientCookie = new BasicClientCookie(cookieArray[0].split("=")[0],
+                            cookieArray[0].split("=")[1]);
+                    clientCookie.setDomain("pangolin.bytedance.com");
+                    clientCookie.setPath(cookieMap.get("path"));
+                    HttpUtils2.cookieStore.addCookie(clientCookie);
+                }
+            }
+            String result = HttpUtils2.httpGetRequest(url);
+            System.out.println(result);
+            if(checkLogin(bindAccountLogin,result)){
+                JsonNode resultNode = mapper.readTree(result);
+                Iterator<JsonNode> appIterator = resultNode.get("data").get("AppNameList").elements();
+                while (appIterator.hasNext()){
+                    JsonNode appNode = appIterator.next();
+                    Long id = appNode.get("Id").asLong();
+                    String name = appNode.get("Name").asText();
+                    PangolinApp app = new PangolinApp();
+                    app.setAccountName(bindAccountLogin.getAccountName());
+                    app.setId(id);
+                    app.setName(name);
+                    pangolinAppService.saveOrUpdate(app);
+                }
+                Iterator<JsonNode> channelIterator = resultNode.get("data").get("ChannelNameList").elements();
+                while (channelIterator.hasNext()){
+                    JsonNode channelNode = channelIterator.next();
+                    Long id = channelNode.get("Id").asLong();
+                    String name = channelNode.get("Name").asText();
+                    PangolinChannel channel = new PangolinChannel();
+                    channel.setAccountName(bindAccountLogin.getAccountName());
+                    channel.setId(id);
+                    channel.setName(name);
+                    pangolinChannelService.saveOrUpdate(channel);
+                }
+            }
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+
+    public void getActivationList(BindAccountLogin bindAccountLogin,String date,int page){
+        String url = "https://pangolin.bytedance.com/api/activation/list";
+        ObjectMapper mapper = new ObjectMapper();
+        try {
+            List<String> cookieList = mapper.readValue(bindAccountLogin.getCookie(), new TypeReference<List<String>>() {});
+            if(cookieList != null && cookieList.size() > 0){
+                for (String cookieStr : cookieList){
+                    String[] cookieArray = cookieStr.split(";");
+                    Map<String, String> cookieMap = new HashMap<String, String>();
+                    for (String cookie : cookieArray) {
+                        String[] kv = cookie.split("=");
+                        if (kv.length > 1) {
+                            cookieMap.put(kv[0].trim(), kv[1].trim());
+                        }
+                    }
+                    BasicClientCookie clientCookie = new BasicClientCookie(cookieArray[0].split("=")[0],
+                            cookieArray[0].split("=")[1]);
+                    clientCookie.setDomain("pangolin.bytedance.com");
+                    clientCookie.setPath(cookieMap.get("path"));
+                    HttpUtils2.cookieStore.addCookie(clientCookie);
+                }
+            }
+            Map<String,Object> paramMap = new HashMap<>();
+            Map<String,Object> searchMap = new HashMap<>();
+            searchMap.put("DateStart",DateUtils.addDay(date,-2));
+            searchMap.put("DateEnd",date);
+            searchMap.put("AppId","");
+            searchMap.put("ChannelName","");
+            searchMap.put("OS","");
+            Map<String,Object> pageMap = new HashMap<>();
+            pageMap.put("p",page);
+            paramMap.put("search",searchMap);
+            paramMap.put("page",pageMap);
+            String result = HttpUtils2.httpPostRequest(url,paramMap,null);
+            System.out.println(result);
+            if(checkLogin(bindAccountLogin,result)){
+                JsonNode resultNode = mapper.readTree(result);
+                Iterator<JsonNode> listIterator = resultNode.get("data").get("list").elements();
+                while (listIterator.hasNext()){
+                    JsonNode dataNode = listIterator.next();
+                    String startDate = dataNode.get("StatsDate").asText();
+                    String channelName = dataNode.get("ChannelName").asText();
+                    String appName = dataNode.get("AppVerboseName").asText();
+                    String os = dataNode.get("OS").asText();
+                    Integer externalActivation = dataNode.get("ExternalActivation").asInt();
+                    BigDecimal retentionRatio = dataNode.get("RetentionRatio").decimalValue();
+                    BigDecimal totalPrice = dataNode.get("TotalPrice").decimalValue();
+                    PangolinReportChannelDaily pangolinReportChannelDaily = new PangolinReportChannelDaily();
+                    pangolinReportChannelDaily.setAppName(appName);
+                    pangolinReportChannelDaily.setChannelName(channelName);
+                    pangolinReportChannelDaily.setDate(DateUtils.parseDate(startDate,"yyyy-MM-dd"));
+                    pangolinReportChannelDaily.setExternalActivation(externalActivation);
+                    pangolinReportChannelDaily.setOs(os);
+                    pangolinReportChannelDaily.setRetentionRatio(retentionRatio);
+                    pangolinReportChannelDaily.setTotalPrice(totalPrice);
+                    QueryWrapper<PangolinReportChannelDaily> queryWrapper = new QueryWrapper<>();
+                    queryWrapper.eq("date",pangolinReportChannelDaily.getDate());
+                    queryWrapper.eq("channel_name",pangolinReportChannelDaily.getChannelName());
+                    if(pangolinReportChannelDailyService.getOne(queryWrapper) == null){
+                        pangolinReportChannelDailyService.save(pangolinReportChannelDaily);
+                    }else {
+                        pangolinReportChannelDailyService.update(pangolinReportChannelDaily,queryWrapper);
+                    }
+                }
+                Integer maxPage = resultNode.get("data").get("page").get("MaxPage").asInt();
+                if(page < maxPage){
+                    page++;
+                    getActivationList(bindAccountLogin,date,page);
+                }
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    public void getRealTimeList(BindAccountLogin bindAccountLogin,Long appId){
+        String url = "https://pangolin.bytedance.com/api/real_time/";
+        ObjectMapper mapper = new ObjectMapper();
+        try {
+            List<String> cookieList = mapper.readValue(bindAccountLogin.getCookie(), new TypeReference<List<String>>() {});
+            if(cookieList != null && cookieList.size() > 0){
+                for (String cookieStr : cookieList){
+                    String[] cookieArray = cookieStr.split(";");
+                    Map<String, String> cookieMap = new HashMap<String, String>();
+                    for (String cookie : cookieArray) {
+                        String[] kv = cookie.split("=");
+                        if (kv.length > 1) {
+                            cookieMap.put(kv[0].trim(), kv[1].trim());
+                        }
+                    }
+                    BasicClientCookie clientCookie = new BasicClientCookie(cookieArray[0].split("=")[0],
+                            cookieArray[0].split("=")[1]);
+                    clientCookie.setDomain("pangolin.bytedance.com");
+                    clientCookie.setPath(cookieMap.get("path"));
+                    HttpUtils2.cookieStore.addCookie(clientCookie);
+                }
+            }
+            Map<String,Object> paramMap = new HashMap<>();
+            List<Long> appIds = new ArrayList<>();
+            appIds.add(appId);
+            paramMap.put("isInternal",0);
+            paramMap.put("channelName","");
+            paramMap.put("appId",appIds);
+            String result = HttpUtils2.httpPostRequest(url,paramMap,null);
+            System.out.println(result);
+            if(checkLogin(bindAccountLogin,result)){
+                JsonNode resultNode = mapper.readTree(result);
+                Iterator<String> clickIterator = resultNode.get("data").get("ChartData").get("click_num").fieldNames();
+                while (clickIterator.hasNext()){
+                    String date = clickIterator.next();
+                    JsonNode clickNode = resultNode.get("data").get("ChartData").get("click_num").get(date);
+                    int size = clickNode.size();
+                    for(int i = 0; i < size; i++){
+                        BigDecimal clickNum = clickNode.get(i).decimalValue();
+                        PangolinReportAppHour pangolinReportAppHour = new PangolinReportAppHour();
+                        pangolinReportAppHour.setAppId(appId);
+                        pangolinReportAppHour.setDate(DateUtils.parseDate(date,"yyyy-MM-dd"));
+                        pangolinReportAppHour.setNum(clickNum);
+                        pangolinReportAppHour.setType("click");
+                        pangolinReportAppHour.setHour(i);
+                        QueryWrapper<PangolinReportAppHour> queryWrapper = new QueryWrapper<>();
+                        queryWrapper.eq("date",pangolinReportAppHour.getDate());
+                        queryWrapper.eq("app_id",pangolinReportAppHour.getAppId());
+                        queryWrapper.eq("type",pangolinReportAppHour.getType());
+                        queryWrapper.eq("hour",pangolinReportAppHour.getHour());
+                        if(pangolinReportAppHourService.getOne(queryWrapper) == null){
+                            pangolinReportAppHourService.save(pangolinReportAppHour);
+                        }else {
+                            pangolinReportAppHourService.update(pangolinReportAppHour,queryWrapper);
+                        }
+                    }
+                }
+                Iterator<String> activeIterator = resultNode.get("data").get("ChartData").get("active_num").fieldNames();
+                while (activeIterator.hasNext()){
+                    String date = activeIterator.next();
+                    JsonNode activeNode = resultNode.get("data").get("ChartData").get("active_num").get(date);
+                    int size = activeNode.size();
+                    for(int i = 0; i < size; i++){
+                        BigDecimal activeNum = activeNode.get(i).decimalValue();
+                        PangolinReportAppHour pangolinReportAppHour = new PangolinReportAppHour();
+                        pangolinReportAppHour.setAppId(appId);
+                        pangolinReportAppHour.setDate(DateUtils.parseDate(date,"yyyy-MM-dd"));
+                        pangolinReportAppHour.setNum(activeNum);
+                        pangolinReportAppHour.setType("active");
+                        pangolinReportAppHour.setHour(i);
+                        QueryWrapper<PangolinReportAppHour> queryWrapper = new QueryWrapper<>();
+                        queryWrapper.eq("date",pangolinReportAppHour.getDate());
+                        queryWrapper.eq("app_id",pangolinReportAppHour.getAppId());
+                        queryWrapper.eq("type",pangolinReportAppHour.getType());
+                        queryWrapper.eq("hour",pangolinReportAppHour.getHour());
+                        if(pangolinReportAppHourService.getOne(queryWrapper) == null){
+                            pangolinReportAppHourService.save(pangolinReportAppHour);
+                        }else {
+                            pangolinReportAppHourService.update(pangolinReportAppHour,queryWrapper);
+                        }
+                    }
+                }
+                Iterator<String> rateIterator = resultNode.get("data").get("ChartData").get("ratio").fieldNames();
+                while (rateIterator.hasNext()){
+                    String date = rateIterator.next();
+                    JsonNode rateNode = resultNode.get("data").get("ChartData").get("ratio").get(date);
+                    int size = rateNode.size();
+                    for(int i = 0; i < size; i++){
+                        BigDecimal clickNum = rateNode.get(i).decimalValue();
+                        PangolinReportAppHour pangolinReportAppHour = new PangolinReportAppHour();
+                        pangolinReportAppHour.setAppId(appId);
+                        pangolinReportAppHour.setDate(DateUtils.parseDate(date,"yyyy-MM-dd"));
+                        pangolinReportAppHour.setNum(clickNum);
+                        pangolinReportAppHour.setType("rate");
+                        pangolinReportAppHour.setHour(i);
+                        QueryWrapper<PangolinReportAppHour> queryWrapper = new QueryWrapper<>();
+                        queryWrapper.eq("date",pangolinReportAppHour.getDate());
+                        queryWrapper.eq("app_id",pangolinReportAppHour.getAppId());
+                        queryWrapper.eq("type",pangolinReportAppHour.getType());
+                        queryWrapper.eq("hour",pangolinReportAppHour.getHour());
+                        if(pangolinReportAppHourService.getOne(queryWrapper) == null){
+                            pangolinReportAppHourService.save(pangolinReportAppHour);
+                        }else {
+                            pangolinReportAppHourService.update(pangolinReportAppHour,queryWrapper);
+                        }
+                    }
+                }
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    public boolean checkLogin(BindAccountLogin bindAccountLogin,String response){
+        if(response.contains("<html>")){
+            bindAccountLogin.setCookie(null);
+            bindAccountLoginService.updateById(bindAccountLogin);
+            return false;
+        }
+        return true;
+    }
+}