|
@@ -1,368 +0,0 @@
|
|
|
-package cn.com.ctop.crawler.modules.pangolin.service.impl;
|
|
|
-
|
|
|
-import cn.com.ctop.common.module.entity.BindAccountLogin;
|
|
|
-import cn.com.ctop.common.module.service.IBindAccountLoginService;
|
|
|
-import cn.com.ctop.common.module.utils.HttpUtils2;
|
|
|
-import cn.com.ctop.crawler.modules.pangolin.entity.*;
|
|
|
-import cn.com.ctop.crawler.modules.pangolin.service.*;
|
|
|
-import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
|
|
|
-import com.fasterxml.jackson.core.type.TypeReference;
|
|
|
-import com.fasterxml.jackson.databind.JsonNode;
|
|
|
-import com.fasterxml.jackson.databind.ObjectMapper;
|
|
|
-import org.apache.http.impl.cookie.BasicClientCookie;
|
|
|
-import org.jeecg.common.util.DateUtils;
|
|
|
-import org.springframework.beans.factory.annotation.Autowired;
|
|
|
-import org.springframework.stereotype.Service;
|
|
|
-
|
|
|
-import java.io.IOException;
|
|
|
-import java.math.BigDecimal;
|
|
|
-import java.util.*;
|
|
|
-
|
|
|
-@Service
|
|
|
-public class PangolinCrawlerServiceImpl implements PangolinCrawlerService {
|
|
|
- @Autowired
|
|
|
- private IBindAccountLoginService bindAccountLoginService;
|
|
|
- @Autowired
|
|
|
- private PangolinAppService pangolinAppService;
|
|
|
- @Autowired
|
|
|
- private PangolinChannelService pangolinChannelService;
|
|
|
- @Autowired
|
|
|
- private PangolinReportBillMonthService pangolinReportBillMonthService;
|
|
|
- @Autowired
|
|
|
- private PangolinReportChannelDailyService pangolinReportChannelDailyService;
|
|
|
- @Autowired
|
|
|
- private PangolinReportAppHourService pangolinReportAppHourService;
|
|
|
-
|
|
|
- @Override
|
|
|
- public void getCheckList(BindAccountLogin bindAccountLogin) {
|
|
|
- String url = "https://pangolin.bytedance.com/api/check_apps/list";
|
|
|
- ObjectMapper mapper = new ObjectMapper();
|
|
|
- try {
|
|
|
- List<String> cookieList = mapper.readValue(bindAccountLogin.getCookie(), new TypeReference<List<String>>() {
|
|
|
- });
|
|
|
- if (cookieList != null && !cookieList.isEmpty()) {
|
|
|
- for (String cookieStr : cookieList) {
|
|
|
- String[] cookieArray = cookieStr.split(";");
|
|
|
- Map<String, String> cookieMap = new HashMap<>();
|
|
|
- for (String cookie : cookieArray) {
|
|
|
- String[] kv = cookie.split("=");
|
|
|
- if (kv.length > 1) {
|
|
|
- cookieMap.put(kv[0].trim(), kv[1].trim());
|
|
|
- }
|
|
|
- }
|
|
|
- BasicClientCookie clientCookie = new BasicClientCookie(cookieArray[0].split("=")[0],
|
|
|
- cookieArray[0].split("=")[1]);
|
|
|
- clientCookie.setDomain("pangolin.bytedance.com");
|
|
|
- clientCookie.setPath(cookieMap.get("path"));
|
|
|
- HttpUtils2.cookieStore.addCookie(clientCookie);
|
|
|
- }
|
|
|
- }
|
|
|
- String result = HttpUtils2.httpPostRequest(url, null, null);
|
|
|
- if (bindAccountLoginService.checkLogin(bindAccountLogin, result)) {
|
|
|
- JsonNode resultNode = mapper.readTree(result);
|
|
|
- Iterator<JsonNode> appIterator = resultNode.get("data").get("List").elements();
|
|
|
- while (appIterator.hasNext()) {
|
|
|
- JsonNode checkNode = appIterator.next();
|
|
|
- Long appId = checkNode.get("ApplicationId").asLong();
|
|
|
- String date = checkNode.get("Date").asText();
|
|
|
- String os = checkNode.get("Os").asText();
|
|
|
- Integer sumInternalActivations = checkNode.get("SumInternalActivations").asInt();
|
|
|
- BigDecimal actualTotalPrice = checkNode.get("ActualTotalPrice").decimalValue();
|
|
|
- BigDecimal estimatedTotalPrice = checkNode.get("EstimatedTotalPrice").decimalValue();
|
|
|
- BigDecimal accountQualityScore = checkNode.get("AccountQualityScore").decimalValue();
|
|
|
- PangolinReportBillMonth pangolinReportBillMonth = new PangolinReportBillMonth();
|
|
|
- pangolinReportBillMonth.setAccountQualityScore(accountQualityScore);
|
|
|
- pangolinReportBillMonth.setActualTotalPrice(actualTotalPrice);
|
|
|
- pangolinReportBillMonth.setAppId(appId);
|
|
|
- pangolinReportBillMonth.setEstimatedTotalPrice(estimatedTotalPrice);
|
|
|
- pangolinReportBillMonth.setOs(os);
|
|
|
- pangolinReportBillMonth.setSumInternalActivations(sumInternalActivations);
|
|
|
- pangolinReportBillMonth.setYear(Integer.parseInt(date.split("-")[0]));
|
|
|
- pangolinReportBillMonth.setMonth(Integer.parseInt(date.split("-")[1]));
|
|
|
- pangolinReportBillMonth.setDate(date);
|
|
|
- pangolinReportBillMonth.setAccountName(bindAccountLogin.getAccountName());
|
|
|
- QueryWrapper<PangolinReportBillMonth> queryWrapper = new QueryWrapper<>();
|
|
|
- queryWrapper.eq("os", pangolinReportBillMonth.getOs());
|
|
|
- queryWrapper.eq("app_id", pangolinReportBillMonth.getAppId());
|
|
|
- queryWrapper.eq("month", pangolinReportBillMonth.getMonth());
|
|
|
- queryWrapper.eq("year", pangolinReportBillMonth.getYear());
|
|
|
- queryWrapper.eq("account_name", pangolinReportBillMonth.getAccountName());
|
|
|
- if (pangolinReportBillMonthService.getOne(queryWrapper) == null) {
|
|
|
- pangolinReportBillMonthService.save(pangolinReportBillMonth);
|
|
|
- } else {
|
|
|
- pangolinReportBillMonthService.update(pangolinReportBillMonth, queryWrapper);
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- } catch (IOException e) {
|
|
|
- e.printStackTrace();
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- @Override
|
|
|
- public void getChannelList(BindAccountLogin bindAccountLogin) {
|
|
|
- String url = "https://pangolin.bytedance.com/api/search/external_default";
|
|
|
- ObjectMapper mapper = new ObjectMapper();
|
|
|
- try {
|
|
|
- List<String> cookieList = mapper.readValue(bindAccountLogin.getCookie(), new TypeReference<List<String>>() {
|
|
|
- });
|
|
|
- if (cookieList != null && !cookieList.isEmpty()) {
|
|
|
- for (String cookieStr : cookieList) {
|
|
|
- String[] cookieArray = cookieStr.split(";");
|
|
|
- Map<String, String> cookieMap = new HashMap<>();
|
|
|
- for (String cookie : cookieArray) {
|
|
|
- String[] kv = cookie.split("=");
|
|
|
- if (kv.length > 1) {
|
|
|
- cookieMap.put(kv[0].trim(), kv[1].trim());
|
|
|
- }
|
|
|
- }
|
|
|
- BasicClientCookie clientCookie = new BasicClientCookie(cookieArray[0].split("=")[0],
|
|
|
- cookieArray[0].split("=")[1]);
|
|
|
- clientCookie.setDomain("pangolin.bytedance.com");
|
|
|
- clientCookie.setPath(cookieMap.get("path"));
|
|
|
- HttpUtils2.cookieStore.addCookie(clientCookie);
|
|
|
- }
|
|
|
- }
|
|
|
- String result = HttpUtils2.httpGetRequest(url);
|
|
|
- if (bindAccountLoginService.checkLogin(bindAccountLogin, result)) {
|
|
|
- JsonNode resultNode = mapper.readTree(result);
|
|
|
- Iterator<JsonNode> appIterator = resultNode.get("data").get("AppNameList").elements();
|
|
|
- while (appIterator.hasNext()) {
|
|
|
- JsonNode appNode = appIterator.next();
|
|
|
- Long id = appNode.get("Id").asLong();
|
|
|
- String name = appNode.get("Name").asText();
|
|
|
- PangolinApp app = new PangolinApp();
|
|
|
- app.setAccountName(bindAccountLogin.getAccountName());
|
|
|
- app.setAppId(id);
|
|
|
- app.setName(name);
|
|
|
- pangolinAppService.saveOrUpdateApp(app);
|
|
|
- }
|
|
|
-// Iterator<JsonNode> channelIterator = resultNode.get("data").get("ChannelNameList").elements();
|
|
|
-// while (channelIterator.hasNext()) {
|
|
|
-// JsonNode channelNode = channelIterator.next();
|
|
|
-// Long id = channelNode.get("Id").asLong();
|
|
|
-// String name = channelNode.get("Name").asText();
|
|
|
-// PangolinChannel channel = new PangolinChannel();
|
|
|
-// channel.setAccountName(bindAccountLogin.getAccountName());
|
|
|
-// channel.setId(id);
|
|
|
-// channel.setName(name);
|
|
|
-// pangolinChannelService.saveOrUpdate(channel);
|
|
|
-// }
|
|
|
- }
|
|
|
- } catch (IOException e) {
|
|
|
- e.printStackTrace();
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- @Override
|
|
|
- public void getActivationList(BindAccountLogin bindAccountLogin, String date, int page) {
|
|
|
- String url = "https://pangolin.bytedance.com/api/activation/list";
|
|
|
- ObjectMapper mapper = new ObjectMapper();
|
|
|
- try {
|
|
|
- List<String> cookieList = mapper.readValue(bindAccountLogin.getCookie(), new TypeReference<List<String>>() {
|
|
|
- });
|
|
|
- if (cookieList != null && !cookieList.isEmpty()) {
|
|
|
- for (String cookieStr : cookieList) {
|
|
|
- String[] cookieArray = cookieStr.split(";");
|
|
|
- Map<String, String> cookieMap = new HashMap<>();
|
|
|
- for (String cookie : cookieArray) {
|
|
|
- String[] kv = cookie.split("=");
|
|
|
- if (kv.length > 1) {
|
|
|
- cookieMap.put(kv[0].trim(), kv[1].trim());
|
|
|
- }
|
|
|
- }
|
|
|
- BasicClientCookie clientCookie = new BasicClientCookie(cookieArray[0].split("=")[0],
|
|
|
- cookieArray[0].split("=")[1]);
|
|
|
- clientCookie.setDomain("pangolin.bytedance.com");
|
|
|
- clientCookie.setPath(cookieMap.get("path"));
|
|
|
- HttpUtils2.cookieStore.addCookie(clientCookie);
|
|
|
- }
|
|
|
- }
|
|
|
- Map<String, Object> paramMap = new HashMap<>();
|
|
|
- Map<String, Object> searchMap = new HashMap<>();
|
|
|
- searchMap.put("DateStart", DateUtils.addDay(date, -2));
|
|
|
- searchMap.put("DateEnd", date);
|
|
|
- searchMap.put("AppId", "");
|
|
|
- searchMap.put("ChannelName", "");
|
|
|
- searchMap.put("OS", "");
|
|
|
- Map<String, Object> pageMap = new HashMap<>();
|
|
|
- pageMap.put("p", page);
|
|
|
- paramMap.put("search", searchMap);
|
|
|
- paramMap.put("page", pageMap);
|
|
|
- String result = HttpUtils2.httpPostRequest(url, paramMap, null);
|
|
|
- if (bindAccountLoginService.checkLogin(bindAccountLogin, result)) {
|
|
|
- JsonNode resultNode = mapper.readTree(result);
|
|
|
- Iterator<JsonNode> listIterator = resultNode.get("data").get("list").elements();
|
|
|
- while (listIterator.hasNext()) {
|
|
|
- JsonNode dataNode = listIterator.next();
|
|
|
- String startDate = dataNode.get("StatsDate").asText();
|
|
|
- String channelName = dataNode.get("ChannelName").asText();
|
|
|
- String appName = dataNode.get("AppVerboseName").asText();
|
|
|
- String os = dataNode.get("OS").asText();
|
|
|
- Integer externalActivation = dataNode.get("ExternalActivation").asInt();
|
|
|
- BigDecimal retentionRatio = dataNode.get("RetentionRatio").decimalValue();
|
|
|
- BigDecimal totalPrice = dataNode.get("TotalPrice").decimalValue();
|
|
|
- PangolinReportChannelDaily pangolinReportChannelDaily = new PangolinReportChannelDaily();
|
|
|
- pangolinReportChannelDaily.setAppName(appName);
|
|
|
- pangolinReportChannelDaily.setChannelName(channelName);
|
|
|
- pangolinReportChannelDaily.setDate(DateUtils.parseDate(startDate, "yyyy-MM-dd"));
|
|
|
- pangolinReportChannelDaily.setExternalActivation(externalActivation);
|
|
|
- pangolinReportChannelDaily.setOprateSystem(os);
|
|
|
- pangolinReportChannelDaily.setRetentionRatio(retentionRatio);
|
|
|
- pangolinReportChannelDaily.setTotalPrice(totalPrice);
|
|
|
- pangolinReportChannelDaily.setAccountName(bindAccountLogin.getAccountName());
|
|
|
- QueryWrapper<PangolinReportChannelDaily> queryWrapper = new QueryWrapper<>();
|
|
|
- queryWrapper.eq("date", pangolinReportChannelDaily.getDate());
|
|
|
- queryWrapper.eq("app_name", pangolinReportChannelDaily.getAppName());
|
|
|
- queryWrapper.eq("channel_name", pangolinReportChannelDaily.getChannelName());
|
|
|
- queryWrapper.eq("account_name", bindAccountLogin.getAccountName());
|
|
|
- if (pangolinReportChannelDailyService.getOne(queryWrapper) == null) {
|
|
|
- pangolinReportChannelDailyService.save(pangolinReportChannelDaily);
|
|
|
- } else {
|
|
|
- pangolinReportChannelDailyService.update(pangolinReportChannelDaily, queryWrapper);
|
|
|
- }
|
|
|
- }
|
|
|
- Integer maxPage = resultNode.get("data").get("page").get("MaxPage").asInt();
|
|
|
- if (page < maxPage) {
|
|
|
- page++;
|
|
|
- getActivationList(bindAccountLogin, date, page);
|
|
|
- }
|
|
|
- }
|
|
|
- } catch (Exception e) {
|
|
|
- e.printStackTrace();
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- @Override
|
|
|
- public void getRealTimeList(BindAccountLogin bindAccountLogin, Long appId) {
|
|
|
- String url = "https://pangolin.bytedance.com/api/real_time/";
|
|
|
- ObjectMapper mapper = new ObjectMapper();
|
|
|
- try {
|
|
|
- List<String> cookieList = mapper.readValue(bindAccountLogin.getCookie(), new TypeReference<List<String>>() {
|
|
|
- });
|
|
|
- if (cookieList != null && !cookieList.isEmpty()) {
|
|
|
- for (String cookieStr : cookieList) {
|
|
|
- String[] cookieArray = cookieStr.split(";");
|
|
|
- Map<String, String> cookieMap = new HashMap<>();
|
|
|
- for (String cookie : cookieArray) {
|
|
|
- String[] kv = cookie.split("=");
|
|
|
- if (kv.length > 1) {
|
|
|
- cookieMap.put(kv[0].trim(), kv[1].trim());
|
|
|
- }
|
|
|
- }
|
|
|
- BasicClientCookie clientCookie = new BasicClientCookie(cookieArray[0].split("=")[0],
|
|
|
- cookieArray[0].split("=")[1]);
|
|
|
- clientCookie.setDomain("pangolin.bytedance.com");
|
|
|
- clientCookie.setPath(cookieMap.get("path"));
|
|
|
- HttpUtils2.cookieStore.addCookie(clientCookie);
|
|
|
- }
|
|
|
- }
|
|
|
- Map<String, Object> paramMap = new HashMap<>();
|
|
|
- List<Long> appIds = new ArrayList<>();
|
|
|
- appIds.add(appId);
|
|
|
- paramMap.put("isInternal", 0);
|
|
|
- paramMap.put("channelName", "");
|
|
|
- paramMap.put("appId", appIds);
|
|
|
- String result = HttpUtils2.httpPostRequest(url, paramMap, null);
|
|
|
- if (bindAccountLoginService.checkLogin(bindAccountLogin, result)) {
|
|
|
- JsonNode resultNode = mapper.readTree(result);
|
|
|
- Iterator<String> clickIterator = resultNode.get("data").get("ChartData").get("click_num").fieldNames();
|
|
|
- while (clickIterator.hasNext()) {
|
|
|
- String date = clickIterator.next();
|
|
|
- JsonNode clickNode = resultNode.get("data").get("ChartData").get("click_num").get(date);
|
|
|
- int size = clickNode.size();
|
|
|
- for (int i = 0; i < size; i++) {
|
|
|
- BigDecimal clickNum = clickNode.get(i).decimalValue();
|
|
|
- PangolinReportAppHour pangolinReportAppHour = new PangolinReportAppHour();
|
|
|
- pangolinReportAppHour.setAccountName(bindAccountLogin.getAccountName());
|
|
|
- pangolinReportAppHour.setAppId(appId);
|
|
|
- pangolinReportAppHour.setDate(DateUtils.parseDate(date, "yyyy-MM-dd"));
|
|
|
- pangolinReportAppHour.setNum(clickNum);
|
|
|
- pangolinReportAppHour.setType("click");
|
|
|
- pangolinReportAppHour.setHour(i);
|
|
|
- QueryWrapper<PangolinReportAppHour> queryWrapper = new QueryWrapper<>();
|
|
|
- queryWrapper.eq("date", pangolinReportAppHour.getDate());
|
|
|
- queryWrapper.eq("app_id", pangolinReportAppHour.getAppId());
|
|
|
- queryWrapper.eq("type", pangolinReportAppHour.getType());
|
|
|
- queryWrapper.eq("hour", pangolinReportAppHour.getHour());
|
|
|
- queryWrapper.eq("account_name", pangolinReportAppHour.getAccountName());
|
|
|
- if (pangolinReportAppHourService.getOne(queryWrapper) == null) {
|
|
|
- pangolinReportAppHourService.save(pangolinReportAppHour);
|
|
|
- } else {
|
|
|
- pangolinReportAppHourService.update(pangolinReportAppHour, queryWrapper);
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- Iterator<String> activeIterator = resultNode.get("data").get("ChartData").get("active_num").fieldNames();
|
|
|
- while (activeIterator.hasNext()) {
|
|
|
- String date = activeIterator.next();
|
|
|
- JsonNode activeNode = resultNode.get("data").get("ChartData").get("active_num").get(date);
|
|
|
- int size = activeNode.size();
|
|
|
- for (int i = 0; i < size; i++) {
|
|
|
- BigDecimal activeNum = activeNode.get(i).decimalValue();
|
|
|
- PangolinReportAppHour pangolinReportAppHour = new PangolinReportAppHour();
|
|
|
- pangolinReportAppHour.setAccountName(bindAccountLogin.getAccountName());
|
|
|
- pangolinReportAppHour.setAppId(appId);
|
|
|
- pangolinReportAppHour.setDate(DateUtils.parseDate(date, "yyyy-MM-dd"));
|
|
|
- pangolinReportAppHour.setNum(activeNum);
|
|
|
- pangolinReportAppHour.setType("active");
|
|
|
- pangolinReportAppHour.setHour(i);
|
|
|
- QueryWrapper<PangolinReportAppHour> queryWrapper = new QueryWrapper<>();
|
|
|
- queryWrapper.eq("date", pangolinReportAppHour.getDate());
|
|
|
- queryWrapper.eq("app_id", pangolinReportAppHour.getAppId());
|
|
|
- queryWrapper.eq("type", pangolinReportAppHour.getType());
|
|
|
- queryWrapper.eq("hour", pangolinReportAppHour.getHour());
|
|
|
- queryWrapper.eq("account_name", pangolinReportAppHour.getAccountName());
|
|
|
- if (pangolinReportAppHourService.getOne(queryWrapper) == null) {
|
|
|
- pangolinReportAppHourService.save(pangolinReportAppHour);
|
|
|
- } else {
|
|
|
- pangolinReportAppHourService.update(pangolinReportAppHour, queryWrapper);
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- Iterator<String> rateIterator = resultNode.get("data").get("ChartData").get("ratio").fieldNames();
|
|
|
- while (rateIterator.hasNext()) {
|
|
|
- String date = rateIterator.next();
|
|
|
- JsonNode rateNode = resultNode.get("data").get("ChartData").get("ratio").get(date);
|
|
|
- int size = rateNode.size();
|
|
|
- for (int i = 0; i < size; i++) {
|
|
|
- BigDecimal clickNum = rateNode.get(i).decimalValue();
|
|
|
- PangolinReportAppHour pangolinReportAppHour = new PangolinReportAppHour();
|
|
|
- pangolinReportAppHour.setAccountName(bindAccountLogin.getAccountName());
|
|
|
- pangolinReportAppHour.setAppId(appId);
|
|
|
- pangolinReportAppHour.setDate(DateUtils.parseDate(date, "yyyy-MM-dd"));
|
|
|
- pangolinReportAppHour.setNum(clickNum);
|
|
|
- pangolinReportAppHour.setType("rate");
|
|
|
- pangolinReportAppHour.setHour(i);
|
|
|
- QueryWrapper<PangolinReportAppHour> queryWrapper = new QueryWrapper<>();
|
|
|
- queryWrapper.eq("date", pangolinReportAppHour.getDate());
|
|
|
- queryWrapper.eq("app_id", pangolinReportAppHour.getAppId());
|
|
|
- queryWrapper.eq("type", pangolinReportAppHour.getType());
|
|
|
- queryWrapper.eq("hour", pangolinReportAppHour.getHour());
|
|
|
- queryWrapper.eq("account_name", pangolinReportAppHour.getAccountName());
|
|
|
- if (pangolinReportAppHourService.getOne(queryWrapper) == null) {
|
|
|
- pangolinReportAppHourService.save(pangolinReportAppHour);
|
|
|
- } else {
|
|
|
- pangolinReportAppHourService.update(pangolinReportAppHour, queryWrapper);
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- } catch (Exception e) {
|
|
|
- e.printStackTrace();
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- @Override
|
|
|
- public void getNowData(BindAccountLogin bindAccountLogin) {
|
|
|
- getChannelList(bindAccountLogin);
|
|
|
- getActivationList(bindAccountLogin, DateUtils.getNowDate("yyyy-MM-dd"), 1);
|
|
|
- getCheckList(bindAccountLogin);
|
|
|
- QueryWrapper<PangolinApp> queryWrapper1 = new QueryWrapper<>();
|
|
|
- queryWrapper1.eq("account_name", bindAccountLogin.getAccountName());
|
|
|
- List<PangolinApp> appList = pangolinAppService.list(queryWrapper1);
|
|
|
- if (appList != null && !appList.isEmpty()) {
|
|
|
- appList.forEach(pangolinApp -> getRealTimeList(bindAccountLogin, pangolinApp.getId()));
|
|
|
- }
|
|
|
- }
|
|
|
-}
|