Explorar el Código

Merge remote-tracking branch 'origin/test' into test

syh hace 5 años
padre
commit
f5cc2b3315

+ 17 - 2
module-common/src/main/java/cn/com/ctop/common/module/service/impl/IpPoolServiceImpl.java

@@ -10,12 +10,13 @@ import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Service;
 
 import java.util.Date;
 import java.util.List;
 
-
+@Slf4j
 @Service
 public class IpPoolServiceImpl extends ServiceImpl<IpPoolMapper, IpPool> implements IIpPoolService {
 
@@ -44,7 +45,21 @@ public class IpPoolServiceImpl extends ServiceImpl<IpPoolMapper, IpPool> impleme
                 List<IpPool> ipList = JSON.parseObject(jsonNode.get("data").toString(), new TypeReference<List<IpPool>>() {
                 });
                 if (ipList != null && ipList.size() > 0) {
-                    this.saveBatch(ipList);
+                    for (IpPool ipPool : ipList) {
+                        Thread thread = new Thread() {
+                            @Override
+                            public void run() {
+                                Boolean avaliable = httpClientUtils.checkProxyIp(ipPool.getIp(), ipPool.getPort());
+                                if (!avaliable) {
+                                    ipPool.setStatus(2);
+                                    log.info("ip" + ipPool.getIp() + " is not avaliable");
+                                }
+                                save(ipPool);
+                            }
+                        };
+                        thread.start();
+                    }
+//                    this.saveBatch(ipList);
                 }
             }
         } catch (Exception e) {

+ 33 - 8
module-common/src/main/java/cn/com/ctop/common/module/utils/HttpClientUtils.java

@@ -70,13 +70,39 @@ public class HttpClientUtils {
         return HttpClients.createDefault();
     }
 
-    public String get(String urlStr) throws Exception {
+    public Boolean checkProxyIp(String ip, int port) {
+        int status = 0;
+        try {
+            SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {
+                //信任所有证书
+                @Override
+                public boolean isTrusted(X509Certificate[] chain, String authType) throws CertificateException {
+                    return true;
+                }
+            }).build();
+            ConnectionConfig connectionConfig = ConnectionConfig.custom()
+                    .setBufferSize(4096)
+                    .build();
+            HttpHost proxy = new HttpHost(ip, port, "http");
+            RequestConfig globalConfig = RequestConfig.custom().setProxy(proxy).setConnectTimeout(1000).setSocketTimeout(1000).setCookieSpec(CookieSpecs.STANDARD).build();
+            SSLConnectionSocketFactory sslFactory = new SSLConnectionSocketFactory(sslContext);
+
+            HttpClient httpClient = HttpClients.custom().setDefaultConnectionConfig(connectionConfig).setDefaultCookieStore(cookieStore).setDefaultRequestConfig(globalConfig).setConnectionReuseStrategy((response, context) -> false).setSSLSocketFactory(sslFactory).build();
+            HttpGet httpGet = new HttpGet("https://www.baidu.com");
+            status = httpClient.execute(httpGet).getStatusLine().getStatusCode();
+        } catch (Exception e) {
+            log.info("ip: " + ip + " is not aviable");
+        }
+        if (status == 200) {
+            return true;
+        } else {
+            return false;
+        }
+    }
 
+    public String get(String urlStr) throws Exception {
         URL url = new URL(urlStr);
         URLConnection urlConnection = url.openConnection(); // 打开连接
-
-        System.out.println(urlConnection.getURL().toString());
-
         BufferedReader br = new BufferedReader(new InputStreamReader(urlConnection.getInputStream(), "utf-8")); // 获取输入流
         String line = null;
         StringBuilder sb = new StringBuilder();
@@ -104,14 +130,12 @@ public class HttpClientUtils {
             Header[] headers = response.getHeaders("Set-Cookie");
             if (headers != null && headers.length > 0) {
                 for (Header header : headers) {
-                    System.out.println(header.getValue());
                     String[] cookieArr = header.getValue().split(";");
                     String[] ck = cookieArr[0].split("=");
                     BasicClientCookie basicClientCookie = new BasicClientCookie(ck[0], ck[1]);
                     Map<String, String> cookieMap = new HashMap<>();
                     if (cookieArr.length > 1) {
                         for (int i = 1; i < cookieArr.length; i++) {
-                            System.out.println(cookieArr[i]);
                             String[] ckParam = cookieArr[i].split("=");
                             if (ckParam.length > 1) {
                                 cookieMap.put(ckParam[0].trim(), ckParam[1]);
@@ -148,6 +172,7 @@ public class HttpClientUtils {
             log.info(httpClientEntity.getResult());
         } catch (Exception e) {
             e.printStackTrace();
+            return httpGetRequest(httpClientEntity);
         }
         httpClientEntity.setResult(result);
         httpClientEntity.setCookieStore(cookieStore);
@@ -216,6 +241,7 @@ public class HttpClientUtils {
         } catch (Exception e) {
             e.printStackTrace();
             log.error(e.getMessage());
+            return httpGetRequest(httpClientEntity);
         }
         httpClientEntity.setResult(result);
         httpClientEntity.setCookieStore(cookieStore);
@@ -253,14 +279,12 @@ public class HttpClientUtils {
             Header[] headers = response.getHeaders("Set-Cookie");
             if (headers != null && headers.length > 0) {
                 for (Header header : headers) {
-                    System.out.println(header.getValue());
                     String[] cookieArr = header.getValue().split(";");
                     String[] ck = cookieArr[0].split("=");
                     BasicClientCookie basicClientCookie = new BasicClientCookie(ck[0], ck[1]);
                     Map<String, String> cookieMap = new HashMap<>();
                     if (cookieArr.length > 1) {
                         for (int i = 1; i < cookieArr.length; i++) {
-                            System.out.println(cookieArr[i]);
                             String[] ckParam = cookieArr[i].split("=");
                             if (ckParam.length > 1) {
                                 cookieMap.put(ckParam[0].trim(), ckParam[1]);
@@ -283,6 +307,7 @@ public class HttpClientUtils {
             log.info(httpClientEntity.getResult());
         } catch (Exception e) {
             e.printStackTrace();
+            return httpGetRequest(httpClientEntity);
         }
         httpClientEntity.setCookieStore(cookieStore);
         return httpClientEntity;

+ 7 - 7
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/graphql/service/impl/KuaishouWebInterfaceServiceImpl.java

@@ -133,9 +133,9 @@ public class KuaishouWebInterfaceServiceImpl implements IKuaishouWebInterfaceSer
                 ObjectMapper mapper = new ObjectMapper();
                 resultMap = mapper.readValue(httpClientEntity.getResult(), new TypeReference<Map<String, Object>>() {
                 });
+                JsonNode resultNode = mapper.readTree(httpClientEntity.getResult()).get("data");
 //                checkCaptcha(httpClientEntity.getResult());
-                String psor = mapper.readTree(httpClientEntity.getResult()).get("data").get("publicFeeds").get("pcursor").asText();
-                if (resultMap.containsKey("captcha") || psor == null) {
+                if (resultNode.get("subCommentList").isNull() || resultNode.get("subCommentList").get("pcursor").isNull()) {
                     ipPool.setStatus(2);
                     iIpPoolService.updateById(ipPool);
                     return subCommentList(ksid, photoId, rootCommentId, pcursor);
@@ -187,9 +187,8 @@ public class KuaishouWebInterfaceServiceImpl implements IKuaishouWebInterfaceSer
                 ObjectMapper mapper = new ObjectMapper();
                 resultMap = mapper.readValue(httpClientEntity.getResult(), new TypeReference<Map<String, Object>>() {
                 });
-                String psor = mapper.readTree(httpClientEntity.getResult()).get("data").get("publicFeeds").get("pcursor").asText();
-//                checkCaptcha(httpClientEntity.getResult());
-                if (resultMap.containsKey("captcha") || psor == null) {
+                JsonNode resultNode = mapper.readTree(httpClientEntity.getResult()).get("data");
+                if (resultNode.get("shortVideoCommentList").isNull() || resultNode.get("shortVideoCommentList").get("pcursor").isNull()) {
                     ipPool.setStatus(2);
                     iIpPoolService.updateById(ipPool);
                     return commentList(ksid, photoId, pcursor);
@@ -259,8 +258,9 @@ public class KuaishouWebInterfaceServiceImpl implements IKuaishouWebInterfaceSer
                 ObjectMapper mapper = new ObjectMapper();
                 resultMap = mapper.readValue(result, new TypeReference<Map<String, Object>>() {
                 });
-                String psor = mapper.readTree(result).get("data").get("publicFeeds").get("pcursor").asText();
-                if (resultMap.containsKey("captcha") || psor == null) {
+                JsonNode resultNode = mapper.readTree(httpClientEntity.getResult()).get("data");
+//                String psor = mapper.readTree(result).get("data").get("publicFeeds").get("pcursor").asText();
+                if (resultNode.get("publicFeeds").isNull() || resultNode.get("publicFeeds").get("pcursor").isNull()) {
                     ipPool.setStatus(2);
                     iIpPoolService.updateById(ipPool);
                     return videoList(ksid, pcursor);

+ 1 - 7
module-report/src/main/java/cn/com/ctop/bytedance/mapper/BytedanceCampaignDailyReportMapper.java

@@ -12,11 +12,5 @@ import org.apache.ibatis.annotations.Param;
  * @Version: V1.0
  */
 public interface BytedanceCampaignDailyReportMapper extends BaseMapper<BytedanceCampaignDailyReport> {
-    /**
-     * 今日消耗
-     *
-     * @param date
-     * @return
-     */
-    ReportVO selectDayReport(@Param("date") String date);
+
 }

+ 2 - 8
module-report/src/main/java/cn/com/ctop/bytedance/mapper/BytedanceReportMapper.java

@@ -14,7 +14,8 @@ public interface BytedanceReportMapper {
      * @param date
      * @return
      */
-    //   ReportVO selectDayReport(@Param("date") String date);
+    JSONObject selectDayReport(@Param("date") String date);
+
 
     /**
      * 查询明细
@@ -40,13 +41,6 @@ public interface BytedanceReportMapper {
      */
     JSONObject selectAllAccount();
 
-    /**
-     * 查询 昨日消耗
-     *
-     * @param anotherDay
-     * @return
-     */
-    JSONObject selectYesterday(@Param("date") String anotherDay);
 
     /**
      * 根据日期 小时 查询信息

+ 1 - 4
module-report/src/main/java/cn/com/ctop/bytedance/mapper/xml/BytedanceCampaignDailyReportMapper.xml

@@ -1,15 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
 <mapper namespace="cn.com.ctop.bytedance.mapper.BytedanceCampaignDailyReportMapper">
-    <select id="selectDayReport" resultType="cn.com.ctop.bytedance.vo.ReportVO">
-      select
-1,2,3,4
-    </select>
+
 </mapper>

+ 2 - 12
module-report/src/main/java/cn/com/ctop/bytedance/mapper/xml/BytedanceReportMapper.xml

@@ -3,7 +3,7 @@
 <mapper namespace="cn.com.ctop.bytedance.mapper.BytedanceReportMapper">
 
 
-    <!--  <select id="selectDayReport" resultType="cn.com.ctop.bytedance.vo.ReportVO">
+    <select id="selectDayReport" resultType="com.alibaba.fastjson.JSONObject">
         select
          sum(charge) cost,
          sum(photo_show) photoShow,
@@ -12,18 +12,8 @@
        from
         ctop_kuaishou_report_hourly_account
         where stat_date = #{date}
-      </select>-->
+      </select>
 
-    <select id="selectYesterday" resultType="com.alibaba.fastjson.JSONObject">
-      select
-       sum(charge) cost,
-       sum(photo_show) photoShow,
-       sum(photo_click) photoClick,
-       sum(aclick) aclick
-     from
-      ctop_kuaishou_report_hourly_account
-      where stat_date = #{date}
-    </select>
 
 
     <select id="selectByDate" resultType="com.alibaba.fastjson.JSONObject">

+ 44 - 47
module-report/src/main/java/cn/com/ctop/bytedance/service/impl/BytedanceReportServiceImpl.java

@@ -5,7 +5,6 @@ import cn.com.ctop.bytedance.mapper.BytedanceAdvertiserDailyReportMapper;
 import cn.com.ctop.bytedance.mapper.BytedanceCampaignDailyReportMapper;
 import cn.com.ctop.bytedance.mapper.BytedanceReportMapper;
 import cn.com.ctop.bytedance.service.IBytedanceReportService;
-import cn.com.ctop.bytedance.vo.ReportVO;
 import cn.com.ctop.common.module.entity.UserAllocation;
 import cn.com.ctop.common.module.mapper.UserAllocationMapper;
 import cn.com.ctop.common.module.utils.Check;
@@ -208,53 +207,51 @@ public class BytedanceReportServiceImpl implements IBytedanceReportService {
             String nowDate = DateUtils.getNowDate("yyyy-MM-dd"); // 当前日期
             String anotherDay = DateUtils.getAnotherDay("yyyy-MM-dd", nowDate, -1); // 当前日期-1
             if (type == 1) { // 今日总消耗
-                ReportVO reportVO = campaignDailyReportMapper.selectDayReport(nowDate);
-                System.err.println("12312312313131313131313131313131");
-                System.err.println(reportVO.toString());
-                System.err.println("----------------------------------------------");
-                JSONObject yesterJson = reportMapper.selectYesterday(anotherDay);
-                if (!Check.isNull(yesterJson)) {
-                    BigDecimal cost = accountJson.getBigDecimal("cost"); // 今日消耗
-                    BigDecimal yesterCost = yesterJson.getBigDecimal("cost");// 昨日消耗
-                    BigDecimal compareBigDecima = new BigDecimal(0);
-                    if (yesterCost.compareTo(compareBigDecima) != 0) {
-                        BigDecimal costProportion = (cost.subtract(yesterCost)).divide(yesterCost, BigDecimal.ROUND_HALF_UP);
-                        accountJson.put("costProportion", costProportion); // 较昨日增加比例
-                    } else {
-                        accountJson.put("costProportion", 0); // 较昨日增加比例
-                    }
+                accountJson = reportMapper.selectDayReport(nowDate);
+                if (!Check.isNull(accountJson)) {
+                    JSONObject yesterJson = reportMapper.selectDayReport(anotherDay);
+                    if (!Check.isNull(yesterJson)) {
+                        BigDecimal cost = accountJson.getBigDecimal("cost"); // 今日消耗
+                        BigDecimal yesterCost = yesterJson.getBigDecimal("cost");// 昨日消耗
+                        BigDecimal compareBigDecima = new BigDecimal(0);
+                        if (yesterCost.compareTo(compareBigDecima) != 0) {
+                            BigDecimal costProportion = (cost.subtract(yesterCost)).divide(yesterCost, BigDecimal.ROUND_HALF_UP);
+                            accountJson.put("costProportion", costProportion); // 较昨日增加比例
+                        } else {
+                            accountJson.put("costProportion", 0); // 较昨日增加比例
+                        }
 
-                    Long photoShow = accountJson.getLong("photoShow");// 今日封面展示
-                    Long yesterPhotoShow = yesterJson.getLong("photoShow");// 昨日封面展示
-                    if (yesterPhotoShow != 0L) {
-                        double showProportion = (photoShow.doubleValue() - yesterPhotoShow.doubleValue()) / yesterPhotoShow.doubleValue();
-                        accountJson.put("showProportion", showProportion); // 较昨日增加比例
-                    } else {
-                        accountJson.put("showProportion", 0); // 较昨日增加比例
-                    }
+                        Long photoShow = accountJson.getLong("photoShow");// 今日封面展示
+                        Long yesterPhotoShow = yesterJson.getLong("photoShow");// 昨日封面展示
+                        if (yesterPhotoShow != 0L) {
+                            double showProportion = (photoShow.doubleValue() - yesterPhotoShow.doubleValue()) / yesterPhotoShow.doubleValue();
+                            accountJson.put("showProportion", showProportion); // 较昨日增加比例
+                        } else {
+                            accountJson.put("showProportion", 0); // 较昨日增加比例
+                        }
 
-                    Long photoClick = accountJson.getLong("photoClick");// 今日封面展示
-                    Long yesterphotoClick = yesterJson.getLong("photoShow");// 昨日封面展示
-                    if (yesterPhotoShow != 0L) {
-                        double photoClickProportion = (photoClick.doubleValue() - yesterphotoClick.doubleValue()) / yesterphotoClick.doubleValue();
-                        accountJson.put("photoClickProportion", photoClickProportion); // 较昨日增加比例
-                    } else {
-                        accountJson.put("photoClickProportion", 0); // 较昨日增加比例
-                    }
+                        Long photoClick = accountJson.getLong("photoClick");// 今日封面展示
+                        Long yesterphotoClick = yesterJson.getLong("photoShow");// 昨日封面展示
+                        if (yesterPhotoShow != 0L) {
+                            double photoClickProportion = (photoClick.doubleValue() - yesterphotoClick.doubleValue()) / yesterphotoClick.doubleValue();
+                            accountJson.put("photoClickProportion", photoClickProportion); // 较昨日增加比例
+                        } else {
+                            accountJson.put("photoClickProportion", 0); // 较昨日增加比例
+                        }
 
-                    Long aclick = accountJson.getLong("aclick");// 今日封面展示
-                    Long yesterAclick = yesterJson.getLong("photoShow");// 昨日封面展示
-                    if (yesterPhotoShow != 0L) {
-                        double aClickProportion = (aclick.doubleValue() - yesterAclick.doubleValue()) / yesterAclick.doubleValue();
-                        accountJson.put("aClickProportion", aClickProportion); // 较昨日增加比例
-                    } else {
-                        accountJson.put("aClickProportion", 0); // 较昨日增加比例
-                    }
+                        Long aclick = accountJson.getLong("aclick");// 今日封面展示
+                        Long yesterAclick = yesterJson.getLong("photoShow");// 昨日封面展示
+                        if (yesterPhotoShow != 0L) {
+                            double aClickProportion = (aclick.doubleValue() - yesterAclick.doubleValue()) / yesterAclick.doubleValue();
+                            accountJson.put("aClickProportion", aClickProportion); // 较昨日增加比例
+                        } else {
+                            accountJson.put("aClickProportion", 0); // 较昨日增加比例
+                        }
 
+                    }
                 }
-
             } else if (type == 2) { // 昨日总消耗
-                accountJson = reportMapper.selectYesterday(anotherDay);
+                accountJson = reportMapper.selectDayReport(anotherDay);
 
             } else if (type == 3) { // 近七天
                 String endDate = DateUtils.getAnotherDay("yyyy-MM-dd", anotherDay, -7);
@@ -334,6 +331,7 @@ public class BytedanceReportServiceImpl implements IBytedanceReportService {
             e.printStackTrace();
         }
 
+        System.err.println(accountJson);
         return accountJson;
     }
 
@@ -349,7 +347,6 @@ public class BytedanceReportServiceImpl implements IBytedanceReportService {
                 for (int i = 0; i < dayDetail.size(); i++) {
                     JSONObject detailJson = dayDetail.get(i);
                     String statHour = detailJson.getString("statHour");
-
                     JSONObject yesterDayJson = reportMapper.selectYesterdayByHourAndDate(anotherDay, statHour);
                     if (!Check.isNull(yesterDayJson)) {
                         BigDecimal cost = detailJson.getBigDecimal("cost"); // 今日消耗
@@ -386,14 +383,13 @@ public class BytedanceReportServiceImpl implements IBytedanceReportService {
                         } else {
                             detailJson.put("proportion", 0); // 较昨日增加比例
                         }
-                        detail.add(detailJson);
+
                     }
+                    detail.add(detailJson);
 
                 }
-                System.err.println(detail);
             } else if (type == 2) { // 昨日明细
                 detail = reportMapper.selectReportDetail(anotherDay);
-                System.err.println(detail);
             } else if (type == 3) { // 近七天明细
                 String endDate = DateUtils.getAnotherDay("yyyy-MM-dd", anotherDay, -7);
                 detail = reportMapper.selectDayDetailByDate(anotherDay, endDate);
@@ -423,7 +419,7 @@ public class BytedanceReportServiceImpl implements IBytedanceReportService {
                 if ("month".equals(sign)) {
                     Map<String, Object> dateMap = DateUtils.compare_date("yyyy-MM", bigDate, smallDate);
                     List<JSONObject> comparedArr = reportMapper.selectDayDetailsByMonth((String) dateMap.get("bigDate"));   // 除数数据
-                    String comparedDate = (String) dateMap.get("bigDate");
+                    String comparedDate = (String) dateMap.get("smallDate");
                     for (int i = 0; i < comparedArr.size(); i++) {
                         JSONObject comparedJson = comparedArr.get(i);
                         if (!Check.isNull(comparedJson)) {
@@ -497,6 +493,7 @@ public class BytedanceReportServiceImpl implements IBytedanceReportService {
 
         }
 
+        System.err.println(detail);
 
         return detail;
     }