瀏覽代碼

Merge remote-tracking branch 'origin/test' into test

yumeng 5 年之前
父節點
當前提交
931a9549d5

+ 1 - 0
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/KuaishouCrawlerJob.java

@@ -25,6 +25,7 @@ public class KuaishouCrawlerJob implements Job {
     public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
         try {
             Integer page = 1;
+            System.out.println("begin");
             QueryWrapper<KuaishouAppAccount> queryWrapper = new QueryWrapper<KuaishouAppAccount>();
             queryWrapper.eq("path","/rest/n/feed/hot");
             List<KuaishouAppAccount> list = kuaishouAppAccountService.list(queryWrapper);

+ 25 - 0
module-common/src/main/java/cn/com/ctop/common/module/entity/IpPool.java

@@ -0,0 +1,25 @@
+package cn.com.ctop.common.module.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+import org.springframework.format.annotation.DateTimeFormat;
+
+import java.util.Date;
+
+@Data
+@TableName("ctop_ip_pool")
+public class IpPool {
+    @TableId(value = "id", type = IdType.INPUT)
+    private Long id;
+    private String ip;
+    private Integer port;
+    private String city;
+    @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss")
+    private Date expireTime;
+    @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss")
+    private Date lastUseTime;
+    private String isp;
+    private Integer status;
+}

+ 7 - 0
module-common/src/main/java/cn/com/ctop/common/module/mapper/IpPoolMapper.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.common.module.mapper;
+
+import cn.com.ctop.common.module.entity.IpPool;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+public interface IpPoolMapper extends BaseMapper<IpPool> {
+}

+ 8 - 0
module-common/src/main/java/cn/com/ctop/common/module/service/IIpPoolService.java

@@ -0,0 +1,8 @@
+package cn.com.ctop.common.module.service;
+
+import cn.com.ctop.common.module.entity.IpPool;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+public interface IIpPoolService extends IService<IpPool> {
+    public IpPool getAvaliableIp();
+}

+ 54 - 0
module-common/src/main/java/cn/com/ctop/common/module/service/impl/IpPoolServiceImpl.java

@@ -0,0 +1,54 @@
+package cn.com.ctop.common.module.service.impl;
+
+import cn.com.ctop.common.module.entity.IpPool;
+import cn.com.ctop.common.module.mapper.IpPoolMapper;
+import cn.com.ctop.common.module.service.IIpPoolService;
+import cn.com.ctop.common.module.utils.HttpClientUtils;
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.TypeReference;
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.springframework.stereotype.Service;
+
+import java.util.Date;
+import java.util.List;
+
+
+@Service
+public class IpPoolServiceImpl extends ServiceImpl<IpPoolMapper, IpPool> implements IIpPoolService {
+
+    public IpPool getAvaliableIp() {
+        QueryWrapper<IpPool> queryWrapper = new QueryWrapper<>();
+        queryWrapper.eq("status", 1);
+        queryWrapper.gt("expire_time", new Date());
+        queryWrapper.orderByAsc("last_use_time");
+        IpPool ipPool = this.getOne(queryWrapper);
+        if (ipPool == null) {
+            syncIpPool();
+        } else {
+            ipPool.setLastUseTime(new Date());
+            this.updateById(ipPool);
+        }
+        return ipPool;
+    }
+
+    private void syncIpPool() {
+        HttpClientUtils httpClientUtils = new HttpClientUtils();
+        try {
+            String result = httpClientUtils.get("http://d.jghttp.golangapi.com/getip?num=200&type=2&pro=&city=0&yys=0&port=11&pack=14477&ts=1&ys=1&cs=1&lb=1&sb=0&pb=45&mr=2&regions=");
+            ObjectMapper mapper = new ObjectMapper();
+            JsonNode jsonNode = mapper.readTree(result);
+            if (jsonNode.get("code").asInt() == 0) {
+                List<IpPool> ipList = JSON.parseObject(jsonNode.get("data").toString(), new TypeReference<List<IpPool>>() {
+                });
+                if (ipList != null && ipList.size() > 0) {
+                    this.saveBatch(ipList);
+                }
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+}

+ 125 - 15
module-common/src/main/java/cn/com/ctop/common/module/utils/HttpClientUtils.java

@@ -1,12 +1,12 @@
 package cn.com.ctop.common.module.utils;
 
 import cn.com.ctop.common.module.entity.HttpClientEntity;
+import cn.com.ctop.common.module.entity.IpPool;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.gson.Gson;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.http.Header;
-import org.apache.http.HttpResponse;
-import org.apache.http.HttpStatus;
-import org.apache.http.NameValuePair;
+import org.apache.http.*;
 import org.apache.http.client.CookieStore;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.config.CookieSpecs;
@@ -14,6 +14,7 @@ import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.entity.UrlEncodedFormEntity;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
+import org.apache.http.config.ConnectionConfig;
 import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
 import org.apache.http.conn.ssl.TrustStrategy;
 import org.apache.http.cookie.Cookie;
@@ -21,6 +22,7 @@ import org.apache.http.entity.StringEntity;
 import org.apache.http.impl.client.BasicCookieStore;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.cookie.BasicClientCookie;
 import org.apache.http.message.BasicNameValuePair;
 import org.apache.http.ssl.SSLContextBuilder;
 import org.apache.http.util.EntityUtils;
@@ -28,16 +30,17 @@ import org.apache.http.util.EntityUtils;
 import javax.net.ssl.SSLContext;
 import java.io.BufferedReader;
 import java.io.InputStreamReader;
+import java.net.URL;
+import java.net.URLConnection;
 import java.security.cert.CertificateException;
 import java.security.cert.X509Certificate;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.*;
 
 @Slf4j
 public class HttpClientUtils {
     public CookieStore cookieStore = new BasicCookieStore();
 
-    public CloseableHttpClient createSSLClientDefault() {
+    public CloseableHttpClient createSSLClientDefault(IpPool ipPool) {
         try {
             SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {
                 //信任所有证书
@@ -46,16 +49,45 @@ public class HttpClientUtils {
                     return true;
                 }
             }).build();
-            RequestConfig globalConfig = RequestConfig.custom().setConnectTimeout(60000).setSocketTimeout(60000).setCookieSpec(CookieSpecs.STANDARD).build();
-            SSLConnectionSocketFactory sslFactory = new SSLConnectionSocketFactory(sslContext);
-            return HttpClients.custom().setDefaultCookieStore(cookieStore).setDefaultRequestConfig(globalConfig).setConnectionReuseStrategy((response, context) -> false).setSSLSocketFactory(sslFactory).build();
+            RequestConfig globalConfig = null;
+            SSLConnectionSocketFactory sslFactory = null;
+            ConnectionConfig connectionConfig = ConnectionConfig.custom()
+                    .setBufferSize(4096)
+                    .build();
+            if (ipPool != null) {
+                HttpHost proxy = new HttpHost(ipPool.getIp(), ipPool.getPort(), "http");
+                globalConfig = RequestConfig.custom().setProxy(proxy).setConnectTimeout(60000).setSocketTimeout(60000).setCookieSpec(CookieSpecs.STANDARD).build();
+                sslFactory = new SSLConnectionSocketFactory(sslContext);
+            } else {
+                globalConfig = RequestConfig.custom().setConnectTimeout(60000).setSocketTimeout(60000).setCookieSpec(CookieSpecs.STANDARD).build();
+                sslFactory = new SSLConnectionSocketFactory(sslContext);
+            }
+            return HttpClients.custom().setDefaultConnectionConfig(connectionConfig).setDefaultCookieStore(cookieStore).setDefaultRequestConfig(globalConfig).setConnectionReuseStrategy((response, context) -> false).setSSLSocketFactory(sslFactory).build();
         } catch (Exception e) {
             log.error("处理Https证书异常", e);
         }
         return HttpClients.createDefault();
     }
 
+    public String get(String urlStr) throws Exception {
+
+        URL url = new URL(urlStr);
+        URLConnection urlConnection = url.openConnection(); // 打开连接
+
+        System.out.println(urlConnection.getURL().toString());
+
+        BufferedReader br = new BufferedReader(new InputStreamReader(urlConnection.getInputStream(), "utf-8")); // 获取输入流
+        String line = null;
+        StringBuilder sb = new StringBuilder();
+        while ((line = br.readLine()) != null) {
+            sb.append(line + "\n");
+        }
+        br.close();
+        return sb.toString();
+    }
+
     public HttpClientEntity httpGetRequest(HttpClientEntity httpClientEntity) {
+        String result = null;
         try {
             CookieStore cs = httpClientEntity.getCookieStore();
             if (cs != null) {
@@ -65,10 +97,39 @@ public class HttpClientUtils {
             }
             HttpResponse response = null;
             HttpGet httpGet = new HttpGet(httpClientEntity.getUrl());
-            String result = null;
+
             CloseableHttpClient httpClient = httpClientEntity.getCloseableHttpClient();
             response = httpClient.execute(httpGet);
+            Header[] headers = response.getHeaders("Set-Cookie");
+            if (headers != null && headers.length > 0) {
+                for (Header header : headers) {
+                    System.out.println(header.getValue());
+                    String[] cookieArr = header.getValue().split(";");
+                    String[] ck = cookieArr[0].split("=");
+                    BasicClientCookie basicClientCookie = new BasicClientCookie(ck[0], ck[1]);
+                    Map<String, String> cookieMap = new HashMap<>();
+                    if (cookieArr.length > 1) {
+                        for (int i = 1; i < cookieArr.length; i++) {
+                            System.out.println(cookieArr[i]);
+                            String[] ckParam = cookieArr[i].split("=");
+                            if (ckParam.length > 1) {
+                                cookieMap.put(ckParam[0].trim(), ckParam[1]);
+                            }
+                        }
+                    }
 
+                    basicClientCookie.setPath(cookieMap.get("Path"));
+                    if (cookieMap.get("Domain") == null || cookieMap.get("Domain").equals("")) {
+                        if (httpClientEntity.getHeaders() != null && httpClientEntity.getHeaders().get("Origin") != null) {
+                            basicClientCookie.setDomain(httpClientEntity.getHeaders().get("Origin").split("//")[1]);
+                        }
+                    } else {
+                        basicClientCookie.setDomain(cookieMap.get("Domain"));
+                    }
+                    basicClientCookie.setExpiryDate(new Date(new Date().getTime() + 1000 * 60 * 60 * 24 * 365));
+                    cookieStore.addCookie(basicClientCookie);
+                }
+            }
             int statusCode = response.getStatusLine().getStatusCode();
             if (statusCode == HttpStatus.SC_MOVED_PERMANENTLY || statusCode == HttpStatus.SC_MOVED_TEMPORARILY) {
                 String newUrl = response.getFirstHeader("Location").getValue();
@@ -87,6 +148,7 @@ public class HttpClientUtils {
         } catch (Exception e) {
             e.printStackTrace();
         }
+        httpClientEntity.setResult(result);
         httpClientEntity.setCookieStore(cookieStore);
         return httpClientEntity;
     }
@@ -114,7 +176,31 @@ public class HttpClientUtils {
             httppost.setEntity(formEntity);
             CloseableHttpClient httpClient = httpClientEntity.getCloseableHttpClient();
             HttpResponse response = httpClient.execute(httppost);
-
+            Header[] headers = response.getHeaders("Set-Cookie");
+            if (headers != null && headers.length > 0) {
+                for (Header header : headers) {
+                    String[] cookieArr = header.getValue().split(";");
+                    String[] ck = cookieArr[0].split("=");
+                    BasicClientCookie basicClientCookie = new BasicClientCookie(ck[0], ck[1]);
+                    Map<String, String> cookieMap = new HashMap<>();
+                    if (cookieArr.length > 1) {
+                        for (int i = 1; i < cookieArr.length; i++) {
+                            String[] ckParam = cookieArr[i].split("=");
+                            if (ckParam.length > 1) {
+                                cookieMap.put(ckParam[0].trim(), ckParam[1]);
+                            }
+                        }
+                    }
+                    basicClientCookie.setPath(cookieMap.get("Path"));
+                    if (cookieMap.get("Domain") == null || cookieMap.get("Domain").equals("")) {
+                        basicClientCookie.setDomain(httpClientEntity.getHeaders().get("Origin").split("//")[1]);
+                    } else {
+                        basicClientCookie.setDomain(cookieMap.get("Domain"));
+                    }
+                    basicClientCookie.setExpiryDate(new Date(new Date().getTime() + 1000 * 60 * 60 * 24 * 365));
+                    cookieStore.addCookie(basicClientCookie);
+                }
+            }
             int statusCode = response.getStatusLine().getStatusCode();
             if (statusCode == HttpStatus.SC_MOVED_PERMANENTLY || statusCode == HttpStatus.SC_MOVED_TEMPORARILY) {
                 String newUrl = response.getFirstHeader("Location").getValue();
@@ -131,7 +217,6 @@ public class HttpClientUtils {
             log.error(e.getMessage());
         }
         httpClientEntity.setResult(result);
-        System.out.println(new Gson().toJson(cookieStore.getCookies()));
         httpClientEntity.setCookieStore(cookieStore);
         return httpClientEntity;
     }
@@ -155,7 +240,33 @@ public class HttpClientUtils {
             StringEntity entity = new StringEntity(json, "utf-8");
             httpPost.setEntity(entity);
             response = httpClient.execute(httpPost);
-
+            Header[] headers = response.getHeaders("Set-Cookie");
+            if (headers != null && headers.length > 0) {
+                for (Header header : headers) {
+                    System.out.println(header.getValue());
+                    String[] cookieArr = header.getValue().split(";");
+                    String[] ck = cookieArr[0].split("=");
+                    BasicClientCookie basicClientCookie = new BasicClientCookie(ck[0], ck[1]);
+                    Map<String, String> cookieMap = new HashMap<>();
+                    if (cookieArr.length > 1) {
+                        for (int i = 1; i < cookieArr.length; i++) {
+                            System.out.println(cookieArr[i]);
+                            String[] ckParam = cookieArr[i].split("=");
+                            if (ckParam.length > 1) {
+                                cookieMap.put(ckParam[0].trim(), ckParam[1]);
+                            }
+                        }
+                    }
+                    basicClientCookie.setPath(cookieMap.get("Path"));
+                    if (cookieMap.get("Domain") == null || cookieMap.get("Domain").equals("")) {
+                        basicClientCookie.setDomain(httpClientEntity.getHeaders().get("Origin").split("//")[1]);
+                    } else {
+                        basicClientCookie.setDomain(cookieMap.get("Domain"));
+                    }
+                    basicClientCookie.setExpiryDate(new Date(new Date().getTime() + 1000 * 60 * 60 * 24 * 365));
+                    cookieStore.addCookie(basicClientCookie);
+                }
+            }
             BufferedReader reader = new BufferedReader(new InputStreamReader(response.getEntity().getContent()));
             String line = null;
             StringBuilder builder = new StringBuilder();
@@ -168,7 +279,6 @@ public class HttpClientUtils {
             e.printStackTrace();
         }
         httpClientEntity.setCookieStore(cookieStore);
-        System.out.println(new Gson().toJson(cookieStore.getCookies()));
         return httpClientEntity;
     }
 }

+ 0 - 10
module-common/src/main/java/cn/com/ctop/common/module/utils/HttpUtils2.java

@@ -395,16 +395,6 @@ public class HttpUtils2 {
     public static Integer getKuaishouNumber(String key) {
         if (kuaishouNumberMap == null) {
             kuaishouNumberMap = new HashMap<>();
-//            kuaishouNumberMap.put("32.0#-6.0#526.0#729.0",0);//735
-//            kuaishouNumberMap.put("98.0#13.0#363.0#726.0",1);//713
-//            kuaishouNumberMap.put("32.0#13.0#527.0#732.0",2);//719
-//            kuaishouNumberMap.put("25.0#-6.0#525.0#730.0",3);//736
-//            kuaishouNumberMap.put("26.0#13.0#536.0#731.0",4);//718
-//            kuaishouNumberMap.put("33.0#-5.0#526.0#717.0",5);//722
-//            kuaishouNumberMap.put("39.0#-5.0#530.0#732.0",6);//737
-//            kuaishouNumberMap.put("38.0#13.0#536.0#717.0",7);//704
-//            kuaishouNumberMap.put("33.0#-7.0#525.0#731.0",8);//738
-//            kuaishouNumberMap.put("37.0#-7.0#521.0#730.0",9);//737
             kuaishouNumberMap.put("494.0#735.0", 0);
             kuaishouNumberMap.put("265.0#713.0", 1);
             kuaishouNumberMap.put("495.0#719.0", 2);

文件差異過大導致無法顯示
+ 161 - 129
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/graphql/service/impl/KuaishouWebInterfaceServiceImpl.java