Преглед изворни кода

完善代理池错误处理

xuzuoyun пре 5 година
родитељ
комит
ca3971fdf3

+ 17 - 2
module-common/src/main/java/cn/com/ctop/common/module/service/impl/IpPoolServiceImpl.java

@@ -10,12 +10,13 @@ import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Service;
 
 import java.util.Date;
 import java.util.List;
 
-
+@Slf4j
 @Service
 public class IpPoolServiceImpl extends ServiceImpl<IpPoolMapper, IpPool> implements IIpPoolService {
 
@@ -44,7 +45,21 @@ public class IpPoolServiceImpl extends ServiceImpl<IpPoolMapper, IpPool> impleme
                 List<IpPool> ipList = JSON.parseObject(jsonNode.get("data").toString(), new TypeReference<List<IpPool>>() {
                 });
                 if (ipList != null && ipList.size() > 0) {
-                    this.saveBatch(ipList);
+                    for (IpPool ipPool : ipList) {
+                        Thread thread = new Thread() {
+                            @Override
+                            public void run() {
+                                Boolean avaliable = httpClientUtils.checkProxyIp(ipPool.getIp(), ipPool.getPort());
+                                if (!avaliable) {
+                                    ipPool.setStatus(2);
+                                    log.info("ip" + ipPool.getIp() + " is not avaliable");
+                                }
+                                save(ipPool);
+                            }
+                        };
+                        thread.start();
+                    }
+//                    this.saveBatch(ipList);
                 }
             }
         } catch (Exception e) {

+ 30 - 8
module-common/src/main/java/cn/com/ctop/common/module/utils/HttpClientUtils.java

@@ -70,13 +70,39 @@ public class HttpClientUtils {
         return HttpClients.createDefault();
     }
 
-    public String get(String urlStr) throws Exception {
+    public Boolean checkProxyIp(String ip, int port) {
+        int status = 0;
+        try {
+            SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {
+                //信任所有证书
+                @Override
+                public boolean isTrusted(X509Certificate[] chain, String authType) throws CertificateException {
+                    return true;
+                }
+            }).build();
+            ConnectionConfig connectionConfig = ConnectionConfig.custom()
+                    .setBufferSize(4096)
+                    .build();
+            HttpHost proxy = new HttpHost(ip, port, "http");
+            RequestConfig globalConfig = RequestConfig.custom().setProxy(proxy).setConnectTimeout(1000).setSocketTimeout(1000).setCookieSpec(CookieSpecs.STANDARD).build();
+            SSLConnectionSocketFactory sslFactory = new SSLConnectionSocketFactory(sslContext);
+
+            HttpClient httpClient = HttpClients.custom().setDefaultConnectionConfig(connectionConfig).setDefaultCookieStore(cookieStore).setDefaultRequestConfig(globalConfig).setConnectionReuseStrategy((response, context) -> false).setSSLSocketFactory(sslFactory).build();
+            HttpGet httpGet = new HttpGet("https://www.baidu.com");
+            status = httpClient.execute(httpGet).getStatusLine().getStatusCode();
+        } catch (Exception e) {
+            log.info("ip: " + ip + " is not aviable");
+        }
+        if (status == 200) {
+            return true;
+        } else {
+            return false;
+        }
+    }
 
+    public String get(String urlStr) throws Exception {
         URL url = new URL(urlStr);
         URLConnection urlConnection = url.openConnection(); // 打开连接
-
-        System.out.println(urlConnection.getURL().toString());
-
         BufferedReader br = new BufferedReader(new InputStreamReader(urlConnection.getInputStream(), "utf-8")); // 获取输入流
         String line = null;
         StringBuilder sb = new StringBuilder();
@@ -104,14 +130,12 @@ public class HttpClientUtils {
             Header[] headers = response.getHeaders("Set-Cookie");
             if (headers != null && headers.length > 0) {
                 for (Header header : headers) {
-                    System.out.println(header.getValue());
                     String[] cookieArr = header.getValue().split(";");
                     String[] ck = cookieArr[0].split("=");
                     BasicClientCookie basicClientCookie = new BasicClientCookie(ck[0], ck[1]);
                     Map<String, String> cookieMap = new HashMap<>();
                     if (cookieArr.length > 1) {
                         for (int i = 1; i < cookieArr.length; i++) {
-                            System.out.println(cookieArr[i]);
                             String[] ckParam = cookieArr[i].split("=");
                             if (ckParam.length > 1) {
                                 cookieMap.put(ckParam[0].trim(), ckParam[1]);
@@ -253,14 +277,12 @@ public class HttpClientUtils {
             Header[] headers = response.getHeaders("Set-Cookie");
             if (headers != null && headers.length > 0) {
                 for (Header header : headers) {
-                    System.out.println(header.getValue());
                     String[] cookieArr = header.getValue().split(";");
                     String[] ck = cookieArr[0].split("=");
                     BasicClientCookie basicClientCookie = new BasicClientCookie(ck[0], ck[1]);
                     Map<String, String> cookieMap = new HashMap<>();
                     if (cookieArr.length > 1) {
                         for (int i = 1; i < cookieArr.length; i++) {
-                            System.out.println(cookieArr[i]);
                             String[] ckParam = cookieArr[i].split("=");
                             if (ckParam.length > 1) {
                                 cookieMap.put(ckParam[0].trim(), ckParam[1]);

+ 7 - 7
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/graphql/service/impl/KuaishouWebInterfaceServiceImpl.java

@@ -133,9 +133,9 @@ public class KuaishouWebInterfaceServiceImpl implements IKuaishouWebInterfaceSer
                 ObjectMapper mapper = new ObjectMapper();
                 resultMap = mapper.readValue(httpClientEntity.getResult(), new TypeReference<Map<String, Object>>() {
                 });
+                JsonNode resultNode = mapper.readTree(httpClientEntity.getResult()).get("data");
 //                checkCaptcha(httpClientEntity.getResult());
-                String psor = mapper.readTree(httpClientEntity.getResult()).get("data").get("publicFeeds").get("pcursor").asText();
-                if (resultMap.containsKey("captcha") || psor == null) {
+                if (resultNode.get("subCommentList").isNull() || resultNode.get("subCommentList").get("pcursor").isNull()) {
                     ipPool.setStatus(2);
                     iIpPoolService.updateById(ipPool);
                     return subCommentList(ksid, photoId, rootCommentId, pcursor);
@@ -187,9 +187,8 @@ public class KuaishouWebInterfaceServiceImpl implements IKuaishouWebInterfaceSer
                 ObjectMapper mapper = new ObjectMapper();
                 resultMap = mapper.readValue(httpClientEntity.getResult(), new TypeReference<Map<String, Object>>() {
                 });
-                String psor = mapper.readTree(httpClientEntity.getResult()).get("data").get("publicFeeds").get("pcursor").asText();
-//                checkCaptcha(httpClientEntity.getResult());
-                if (resultMap.containsKey("captcha") || psor == null) {
+                JsonNode resultNode = mapper.readTree(httpClientEntity.getResult()).get("data");
+                if (resultNode.get("shortVideoCommentList").isNull() || resultNode.get("shortVideoCommentList").get("pcursor").isNull()) {
                     ipPool.setStatus(2);
                     iIpPoolService.updateById(ipPool);
                     return commentList(ksid, photoId, pcursor);
@@ -259,8 +258,9 @@ public class KuaishouWebInterfaceServiceImpl implements IKuaishouWebInterfaceSer
                 ObjectMapper mapper = new ObjectMapper();
                 resultMap = mapper.readValue(result, new TypeReference<Map<String, Object>>() {
                 });
-                String psor = mapper.readTree(result).get("data").get("publicFeeds").get("pcursor").asText();
-                if (resultMap.containsKey("captcha") || psor == null) {
+                JsonNode resultNode = mapper.readTree(httpClientEntity.getResult()).get("data");
+//                String psor = mapper.readTree(result).get("data").get("publicFeeds").get("pcursor").asText();
+                if (resultNode.get("publicFeeds").isNull() || resultNode.get("publicFeeds").get("pcursor").isNull()) {
                     ipPool.setStatus(2);
                     iIpPoolService.updateById(ipPool);
                     return videoList(ksid, pcursor);