|
@@ -1,21 +1,19 @@
|
|
|
package cn.com.ctop.crawler.modules.core.util;
|
|
|
|
|
|
+import cn.com.ctop.common.module.entity.IpPool;
|
|
|
import cn.com.ctop.crawler.modules.core.entity.HttpBody;
|
|
|
import cn.com.ctop.crawler.modules.core.entity.HttpCookie;
|
|
|
import cn.com.ctop.crawler.modules.core.entity.HttpEntity;
|
|
|
import cn.com.ctop.crawler.modules.core.entity.HttpHeader;
|
|
|
import cn.com.ctop.crawler.modules.log.entity.CrawlerLog;
|
|
|
import com.google.gson.Gson;
|
|
|
+import org.apache.http.HttpHost;
|
|
|
import org.apache.http.HttpResponse;
|
|
|
-import org.apache.http.client.CookieStore;
|
|
|
import org.apache.http.client.config.CookieSpecs;
|
|
|
import org.apache.http.client.config.RequestConfig;
|
|
|
import org.apache.http.client.methods.HttpPost;
|
|
|
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
|
|
|
import org.apache.http.conn.ssl.TrustStrategy;
|
|
|
-import org.apache.http.cookie.ClientCookie;
|
|
|
-import org.apache.http.cookie.Cookie;
|
|
|
-import org.apache.http.cookie.CookieOrigin;
|
|
|
import org.apache.http.entity.StringEntity;
|
|
|
import org.apache.http.impl.client.BasicCookieStore;
|
|
|
import org.apache.http.impl.client.CloseableHttpClient;
|
|
@@ -30,7 +28,7 @@ import java.security.cert.X509Certificate;
|
|
|
import java.util.*;
|
|
|
|
|
|
public class KuaishouUtil {
|
|
|
- public static CrawlerLog requestData(HttpEntity httpEntity){
|
|
|
+ public static CrawlerLog requestData(HttpEntity httpEntity, IpPool ipPool) {
|
|
|
long beginTime = System.currentTimeMillis();
|
|
|
CrawlerLog crawlerLog = new CrawlerLog();
|
|
|
String salt = "382700b563f4";
|
|
@@ -64,8 +62,14 @@ public class KuaishouUtil {
|
|
|
return true;
|
|
|
}
|
|
|
}).build();
|
|
|
- RequestConfig globalConfig = RequestConfig.custom().setConnectTimeout(30000).setSocketTimeout(30000).setCookieSpec(CookieSpecs.STANDARD).build();
|
|
|
+ RequestConfig globalConfig = null;
|
|
|
SSLConnectionSocketFactory sslFactory = new SSLConnectionSocketFactory(sslContext);
|
|
|
+ if (ipPool != null) {
|
|
|
+ HttpHost proxy = new HttpHost(ipPool.getIp(), ipPool.getPort(), "http");
|
|
|
+ globalConfig = RequestConfig.custom().setProxy(proxy).setConnectTimeout(60000).setSocketTimeout(60000).setCookieSpec(CookieSpecs.STANDARD).build();
|
|
|
+ } else {
|
|
|
+ globalConfig = RequestConfig.custom().setConnectTimeout(60000).setSocketTimeout(60000).setCookieSpec(CookieSpecs.STANDARD).build();
|
|
|
+ }
|
|
|
CloseableHttpClient httpClient = HttpClients.custom().setDefaultCookieStore(cookieStore).setDefaultRequestConfig(globalConfig).setConnectionReuseStrategy((response, context) -> false ).setSSLSocketFactory(sslFactory).build();
|
|
|
HttpPost httppost = new HttpPost(httpEntity.getUrl()+httpEntity.getPath()+"?"+staticQueryString);
|
|
|
HttpHeader header = httpEntity.getHttpHeader();
|
|
@@ -106,7 +110,7 @@ public class KuaishouUtil {
|
|
|
return String.valueOf(nextInt);
|
|
|
}
|
|
|
|
|
|
- public static CrawlerLog kuaishouAppDataFeatch(String url,String path,String urlParam,String bodyParam,String headerParam,String token){
|
|
|
+ public static CrawlerLog kuaishouAppDataFeatch(String url, String path, String urlParam, String bodyParam, String headerParam, String token, IpPool ipPool) {
|
|
|
HttpEntity httpEntity = new HttpEntity();
|
|
|
Map<String,String> staticParams = HttpParamUtil.urlParamToMap(urlParam);
|
|
|
Map<String,String> dynamicParams = HttpParamUtil.urlParamToMap(bodyParam);
|
|
@@ -128,8 +132,6 @@ public class KuaishouUtil {
|
|
|
httpBody.setStaticParams(staticParams);
|
|
|
httpEntity.setHttpHeader(header);
|
|
|
httpEntity.setHttpBody(httpBody);
|
|
|
- return KuaishouUtil.requestData(httpEntity);
|
|
|
+ return KuaishouUtil.requestData(httpEntity, ipPool);
|
|
|
}
|
|
|
-
|
|
|
-
|
|
|
}
|