|
@@ -1,12 +1,12 @@
|
|
|
package cn.com.ctop.common.module.utils;
|
|
|
|
|
|
import cn.com.ctop.common.module.entity.HttpClientEntity;
|
|
|
+import cn.com.ctop.common.module.entity.IpPool;
|
|
|
+import com.fasterxml.jackson.databind.JsonNode;
|
|
|
+import com.fasterxml.jackson.databind.ObjectMapper;
|
|
|
import com.google.gson.Gson;
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
-import org.apache.http.Header;
|
|
|
-import org.apache.http.HttpResponse;
|
|
|
-import org.apache.http.HttpStatus;
|
|
|
-import org.apache.http.NameValuePair;
|
|
|
+import org.apache.http.*;
|
|
|
import org.apache.http.client.CookieStore;
|
|
|
import org.apache.http.client.HttpClient;
|
|
|
import org.apache.http.client.config.CookieSpecs;
|
|
@@ -14,6 +14,7 @@ import org.apache.http.client.config.RequestConfig;
|
|
|
import org.apache.http.client.entity.UrlEncodedFormEntity;
|
|
|
import org.apache.http.client.methods.HttpGet;
|
|
|
import org.apache.http.client.methods.HttpPost;
|
|
|
+import org.apache.http.config.ConnectionConfig;
|
|
|
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
|
|
|
import org.apache.http.conn.ssl.TrustStrategy;
|
|
|
import org.apache.http.cookie.Cookie;
|
|
@@ -21,6 +22,7 @@ import org.apache.http.entity.StringEntity;
|
|
|
import org.apache.http.impl.client.BasicCookieStore;
|
|
|
import org.apache.http.impl.client.CloseableHttpClient;
|
|
|
import org.apache.http.impl.client.HttpClients;
|
|
|
+import org.apache.http.impl.cookie.BasicClientCookie;
|
|
|
import org.apache.http.message.BasicNameValuePair;
|
|
|
import org.apache.http.ssl.SSLContextBuilder;
|
|
|
import org.apache.http.util.EntityUtils;
|
|
@@ -28,16 +30,17 @@ import org.apache.http.util.EntityUtils;
|
|
|
import javax.net.ssl.SSLContext;
|
|
|
import java.io.BufferedReader;
|
|
|
import java.io.InputStreamReader;
|
|
|
+import java.net.URL;
|
|
|
+import java.net.URLConnection;
|
|
|
import java.security.cert.CertificateException;
|
|
|
import java.security.cert.X509Certificate;
|
|
|
-import java.util.ArrayList;
|
|
|
-import java.util.List;
|
|
|
+import java.util.*;
|
|
|
|
|
|
@Slf4j
|
|
|
public class HttpClientUtils {
|
|
|
public CookieStore cookieStore = new BasicCookieStore();
|
|
|
|
|
|
- public CloseableHttpClient createSSLClientDefault() {
|
|
|
+ public CloseableHttpClient createSSLClientDefault(IpPool ipPool) {
|
|
|
try {
|
|
|
SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {
|
|
|
//信任所有证书
|
|
@@ -46,16 +49,45 @@ public class HttpClientUtils {
|
|
|
return true;
|
|
|
}
|
|
|
}).build();
|
|
|
- RequestConfig globalConfig = RequestConfig.custom().setConnectTimeout(60000).setSocketTimeout(60000).setCookieSpec(CookieSpecs.STANDARD).build();
|
|
|
- SSLConnectionSocketFactory sslFactory = new SSLConnectionSocketFactory(sslContext);
|
|
|
- return HttpClients.custom().setDefaultCookieStore(cookieStore).setDefaultRequestConfig(globalConfig).setConnectionReuseStrategy((response, context) -> false).setSSLSocketFactory(sslFactory).build();
|
|
|
+ RequestConfig globalConfig = null;
|
|
|
+ SSLConnectionSocketFactory sslFactory = null;
|
|
|
+ ConnectionConfig connectionConfig = ConnectionConfig.custom()
|
|
|
+ .setBufferSize(4096)
|
|
|
+ .build();
|
|
|
+ if (ipPool != null) {
|
|
|
+ HttpHost proxy = new HttpHost(ipPool.getIp(), ipPool.getPort(), "http");
|
|
|
+ globalConfig = RequestConfig.custom().setProxy(proxy).setConnectTimeout(60000).setSocketTimeout(60000).setCookieSpec(CookieSpecs.STANDARD).build();
|
|
|
+ sslFactory = new SSLConnectionSocketFactory(sslContext);
|
|
|
+ } else {
|
|
|
+ globalConfig = RequestConfig.custom().setConnectTimeout(60000).setSocketTimeout(60000).setCookieSpec(CookieSpecs.STANDARD).build();
|
|
|
+ sslFactory = new SSLConnectionSocketFactory(sslContext);
|
|
|
+ }
|
|
|
+ return HttpClients.custom().setDefaultConnectionConfig(connectionConfig).setDefaultCookieStore(cookieStore).setDefaultRequestConfig(globalConfig).setConnectionReuseStrategy((response, context) -> false).setSSLSocketFactory(sslFactory).build();
|
|
|
} catch (Exception e) {
|
|
|
log.error("处理Https证书异常", e);
|
|
|
}
|
|
|
return HttpClients.createDefault();
|
|
|
}
|
|
|
|
|
|
+ public String get(String urlStr) throws Exception {
|
|
|
+
|
|
|
+ URL url = new URL(urlStr);
|
|
|
+ URLConnection urlConnection = url.openConnection(); // 打开连接
|
|
|
+
|
|
|
+ System.out.println(urlConnection.getURL().toString());
|
|
|
+
|
|
|
+ BufferedReader br = new BufferedReader(new InputStreamReader(urlConnection.getInputStream(), "utf-8")); // 获取输入流
|
|
|
+ String line = null;
|
|
|
+ StringBuilder sb = new StringBuilder();
|
|
|
+ while ((line = br.readLine()) != null) {
|
|
|
+ sb.append(line + "\n");
|
|
|
+ }
|
|
|
+ br.close();
|
|
|
+ return sb.toString();
|
|
|
+ }
|
|
|
+
|
|
|
public HttpClientEntity httpGetRequest(HttpClientEntity httpClientEntity) {
|
|
|
+ String result = null;
|
|
|
try {
|
|
|
CookieStore cs = httpClientEntity.getCookieStore();
|
|
|
if (cs != null) {
|
|
@@ -65,10 +97,39 @@ public class HttpClientUtils {
|
|
|
}
|
|
|
HttpResponse response = null;
|
|
|
HttpGet httpGet = new HttpGet(httpClientEntity.getUrl());
|
|
|
- String result = null;
|
|
|
+
|
|
|
CloseableHttpClient httpClient = httpClientEntity.getCloseableHttpClient();
|
|
|
response = httpClient.execute(httpGet);
|
|
|
+ Header[] headers = response.getHeaders("Set-Cookie");
|
|
|
+ if (headers != null && headers.length > 0) {
|
|
|
+ for (Header header : headers) {
|
|
|
+ System.out.println(header.getValue());
|
|
|
+ String[] cookieArr = header.getValue().split(";");
|
|
|
+ String[] ck = cookieArr[0].split("=");
|
|
|
+ BasicClientCookie basicClientCookie = new BasicClientCookie(ck[0], ck[1]);
|
|
|
+ Map<String, String> cookieMap = new HashMap<>();
|
|
|
+ if (cookieArr.length > 1) {
|
|
|
+ for (int i = 1; i < cookieArr.length; i++) {
|
|
|
+ System.out.println(cookieArr[i]);
|
|
|
+ String[] ckParam = cookieArr[i].split("=");
|
|
|
+ if (ckParam.length > 1) {
|
|
|
+ cookieMap.put(ckParam[0].trim(), ckParam[1]);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
|
|
|
+ basicClientCookie.setPath(cookieMap.get("Path"));
|
|
|
+ if (cookieMap.get("Domain") == null || cookieMap.get("Domain").equals("")) {
|
|
|
+ if (httpClientEntity.getHeaders() != null && httpClientEntity.getHeaders().get("Origin") != null) {
|
|
|
+ basicClientCookie.setDomain(httpClientEntity.getHeaders().get("Origin").split("//")[1]);
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ basicClientCookie.setDomain(cookieMap.get("Domain"));
|
|
|
+ }
|
|
|
+ basicClientCookie.setExpiryDate(new Date(new Date().getTime() + 1000 * 60 * 60 * 24 * 365));
|
|
|
+ cookieStore.addCookie(basicClientCookie);
|
|
|
+ }
|
|
|
+ }
|
|
|
int statusCode = response.getStatusLine().getStatusCode();
|
|
|
if (statusCode == HttpStatus.SC_MOVED_PERMANENTLY || statusCode == HttpStatus.SC_MOVED_TEMPORARILY) {
|
|
|
String newUrl = response.getFirstHeader("Location").getValue();
|
|
@@ -87,6 +148,7 @@ public class HttpClientUtils {
|
|
|
} catch (Exception e) {
|
|
|
e.printStackTrace();
|
|
|
}
|
|
|
+ httpClientEntity.setResult(result);
|
|
|
httpClientEntity.setCookieStore(cookieStore);
|
|
|
return httpClientEntity;
|
|
|
}
|
|
@@ -114,7 +176,31 @@ public class HttpClientUtils {
|
|
|
httppost.setEntity(formEntity);
|
|
|
CloseableHttpClient httpClient = httpClientEntity.getCloseableHttpClient();
|
|
|
HttpResponse response = httpClient.execute(httppost);
|
|
|
-
|
|
|
+ Header[] headers = response.getHeaders("Set-Cookie");
|
|
|
+ if (headers != null && headers.length > 0) {
|
|
|
+ for (Header header : headers) {
|
|
|
+ String[] cookieArr = header.getValue().split(";");
|
|
|
+ String[] ck = cookieArr[0].split("=");
|
|
|
+ BasicClientCookie basicClientCookie = new BasicClientCookie(ck[0], ck[1]);
|
|
|
+ Map<String, String> cookieMap = new HashMap<>();
|
|
|
+ if (cookieArr.length > 1) {
|
|
|
+ for (int i = 1; i < cookieArr.length; i++) {
|
|
|
+ String[] ckParam = cookieArr[i].split("=");
|
|
|
+ if (ckParam.length > 1) {
|
|
|
+ cookieMap.put(ckParam[0].trim(), ckParam[1]);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ basicClientCookie.setPath(cookieMap.get("Path"));
|
|
|
+ if (cookieMap.get("Domain") == null || cookieMap.get("Domain").equals("")) {
|
|
|
+ basicClientCookie.setDomain(httpClientEntity.getHeaders().get("Origin").split("//")[1]);
|
|
|
+ } else {
|
|
|
+ basicClientCookie.setDomain(cookieMap.get("Domain"));
|
|
|
+ }
|
|
|
+ basicClientCookie.setExpiryDate(new Date(new Date().getTime() + 1000 * 60 * 60 * 24 * 365));
|
|
|
+ cookieStore.addCookie(basicClientCookie);
|
|
|
+ }
|
|
|
+ }
|
|
|
int statusCode = response.getStatusLine().getStatusCode();
|
|
|
if (statusCode == HttpStatus.SC_MOVED_PERMANENTLY || statusCode == HttpStatus.SC_MOVED_TEMPORARILY) {
|
|
|
String newUrl = response.getFirstHeader("Location").getValue();
|
|
@@ -131,7 +217,6 @@ public class HttpClientUtils {
|
|
|
log.error(e.getMessage());
|
|
|
}
|
|
|
httpClientEntity.setResult(result);
|
|
|
- System.out.println(new Gson().toJson(cookieStore.getCookies()));
|
|
|
httpClientEntity.setCookieStore(cookieStore);
|
|
|
return httpClientEntity;
|
|
|
}
|
|
@@ -155,7 +240,33 @@ public class HttpClientUtils {
|
|
|
StringEntity entity = new StringEntity(json, "utf-8");
|
|
|
httpPost.setEntity(entity);
|
|
|
response = httpClient.execute(httpPost);
|
|
|
-
|
|
|
+ Header[] headers = response.getHeaders("Set-Cookie");
|
|
|
+ if (headers != null && headers.length > 0) {
|
|
|
+ for (Header header : headers) {
|
|
|
+ System.out.println(header.getValue());
|
|
|
+ String[] cookieArr = header.getValue().split(";");
|
|
|
+ String[] ck = cookieArr[0].split("=");
|
|
|
+ BasicClientCookie basicClientCookie = new BasicClientCookie(ck[0], ck[1]);
|
|
|
+ Map<String, String> cookieMap = new HashMap<>();
|
|
|
+ if (cookieArr.length > 1) {
|
|
|
+ for (int i = 1; i < cookieArr.length; i++) {
|
|
|
+ System.out.println(cookieArr[i]);
|
|
|
+ String[] ckParam = cookieArr[i].split("=");
|
|
|
+ if (ckParam.length > 1) {
|
|
|
+ cookieMap.put(ckParam[0].trim(), ckParam[1]);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ basicClientCookie.setPath(cookieMap.get("Path"));
|
|
|
+ if (cookieMap.get("Domain") == null || cookieMap.get("Domain").equals("")) {
|
|
|
+ basicClientCookie.setDomain(httpClientEntity.getHeaders().get("Origin").split("//")[1]);
|
|
|
+ } else {
|
|
|
+ basicClientCookie.setDomain(cookieMap.get("Domain"));
|
|
|
+ }
|
|
|
+ basicClientCookie.setExpiryDate(new Date(new Date().getTime() + 1000 * 60 * 60 * 24 * 365));
|
|
|
+ cookieStore.addCookie(basicClientCookie);
|
|
|
+ }
|
|
|
+ }
|
|
|
BufferedReader reader = new BufferedReader(new InputStreamReader(response.getEntity().getContent()));
|
|
|
String line = null;
|
|
|
StringBuilder builder = new StringBuilder();
|
|
@@ -168,7 +279,6 @@ public class HttpClientUtils {
|
|
|
e.printStackTrace();
|
|
|
}
|
|
|
httpClientEntity.setCookieStore(cookieStore);
|
|
|
- System.out.println(new Gson().toJson(cookieStore.getCookies()));
|
|
|
return httpClientEntity;
|
|
|
}
|
|
|
}
|