Преглед изворни кода

Merge remote-tracking branch 'origin/master'

syh пре 4 година
родитељ
комит
2e8ffda93a

+ 29 - 5
module-common/src/main/java/cn/com/ctop/common/module/utils/HttpUtils.java

@@ -7,10 +7,7 @@ import org.apache.fontbox.ttf.CmapSubtable;
 import org.apache.fontbox.ttf.GlyphData;
 import org.apache.fontbox.ttf.TTFParser;
 import org.apache.fontbox.ttf.TrueTypeFont;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpResponse;
-import org.apache.http.HttpStatus;
-import org.apache.http.NameValuePair;
+import org.apache.http.*;
 import org.apache.http.client.CookieStore;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.config.CookieSpecs;
@@ -29,6 +26,7 @@ import org.apache.http.impl.client.BasicCookieStore;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.cookie.BasicClientCookie;
 import org.apache.http.message.BasicNameValuePair;
 import org.apache.http.ssl.SSLContextBuilder;
 import org.apache.http.util.EntityUtils;
@@ -417,6 +415,18 @@ public class HttpUtils {
         return strReturn;
     }
 
+    public static void main(String[] args){
+        String res = HttpUtils.getKuaishouShareUrl("http://yongzhou.s.gifshow.com/i/photo/lwx?userId=1400097524&photoId=5191805968879099278");
+        String path = res.split("\\?")[0];
+        String photoId = path.substring(path.lastIndexOf("/")+1,path.length());
+        path =path.substring(0,path.lastIndexOf("/"));
+        String uid = path.substring(path.lastIndexOf("/")+1,path.length());
+        System.out.println(uid);
+        System.out.println(photoId);
+        res = HttpUtils.postKuaishouVideoUrl("https://live.kuaishou.com/m_graphql",uid,photoId);
+        System.out.println(res);
+    }
+
     public static String postKuaishouVideoUrl(String url,String uid,String photoId){
         HttpClient httpClient = createSslClientDefault();
         String strReturn = "";
@@ -482,8 +492,10 @@ public class HttpUtils {
             httpGet.setHeader("User-Agent", USER_AGENT);
             response = httpClient.execute(httpGet);
             int statusCode = response.getStatusLine().getStatusCode();
+            System.out.println(statusCode);
             if (statusCode == HttpStatus.SC_MOVED_PERMANENTLY || statusCode == HttpStatus.SC_MOVED_TEMPORARILY) {
                 String newUrl = response.getFirstHeader("Location").getValue();
+                System.out.println(newUrl);
                 return httpGetRequest(newUrl);
             } else if (statusCode == HttpStatus.SC_OK) {
                 BufferedReader reader = new BufferedReader(new InputStreamReader(response.getEntity().getContent()));
@@ -746,7 +758,7 @@ public class HttpUtils {
                 urlStr = url;
             }
             HttpGet httpget = new HttpGet(urlStr);
-            httpget.setHeader("User-Agent", USER_AGENT);
+//            httpget.setHeader("User-Agent", USER_AGENT);
             if (!Check.isNullMap(headers)) {
                 for (String key : headers.keySet()) {
                     httpget.setHeader(key, headers.get(key));
@@ -755,6 +767,17 @@ public class HttpUtils {
             HttpEntity respEntity;
             HttpResponse response = httpclient.execute(httpget);
             respEntity = response.getEntity();
+            Header[] responseHeaders = response.getHeaders("Set-Cookie");
+            if(responseHeaders != null){
+
+                for(Header header : responseHeaders){
+                    String[] ckArray = header.getValue().split(";");
+                    BasicClientCookie ck = new BasicClientCookie(ckArray[0].split("=")[0], ckArray[0].split("=")[1]);
+                    ck.setPath(ckArray[1].split("=")[1]);
+                    ck.setDomain(ckArray[2].split("=")[1]);
+                    HttpUtils.COOKIESTORE.addCookie(ck);
+                }
+            }
             result = EntityUtils.toString(respEntity);
         } catch (Exception e) {
             log.error("http get请求异常");
@@ -899,6 +922,7 @@ public class HttpUtils {
     }
 
     public static JSONObject feishuGetRequest(String accessToken, String url, JSONObject params) {
+        System.out.println(url);
         // 构造请求
         HttpEntityEnclosingRequestBase httpEntity = new HttpEntityEnclosingRequestBase() {
             @Override

+ 4 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/taobao/service/TaobaoJhsbybtService.java

@@ -0,0 +1,4 @@
+package cn.com.ctop.crawler.modules.taobao.service;
+
+public interface TaobaoJhsbybtService {
+}

Разлика између датотеке није приказан због своје велике величине
+ 122 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/taobao/service/impl/TaobaoJhsbybtServiceImpl.java


+ 7 - 7
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/impl/KuaishouInterfaceServiceImpl.java

@@ -3194,8 +3194,8 @@ public class KuaishouInterfaceServiceImpl implements IKuaishouInterfaceService {
             headers.put("Content-Type", "application/json");
             headers.put("Access-Token", token);
             Map<String, Object> param = new HashMap<>();
-            JSONArray viewComps = new JSONArray();
-            viewComps.add(viewComp);
+          /*  JSONArray viewComps = new JSONArray();
+            viewComps.add(viewComp);*/
             param.put("advertiser_id", accountId);
             param.put("page_size", 500);
             String result = HttpUtils.httpPostRequest(url, param, headers);
@@ -3215,11 +3215,10 @@ public class KuaishouInterfaceServiceImpl implements IKuaishouInterfaceService {
                                         for (int j = 0; j < compsArr.size(); j++) {
                                             JSONObject compJson = compsArr.getJSONObject(j);
                                             if (!Check.isNull(compJson)) {
-                                                Integer type = compJson.getInteger("type");
-                                                if (type == 7) {
-                                                    JSONObject props = compJson.getJSONObject("props");
-                                                    if (!Check.isNull(props)) {
-                                                        Long returnAppId = props.getLong("appId");
+                                                JSONObject props = compJson.getJSONObject("props");
+                                                if (!Check.isNull(props)) {
+                                                    Long returnAppId = props.getLong("appId");
+                                                    if (!Check.isNull(returnAppId)) {
                                                         if (returnAppId.equals(appId)) {
                                                             JSONObject returnJson = new JSONObject();
                                                             returnJson.put("siteId", jsonObject.getString("id"));
@@ -3228,6 +3227,7 @@ public class KuaishouInterfaceServiceImpl implements IKuaishouInterfaceService {
                                                             returnArr.add(returnJson);
                                                         }
                                                     }
+
                                                 }
                                             }
                                         }