Kaynağa Gözat

快手音乐爬虫

xuzuoyun 4 yıl önce
ebeveyn
işleme
bfd1c179e6

Dosya farkı çok büyük olduğundan ihmal edildi
+ 59 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/core/service/impl/KuaishouCrawlerServiceImpl.java


+ 18 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/core/util/HttpParamUtil.java

@@ -42,6 +42,24 @@ public class HttpParamUtil {
         return sb.toString().substring(1);
     }
 
+    public static <K, V> String mapToUrlStringNoEncoder(Map<K, V> map, List<String> ignoreKeyList){
+        if (map == null) {
+            return null;
+        }
+        StringBuffer sb = new StringBuffer();
+        try{
+            for(K key :map.keySet()){
+                if(ignoreKeyList!=null && ignoreKeyList.contains(key)){
+                    continue;
+                }
+                sb.append("&"+key + "=" + String.valueOf(map.get(key)==null?"":map.get(key)));
+            }
+        }catch (Exception e){
+            e.printStackTrace();
+        }
+        return sb.toString().substring(1);
+    }
+
     public static Map<String,String> urlParamToMap(String urlParam){
         if (urlParam == null || "".equalsIgnoreCase(urlParam)) {
             return null;

+ 8 - 1
module-crawler/src/main/java/cn/com/ctop/crawler/modules/core/util/KuaishouUtil.java

@@ -36,17 +36,21 @@ public class KuaishouUtil {
         Map<String,String> dynamicParams = httpEntity.getHttpBody().getDynamicParams();
         List<String> ignoreKeyList = new ArrayList<String>();
         ignoreKeyList.add("sig");
+        ignoreKeyList.add("__NS_sig3");
         ignoreKeyList.add("__NStokensig");
         String staticQueryString = HttpParamUtil.mapToUrlString(staticParams,null);
         Map<String,String> allMap = new HashMap<String,String>();
         allMap.putAll(staticParams);
         allMap.putAll(dynamicParams);
         if (staticParams.get("kpf") != null && "ANDROID_PHONE".equals(staticParams.get("kpf"))) {
+            System.out.println(HttpParamUtil.mapToString(SortUtil.sortMapByKey(allMap), ignoreKeyList));
             String sig = SignUtil.getMd5Sign(HttpParamUtil.mapToString(SortUtil.sortMapByKey(allMap), ignoreKeyList), salt);
             dynamicParams.put("sig",sig);
             allMap.put("sig",sig);
+            System.out.println(sig);
         }
-        String dynamicQueryString = HttpParamUtil.mapToUrlString(dynamicParams,null);
+
+        String dynamicQueryString = HttpParamUtil.mapToUrlStringNoEncoder(dynamicParams,null);
         BasicCookieStore cookieStore = new BasicCookieStore();
         for(HttpCookie cookie : httpEntity.getCookies()){
             BasicClientCookie clientCookie = new BasicClientCookie(cookie.getName(),cookie.getValue());
@@ -72,6 +76,7 @@ public class KuaishouUtil {
             }
             CloseableHttpClient httpClient = HttpClients.custom().setDefaultCookieStore(cookieStore).setDefaultRequestConfig(globalConfig).setConnectionReuseStrategy((response, context) -> false ).setSSLSocketFactory(sslFactory).build();
             HttpPost httppost = new HttpPost(httpEntity.getUrl()+httpEntity.getPath()+"?"+staticQueryString);
+            System.out.println(httpEntity.getUrl()+httpEntity.getPath()+"?"+staticQueryString);
             HttpHeader header = httpEntity.getHttpHeader();
             Map<String,String> headers = header.getHeaders();
             if(headers != null){
@@ -80,10 +85,12 @@ public class KuaishouUtil {
                 }
             }
             httppost.setEntity(new StringEntity(dynamicQueryString));
+            System.out.println(dynamicQueryString);
             HttpResponse response = httpClient.execute(httppost);
             int code = response.getStatusLine().getStatusCode();
             org.apache.http.HttpEntity respentity = response.getEntity();
             String strReturn = EntityUtils.toString(respentity);
+            System.out.println(strReturn);
             long endTime = System.currentTimeMillis();
             crawlerLog.setUrl(httpEntity.getUrl());
             crawlerLog.setPath(httpEntity.getPath());

+ 1 - 1
module-crawler/src/main/java/cn/com/ctop/crawler/modules/taobao/service/impl/TaobaoJhsbybtServiceImpl.java

@@ -57,7 +57,7 @@ public class TaobaoJhsbybtServiceImpl implements TaobaoJhsbybtService {
 
     public static void main(String[] args){
         try {
-            PrintStream out = new PrintStream("D:/bybt0827.csv");
+            PrintStream out = new PrintStream("D:/bybt0901.csv");
             System.setOut(out);
         } catch (FileNotFoundException e) {
             e.printStackTrace();