xuzuoyun 4 роки тому
батько
коміт
8c6f381642

+ 4 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/taobao/service/TaobaoTttmService.java

@@ -0,0 +1,4 @@
+package cn.com.ctop.crawler.modules.taobao.service;
+
+public interface TaobaoTttmService {
+}

+ 3 - 2
module-crawler/src/main/java/cn/com/ctop/crawler/modules/taobao/service/impl/TaobaoJhsbybtServiceImpl.java

@@ -7,7 +7,6 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.http.cookie.Cookie;
 import org.springframework.stereotype.Service;
-
 import java.io.FileNotFoundException;
 import java.io.PrintStream;
 import java.math.BigDecimal;
@@ -58,11 +57,12 @@ public class TaobaoJhsbybtServiceImpl implements TaobaoJhsbybtService {
 
     public static void main(String[] args){
         try {
-            PrintStream out = new PrintStream("D:/bybt0814all.csv");
+            PrintStream out = new PrintStream("D:/bybt0831.csv");
             System.setOut(out);
         } catch (FileNotFoundException e) {
             e.printStackTrace();
         }
+        System.out.println("商品id,商品名,销量,商品logo,设计图,商品图,商品链接,正常价,,,补贴价,补贴券");
         TaobaoJhsbybtServiceImpl tb = new TaobaoJhsbybtServiceImpl();
         int i = 1;
         Boolean hasMore = true;
@@ -71,6 +71,7 @@ public class TaobaoJhsbybtServiceImpl implements TaobaoJhsbybtService {
                 String result = tb.getGoodsList(i);
                 result = result.replace("mtopjsonp2(","");
                 result = result.substring(0,result.length()-1);
+//                System.out.println(result);
                 ObjectMapper mapper = new ObjectMapper();
                 JsonNode rootNode = mapper.readTree(result);
                 Boolean success = rootNode.get("data").get("resultValue").get("8613423").get("success").asBoolean();

+ 158 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/taobao/service/impl/TaobaoTttmServiceImpl.java

@@ -0,0 +1,158 @@
+package cn.com.ctop.crawler.modules.taobao.service.impl;
+
+import cn.com.ctop.common.module.utils.HttpUtils;
+import cn.com.ctop.crawler.modules.taobao.service.TaobaoJhsbybtService;
+import cn.com.ctop.crawler.modules.taobao.service.TaobaoTttmService;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.http.cookie.Cookie;
+import org.springframework.stereotype.Service;
+
+import java.io.FileNotFoundException;
+import java.io.PrintStream;
+import java.util.*;
+
+@Service
+public class TaobaoTttmServiceImpl implements TaobaoTttmService {
+
+    public String getGoodsList(int page){
+        String url = "https://h5api.m.taobao.com/h5/mtop.panama.fiona.commonbizservice/1.1/";
+        Map<String,Object> paramMap = new HashMap<String,Object>();
+        paramMap.put("jsv","2.6.0");
+        paramMap.put("appKey","12574478");
+        paramMap.put("t",new Date().getTime());
+        paramMap.put("sign","56b679fe3a6de0cd4ffa8ccb8680ff83");
+        paramMap.put("timeout",3000);
+        paramMap.put("api","mtop.panama.fiona.commonbizservice");
+        paramMap.put("params","[object Object]");
+        paramMap.put("dataType","jsonp");
+        paramMap.put("v","1.0");
+        paramMap.put("preventFallback",true);
+        paramMap.put("type","jsonp");
+        paramMap.put("callback","mtopjsonp2");
+        paramMap.put("data","{\"bizCode\":\"home_feeds\",\"optStr\":\"preview=false;outerBackupKey=prod-home_feeds-currentPage_"+page+"\",\"currentPage\":"+page+",\"outerBackupKey\":\"prod-home_feeds-currentPage_"+page+"\"}");
+        Map<String,String> headerMap = new HashMap<String,String>();
+        headerMap.put("Accept-Encoding","gzip, deflate, br");
+        headerMap.put("Connection","keep-alive");
+        headerMap.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.108 Safari/537.36");
+        headerMap.put("Accept","*/*");
+        headerMap.put("Content-Type","application/xml");
+        headerMap.put("Referer","https://market.m.taobao.com/app/tmall-def/jhsbybt/web/index.html?disableNav=YES&itemIds=617136337387&spm=608.14027681/8.bubbles.d0");
+        String result = HttpUtils.httpGet(url,paramMap,headerMap);
+        List<Cookie> cookieList = HttpUtils.COOKIESTORE.getCookies();
+        String token = "";
+        String cookie = "";
+        for (Cookie ck : cookieList){
+            if(ck.getName().equals("_m_h5_tk")){
+                token = ck.getValue().split("_")[0];
+            }
+            cookie = cookie + ck.getName()+"="+ck.getValue()+";";
+        }
+        String sign = DigestUtils.md5Hex(token+"&"+paramMap.get("t")+"&"+paramMap.get("appKey")+"&"+paramMap.get("data"));
+        paramMap.put("sign",sign);
+        headerMap.put("Cookie",cookie);
+        result = HttpUtils.httpGet(url,paramMap,headerMap);
+        return result;
+    }
+
+    public static void main(String[] args){
+        try {
+            PrintStream out = new PrintStream("D:/tttm0831.csv");
+            System.setOut(out);
+        } catch (FileNotFoundException e) {
+            e.printStackTrace();
+        }
+        System.out.println("商品id,商品名,销量,商品图,商品链接,优惠价,原价");
+        TaobaoTttmServiceImpl tb = new TaobaoTttmServiceImpl();
+        int i = 2;
+        Boolean hasMore = true;
+        while (hasMore){
+            try {
+                String result = tb.getGoodsList(i);
+                result = result.replace("mtopjsonp2(","");
+                result = result.substring(0,result.length()-1);
+//                System.out.println(result);
+                ObjectMapper mapper = new ObjectMapper();
+                JsonNode rootNode = mapper.readTree(result);
+                Boolean success = rootNode.get("data").get("data").get("homeFeeds").isNull();
+                int size = rootNode.get("data").get("data").get("homeFeeds").size();
+                if(size == 0){
+                    hasMore = false;
+                }
+//                while (!success){
+//                    Thread.sleep(3000L);
+//                    result = tb.getGoodsList(i);
+//                    result = result.replace("mtopjsonp2(","");
+//                    result = result.substring(0,result.length()-1);
+//                    mapper = new ObjectMapper();
+//                    rootNode = mapper.readTree(result);
+//                    success = rootNode.get("data").get("resultValue").get("8613423").get("success").asBoolean();
+//                }
+//                hasMore = rootNode.get("data").get("resultValue").get("8613423").get("data").get("hasMore").asBoolean();
+                Iterator<JsonNode> items = rootNode.get("data").get("data").get("homeFeeds").iterator();
+
+                while (items.hasNext()){
+
+                    JsonNode itemNode = items.next();
+                    String itemType = itemNode.get("itemType").asText();
+                    if (itemType.equals("rankList")){
+                        Iterator<JsonNode> subitems = itemNode.get("items").iterator();
+                        while (subitems.hasNext()){
+                            JsonNode subNode = subitems.next();
+                            String cjmsItemName = subNode.get("shortName").asText();
+                            Integer itemSoldInfo = 0;
+                            String picUrl = subNode.get("preview").asText();
+//                    String cjmsTaobaoEntrance = itemNode.get("cjmsTaobaoEntrance").asText();
+//                    String couponAmount = itemNode.get("couponAmount").asText();
+                            String orgPrice = subNode.get("originPrice").asText();
+//                    String brandLogo = itemNode.get("brandLogo").asText();
+//                    String couponPrice = itemNode.get("couponPrice").asText();
+                            String actPrice = subNode.get("actPrice").asText();
+//                    String cutPrice = itemNode.get("cutPrice").asText();
+                            String itemUrl = subNode.get("itemHref").asText();
+                            String itemId = subNode.get("itemId").asText();
+                            System.out.println(itemId+","+cjmsItemName+","+
+                                    itemSoldInfo+","+
+//                            brandLogo+","+
+                                    picUrl+","+
+//                            cjmsTaobaoEntrance+","+
+                                    itemUrl+","+
+                                    actPrice+","+
+//                            cutPrice+","+
+                                    orgPrice+",");
+                        }
+                    }else if(itemType.equals("item")){
+                        String cjmsItemName = itemNode.get("shortName").asText();
+                        Integer itemSoldInfo = itemNode.get("soldCount").asInt();
+                        String picUrl = itemNode.get("preview").asText();
+//                    String cjmsTaobaoEntrance = itemNode.get("cjmsTaobaoEntrance").asText();
+//                    String couponAmount = itemNode.get("couponAmount").asText();
+                        String orgPrice = itemNode.get("originPrice").asText();
+//                    String brandLogo = itemNode.get("brandLogo").asText();
+//                    String couponPrice = itemNode.get("couponPrice").asText();
+                        String actPrice = itemNode.get("actPrice").asText();
+//                    String cutPrice = itemNode.get("cutPrice").asText();
+                        String itemUrl = itemNode.get("itemHref").asText();
+                        String itemId = itemNode.get("itemId").asText();
+                        System.out.println(itemId+","+cjmsItemName+","+
+                                itemSoldInfo+","+
+//                            brandLogo+","+
+                                picUrl+","+
+//                            cjmsTaobaoEntrance+","+
+                                itemUrl+","+
+                                actPrice+","+
+//                            cutPrice+","+
+                                orgPrice+",");
+                    }
+
+                }
+                i++;
+                Thread.sleep(3000L);
+            }catch (Exception e){
+                e.printStackTrace();
+            }
+        }
+
+    }
+}