|
@@ -0,0 +1,158 @@
|
|
|
+package cn.com.ctop.crawler.modules.taobao.service.impl;
|
|
|
+
|
|
|
+import cn.com.ctop.common.module.utils.HttpUtils;
|
|
|
+import cn.com.ctop.crawler.modules.taobao.service.TaobaoJhsbybtService;
|
|
|
+import cn.com.ctop.crawler.modules.taobao.service.TaobaoTttmService;
|
|
|
+import com.fasterxml.jackson.databind.JsonNode;
|
|
|
+import com.fasterxml.jackson.databind.ObjectMapper;
|
|
|
+import org.apache.commons.codec.digest.DigestUtils;
|
|
|
+import org.apache.http.cookie.Cookie;
|
|
|
+import org.springframework.stereotype.Service;
|
|
|
+
|
|
|
+import java.io.FileNotFoundException;
|
|
|
+import java.io.PrintStream;
|
|
|
+import java.util.*;
|
|
|
+
|
|
|
+@Service
|
|
|
+public class TaobaoTttmServiceImpl implements TaobaoTttmService {
|
|
|
+
|
|
|
+ public String getGoodsList(int page){
|
|
|
+ String url = "https://h5api.m.taobao.com/h5/mtop.panama.fiona.commonbizservice/1.1/";
|
|
|
+ Map<String,Object> paramMap = new HashMap<String,Object>();
|
|
|
+ paramMap.put("jsv","2.6.0");
|
|
|
+ paramMap.put("appKey","12574478");
|
|
|
+ paramMap.put("t",new Date().getTime());
|
|
|
+ paramMap.put("sign","56b679fe3a6de0cd4ffa8ccb8680ff83");
|
|
|
+ paramMap.put("timeout",3000);
|
|
|
+ paramMap.put("api","mtop.panama.fiona.commonbizservice");
|
|
|
+ paramMap.put("params","[object Object]");
|
|
|
+ paramMap.put("dataType","jsonp");
|
|
|
+ paramMap.put("v","1.0");
|
|
|
+ paramMap.put("preventFallback",true);
|
|
|
+ paramMap.put("type","jsonp");
|
|
|
+ paramMap.put("callback","mtopjsonp2");
|
|
|
+ paramMap.put("data","{\"bizCode\":\"home_feeds\",\"optStr\":\"preview=false;outerBackupKey=prod-home_feeds-currentPage_"+page+"\",\"currentPage\":"+page+",\"outerBackupKey\":\"prod-home_feeds-currentPage_"+page+"\"}");
|
|
|
+ Map<String,String> headerMap = new HashMap<String,String>();
|
|
|
+ headerMap.put("Accept-Encoding","gzip, deflate, br");
|
|
|
+ headerMap.put("Connection","keep-alive");
|
|
|
+ headerMap.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.108 Safari/537.36");
|
|
|
+ headerMap.put("Accept","*/*");
|
|
|
+ headerMap.put("Content-Type","application/xml");
|
|
|
+ headerMap.put("Referer","https://market.m.taobao.com/app/tmall-def/jhsbybt/web/index.html?disableNav=YES&itemIds=617136337387&spm=608.14027681/8.bubbles.d0");
|
|
|
+ String result = HttpUtils.httpGet(url,paramMap,headerMap);
|
|
|
+ List<Cookie> cookieList = HttpUtils.COOKIESTORE.getCookies();
|
|
|
+ String token = "";
|
|
|
+ String cookie = "";
|
|
|
+ for (Cookie ck : cookieList){
|
|
|
+ if(ck.getName().equals("_m_h5_tk")){
|
|
|
+ token = ck.getValue().split("_")[0];
|
|
|
+ }
|
|
|
+ cookie = cookie + ck.getName()+"="+ck.getValue()+";";
|
|
|
+ }
|
|
|
+ String sign = DigestUtils.md5Hex(token+"&"+paramMap.get("t")+"&"+paramMap.get("appKey")+"&"+paramMap.get("data"));
|
|
|
+ paramMap.put("sign",sign);
|
|
|
+ headerMap.put("Cookie",cookie);
|
|
|
+ result = HttpUtils.httpGet(url,paramMap,headerMap);
|
|
|
+ return result;
|
|
|
+ }
|
|
|
+
|
|
|
+ public static void main(String[] args){
|
|
|
+ try {
|
|
|
+ PrintStream out = new PrintStream("D:/tttm0831.csv");
|
|
|
+ System.setOut(out);
|
|
|
+ } catch (FileNotFoundException e) {
|
|
|
+ e.printStackTrace();
|
|
|
+ }
|
|
|
+ System.out.println("商品id,商品名,销量,商品图,商品链接,优惠价,原价");
|
|
|
+ TaobaoTttmServiceImpl tb = new TaobaoTttmServiceImpl();
|
|
|
+ int i = 2;
|
|
|
+ Boolean hasMore = true;
|
|
|
+ while (hasMore){
|
|
|
+ try {
|
|
|
+ String result = tb.getGoodsList(i);
|
|
|
+ result = result.replace("mtopjsonp2(","");
|
|
|
+ result = result.substring(0,result.length()-1);
|
|
|
+// System.out.println(result);
|
|
|
+ ObjectMapper mapper = new ObjectMapper();
|
|
|
+ JsonNode rootNode = mapper.readTree(result);
|
|
|
+ Boolean success = rootNode.get("data").get("data").get("homeFeeds").isNull();
|
|
|
+ int size = rootNode.get("data").get("data").get("homeFeeds").size();
|
|
|
+ if(size == 0){
|
|
|
+ hasMore = false;
|
|
|
+ }
|
|
|
+// while (!success){
|
|
|
+// Thread.sleep(3000L);
|
|
|
+// result = tb.getGoodsList(i);
|
|
|
+// result = result.replace("mtopjsonp2(","");
|
|
|
+// result = result.substring(0,result.length()-1);
|
|
|
+// mapper = new ObjectMapper();
|
|
|
+// rootNode = mapper.readTree(result);
|
|
|
+// success = rootNode.get("data").get("resultValue").get("8613423").get("success").asBoolean();
|
|
|
+// }
|
|
|
+// hasMore = rootNode.get("data").get("resultValue").get("8613423").get("data").get("hasMore").asBoolean();
|
|
|
+ Iterator<JsonNode> items = rootNode.get("data").get("data").get("homeFeeds").iterator();
|
|
|
+
|
|
|
+ while (items.hasNext()){
|
|
|
+
|
|
|
+ JsonNode itemNode = items.next();
|
|
|
+ String itemType = itemNode.get("itemType").asText();
|
|
|
+ if (itemType.equals("rankList")){
|
|
|
+ Iterator<JsonNode> subitems = itemNode.get("items").iterator();
|
|
|
+ while (subitems.hasNext()){
|
|
|
+ JsonNode subNode = subitems.next();
|
|
|
+ String cjmsItemName = subNode.get("shortName").asText();
|
|
|
+ Integer itemSoldInfo = 0;
|
|
|
+ String picUrl = subNode.get("preview").asText();
|
|
|
+// String cjmsTaobaoEntrance = itemNode.get("cjmsTaobaoEntrance").asText();
|
|
|
+// String couponAmount = itemNode.get("couponAmount").asText();
|
|
|
+ String orgPrice = subNode.get("originPrice").asText();
|
|
|
+// String brandLogo = itemNode.get("brandLogo").asText();
|
|
|
+// String couponPrice = itemNode.get("couponPrice").asText();
|
|
|
+ String actPrice = subNode.get("actPrice").asText();
|
|
|
+// String cutPrice = itemNode.get("cutPrice").asText();
|
|
|
+ String itemUrl = subNode.get("itemHref").asText();
|
|
|
+ String itemId = subNode.get("itemId").asText();
|
|
|
+ System.out.println(itemId+","+cjmsItemName+","+
|
|
|
+ itemSoldInfo+","+
|
|
|
+// brandLogo+","+
|
|
|
+ picUrl+","+
|
|
|
+// cjmsTaobaoEntrance+","+
|
|
|
+ itemUrl+","+
|
|
|
+ actPrice+","+
|
|
|
+// cutPrice+","+
|
|
|
+ orgPrice+",");
|
|
|
+ }
|
|
|
+ }else if(itemType.equals("item")){
|
|
|
+ String cjmsItemName = itemNode.get("shortName").asText();
|
|
|
+ Integer itemSoldInfo = itemNode.get("soldCount").asInt();
|
|
|
+ String picUrl = itemNode.get("preview").asText();
|
|
|
+// String cjmsTaobaoEntrance = itemNode.get("cjmsTaobaoEntrance").asText();
|
|
|
+// String couponAmount = itemNode.get("couponAmount").asText();
|
|
|
+ String orgPrice = itemNode.get("originPrice").asText();
|
|
|
+// String brandLogo = itemNode.get("brandLogo").asText();
|
|
|
+// String couponPrice = itemNode.get("couponPrice").asText();
|
|
|
+ String actPrice = itemNode.get("actPrice").asText();
|
|
|
+// String cutPrice = itemNode.get("cutPrice").asText();
|
|
|
+ String itemUrl = itemNode.get("itemHref").asText();
|
|
|
+ String itemId = itemNode.get("itemId").asText();
|
|
|
+ System.out.println(itemId+","+cjmsItemName+","+
|
|
|
+ itemSoldInfo+","+
|
|
|
+// brandLogo+","+
|
|
|
+ picUrl+","+
|
|
|
+// cjmsTaobaoEntrance+","+
|
|
|
+ itemUrl+","+
|
|
|
+ actPrice+","+
|
|
|
+// cutPrice+","+
|
|
|
+ orgPrice+",");
|
|
|
+ }
|
|
|
+
|
|
|
+ }
|
|
|
+ i++;
|
|
|
+ Thread.sleep(3000L);
|
|
|
+ }catch (Exception e){
|
|
|
+ e.printStackTrace();
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ }
|
|
|
+}
|