|
@@ -1,7 +1,6 @@
|
|
|
package cn.com.ctop.crawler.modules.taobao.service.impl;
|
|
|
|
|
|
import cn.com.ctop.common.module.utils.HttpUtils;
|
|
|
-import cn.com.ctop.crawler.modules.taobao.service.TaobaoJhsbybtService;
|
|
|
import cn.com.ctop.crawler.modules.taobao.service.TaobaoTttmService;
|
|
|
import com.fasterxml.jackson.databind.JsonNode;
|
|
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
@@ -16,68 +15,66 @@ import java.util.*;
|
|
|
@Service
|
|
|
public class TaobaoTttmServiceImpl implements TaobaoTttmService {
|
|
|
|
|
|
- public String getGoodsList(int page){
|
|
|
+ public String getGoodsList(int page) {
|
|
|
String url = "https://h5api.m.taobao.com/h5/mtop.panama.fiona.commonbizservice/1.1/";
|
|
|
- Map<String,Object> paramMap = new HashMap<String,Object>();
|
|
|
- paramMap.put("jsv","2.6.0");
|
|
|
- paramMap.put("appKey","12574478");
|
|
|
- paramMap.put("t",new Date().getTime());
|
|
|
- paramMap.put("sign","56b679fe3a6de0cd4ffa8ccb8680ff83");
|
|
|
- paramMap.put("timeout",3000);
|
|
|
- paramMap.put("api","mtop.panama.fiona.commonbizservice");
|
|
|
- paramMap.put("params","[object Object]");
|
|
|
- paramMap.put("dataType","jsonp");
|
|
|
- paramMap.put("v","1.0");
|
|
|
- paramMap.put("preventFallback",true);
|
|
|
- paramMap.put("type","jsonp");
|
|
|
- paramMap.put("callback","mtopjsonp2");
|
|
|
- paramMap.put("data","{\"bizCode\":\"home_feeds\",\"optStr\":\"preview=false;outerBackupKey=prod-home_feeds-currentPage_"+page+"\",\"currentPage\":"+page+",\"outerBackupKey\":\"prod-home_feeds-currentPage_"+page+"\"}");
|
|
|
- Map<String,String> headerMap = new HashMap<String,String>();
|
|
|
- headerMap.put("Accept-Encoding","gzip, deflate, br");
|
|
|
- headerMap.put("Connection","keep-alive");
|
|
|
- headerMap.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.108 Safari/537.36");
|
|
|
- headerMap.put("Accept","*/*");
|
|
|
- headerMap.put("Content-Type","application/xml");
|
|
|
- headerMap.put("Referer","https://market.m.taobao.com/app/tmall-def/jhsbybt/web/index.html?disableNav=YES&itemIds=617136337387&spm=608.14027681/8.bubbles.d0");
|
|
|
- String result = HttpUtils.httpGet(url,paramMap,headerMap);
|
|
|
+ Map<String, Object> paramMap = new HashMap<String, Object>();
|
|
|
+ paramMap.put("jsv", "2.6.0");
|
|
|
+ paramMap.put("appKey", "12574478");
|
|
|
+ paramMap.put("t", new Date().getTime());
|
|
|
+ paramMap.put("sign", "56b679fe3a6de0cd4ffa8ccb8680ff83");
|
|
|
+ paramMap.put("timeout", 3000);
|
|
|
+ paramMap.put("api", "mtop.panama.fiona.commonbizservice");
|
|
|
+ paramMap.put("params", "[object Object]");
|
|
|
+ paramMap.put("dataType", "jsonp");
|
|
|
+ paramMap.put("v", "1.0");
|
|
|
+ paramMap.put("preventFallback", true);
|
|
|
+ paramMap.put("type", "jsonp");
|
|
|
+ paramMap.put("callback", "mtopjsonp2");
|
|
|
+ paramMap.put("data", "{\"bizCode\":\"home_feeds\",\"optStr\":\"preview=false;outerBackupKey=prod-home_feeds-currentPage_" + page + "\",\"currentPage\":" + page + ",\"outerBackupKey\":\"prod-home_feeds-currentPage_" + page + "\"}");
|
|
|
+ Map<String, String> headerMap = new HashMap<String, String>();
|
|
|
+ headerMap.put("Accept-Encoding", "gzip, deflate, br");
|
|
|
+ headerMap.put("Connection", "keep-alive");
|
|
|
+ headerMap.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.108 Safari/537.36");
|
|
|
+ headerMap.put("Accept", "*/*");
|
|
|
+ headerMap.put("Content-Type", "application/xml");
|
|
|
+ headerMap.put("Referer", "https://market.m.taobao.com/app/tmall-def/jhsbybt/web/index.html?disableNav=YES&itemIds=617136337387&spm=608.14027681/8.bubbles.d0");
|
|
|
+ String result = HttpUtils.httpGet(url, paramMap, headerMap);
|
|
|
List<Cookie> cookieList = HttpUtils.COOKIESTORE.getCookies();
|
|
|
String token = "";
|
|
|
String cookie = "";
|
|
|
- for (Cookie ck : cookieList){
|
|
|
- if(ck.getName().equals("_m_h5_tk")){
|
|
|
+ for (Cookie ck : cookieList) {
|
|
|
+ if (ck.getName().equals("_m_h5_tk")) {
|
|
|
token = ck.getValue().split("_")[0];
|
|
|
}
|
|
|
- cookie = cookie + ck.getName()+"="+ck.getValue()+";";
|
|
|
+ cookie = cookie + ck.getName() + "=" + ck.getValue() + ";";
|
|
|
}
|
|
|
- String sign = DigestUtils.md5Hex(token+"&"+paramMap.get("t")+"&"+paramMap.get("appKey")+"&"+paramMap.get("data"));
|
|
|
- paramMap.put("sign",sign);
|
|
|
- headerMap.put("Cookie",cookie);
|
|
|
- result = HttpUtils.httpGet(url,paramMap,headerMap);
|
|
|
+ String sign = DigestUtils.md5Hex(token + "&" + paramMap.get("t") + "&" + paramMap.get("appKey") + "&" + paramMap.get("data"));
|
|
|
+ paramMap.put("sign", sign);
|
|
|
+ headerMap.put("Cookie", cookie);
|
|
|
+ result = HttpUtils.httpGet(url, paramMap, headerMap);
|
|
|
return result;
|
|
|
}
|
|
|
|
|
|
- public static void main(String[] args){
|
|
|
+ public static void main(String[] args) {
|
|
|
try {
|
|
|
PrintStream out = new PrintStream("D:/tttm0831.csv");
|
|
|
System.setOut(out);
|
|
|
} catch (FileNotFoundException e) {
|
|
|
e.printStackTrace();
|
|
|
}
|
|
|
- System.out.println("商品id,商品名,销量,商品图,商品链接,优惠价,原价");
|
|
|
TaobaoTttmServiceImpl tb = new TaobaoTttmServiceImpl();
|
|
|
int i = 2;
|
|
|
Boolean hasMore = true;
|
|
|
- while (hasMore){
|
|
|
+ while (hasMore) {
|
|
|
try {
|
|
|
String result = tb.getGoodsList(i);
|
|
|
- result = result.replace("mtopjsonp2(","");
|
|
|
- result = result.substring(0,result.length()-1);
|
|
|
-// System.out.println(result);
|
|
|
+ result = result.replace("mtopjsonp2(", "");
|
|
|
+ result = result.substring(0, result.length() - 1);
|
|
|
ObjectMapper mapper = new ObjectMapper();
|
|
|
JsonNode rootNode = mapper.readTree(result);
|
|
|
Boolean success = rootNode.get("data").get("data").get("homeFeeds").isNull();
|
|
|
int size = rootNode.get("data").get("data").get("homeFeeds").size();
|
|
|
- if(size == 0){
|
|
|
+ if (size == 0) {
|
|
|
hasMore = false;
|
|
|
}
|
|
|
// while (!success){
|
|
@@ -92,13 +89,13 @@ public class TaobaoTttmServiceImpl implements TaobaoTttmService {
|
|
|
// hasMore = rootNode.get("data").get("resultValue").get("8613423").get("data").get("hasMore").asBoolean();
|
|
|
Iterator<JsonNode> items = rootNode.get("data").get("data").get("homeFeeds").iterator();
|
|
|
|
|
|
- while (items.hasNext()){
|
|
|
+ while (items.hasNext()) {
|
|
|
|
|
|
JsonNode itemNode = items.next();
|
|
|
String itemType = itemNode.get("itemType").asText();
|
|
|
- if (itemType.equals("rankList")){
|
|
|
+ if (itemType.equals("rankList")) {
|
|
|
Iterator<JsonNode> subitems = itemNode.get("items").iterator();
|
|
|
- while (subitems.hasNext()){
|
|
|
+ while (subitems.hasNext()) {
|
|
|
JsonNode subNode = subitems.next();
|
|
|
String cjmsItemName = subNode.get("shortName").asText();
|
|
|
Integer itemSoldInfo = 0;
|
|
@@ -112,17 +109,8 @@ public class TaobaoTttmServiceImpl implements TaobaoTttmService {
|
|
|
// String cutPrice = itemNode.get("cutPrice").asText();
|
|
|
String itemUrl = subNode.get("itemHref").asText();
|
|
|
String itemId = subNode.get("itemId").asText();
|
|
|
- System.out.println(itemId+","+cjmsItemName+","+
|
|
|
- itemSoldInfo+","+
|
|
|
-// brandLogo+","+
|
|
|
- picUrl+","+
|
|
|
-// cjmsTaobaoEntrance+","+
|
|
|
- itemUrl+","+
|
|
|
- actPrice+","+
|
|
|
-// cutPrice+","+
|
|
|
- orgPrice+",");
|
|
|
}
|
|
|
- }else if(itemType.equals("item")){
|
|
|
+ } else if (itemType.equals("item")) {
|
|
|
String cjmsItemName = itemNode.get("shortName").asText();
|
|
|
Integer itemSoldInfo = itemNode.get("soldCount").asInt();
|
|
|
String picUrl = itemNode.get("preview").asText();
|
|
@@ -135,21 +123,13 @@ public class TaobaoTttmServiceImpl implements TaobaoTttmService {
|
|
|
// String cutPrice = itemNode.get("cutPrice").asText();
|
|
|
String itemUrl = itemNode.get("itemHref").asText();
|
|
|
String itemId = itemNode.get("itemId").asText();
|
|
|
- System.out.println(itemId+","+cjmsItemName+","+
|
|
|
- itemSoldInfo+","+
|
|
|
-// brandLogo+","+
|
|
|
- picUrl+","+
|
|
|
-// cjmsTaobaoEntrance+","+
|
|
|
- itemUrl+","+
|
|
|
- actPrice+","+
|
|
|
-// cutPrice+","+
|
|
|
- orgPrice+",");
|
|
|
+
|
|
|
}
|
|
|
|
|
|
}
|
|
|
i++;
|
|
|
Thread.sleep(3000L);
|
|
|
- }catch (Exception e){
|
|
|
+ } catch (Exception e) {
|
|
|
e.printStackTrace();
|
|
|
}
|
|
|
}
|