//package cn.com.ctop.common.utils; // //import us.codecraft.webmagic.Page; //import us.codecraft.webmagic.ResultItems; //import us.codecraft.webmagic.Site; //import us.codecraft.webmagic.Spider; //import us.codecraft.webmagic.downloader.PhantomJSDownloader; //import us.codecraft.webmagic.pipeline.CollectorPipeline; //import us.codecraft.webmagic.pipeline.ResultItemsCollectorPipeline; //import us.codecraft.webmagic.processor.PageProcessor; // //import java.util.List; // //public class WebMasterUtils implements PageProcessor { // private Site site = Site.me() // .setDomain("live.kuaishou.com") // .setCharset("UTF-8") // .addHeader("Referer","https://live.kuaishou.com") // .setRetryTimes(1).setSleepTime(1000); // // public static void main(String[] args){ // PhantomJSDownloader phantomDownloader = new PhantomJSDownloader("D:/phantomjs-2.1.1-windows/bin/phantomjs.exe","D:/phantomjs-2.1.1-windows/bin/crawler.js").setRetryNum(3); // // CollectorPipeline collectorPipeline = new ResultItemsCollectorPipeline(); // // Spider.create(new WebMasterUtils()) // .addUrl("https://live.kuaishou.com/profile/mumu6666668") // .setDownloader(phantomDownloader) // .addPipeline(collectorPipeline) // .thread((Runtime.getRuntime().availableProcessors() - 1) << 1) // .run(); // // List resultItemsList = collectorPipeline.getCollected(); // System.out.println(resultItemsList.get(0).get("html").toString()); // // Spider.create(new WebMasterUtils()) // .addUrl("https://live.kuaishou.com/profile/mumu6666668") // .setDownloader(phantomDownloader) // .addPipeline(collectorPipeline) // .thread((Runtime.getRuntime().availableProcessors() - 1) << 1) // .run(); // // resultItemsList = collectorPipeline.getCollected(); // ResultItems item = resultItemsList.get(0); // // System.out.println(resultItemsList.get(0).get("html").toString()); //// System.out.println(resultItemsList.get(1).get("html").toString()); // // } // // @Override // public void process(Page page) { // if(page.getRawText() != null) { // page.putField("html", page.getRawText()); // } // } // // @Override // public Site getSite() { // return site; // } //}