1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162 |
- //package cn.com.ctop.common.utils;
- //
- //import us.codecraft.webmagic.Page;
- //import us.codecraft.webmagic.ResultItems;
- //import us.codecraft.webmagic.Site;
- //import us.codecraft.webmagic.Spider;
- //import us.codecraft.webmagic.downloader.PhantomJSDownloader;
- //import us.codecraft.webmagic.pipeline.CollectorPipeline;
- //import us.codecraft.webmagic.pipeline.ResultItemsCollectorPipeline;
- //import us.codecraft.webmagic.processor.PageProcessor;
- //
- //import java.util.List;
- //
- //public class WebMasterUtils implements PageProcessor {
- // private Site site = Site.me()
- // .setDomain("live.kuaishou.com")
- // .setCharset("UTF-8")
- // .addHeader("Referer","https://live.kuaishou.com")
- // .setRetryTimes(1).setSleepTime(1000);
- //
- // public static void main(String[] args){
- // PhantomJSDownloader phantomDownloader = new PhantomJSDownloader("D:/phantomjs-2.1.1-windows/bin/phantomjs.exe","D:/phantomjs-2.1.1-windows/bin/crawler.js").setRetryNum(3);
- //
- // CollectorPipeline<ResultItems> collectorPipeline = new ResultItemsCollectorPipeline();
- //
- // Spider.create(new WebMasterUtils())
- // .addUrl("https://live.kuaishou.com/profile/mumu6666668")
- // .setDownloader(phantomDownloader)
- // .addPipeline(collectorPipeline)
- // .thread((Runtime.getRuntime().availableProcessors() - 1) << 1)
- // .run();
- //
- // List<ResultItems> resultItemsList = collectorPipeline.getCollected();
- // System.out.println(resultItemsList.get(0).get("html").toString());
- //
- // Spider.create(new WebMasterUtils())
- // .addUrl("https://live.kuaishou.com/profile/mumu6666668")
- // .setDownloader(phantomDownloader)
- // .addPipeline(collectorPipeline)
- // .thread((Runtime.getRuntime().availableProcessors() - 1) << 1)
- // .run();
- //
- // resultItemsList = collectorPipeline.getCollected();
- // ResultItems item = resultItemsList.get(0);
- //
- // System.out.println(resultItemsList.get(0).get("html").toString());
- //// System.out.println(resultItemsList.get(1).get("html").toString());
- //
- // }
- //
- // @Override
- // public void process(Page page) {
- // if(page.getRawText() != null) {
- // page.putField("html", page.getRawText());
- // }
- // }
- //
- // @Override
- // public Site getSite() {
- // return site;
- // }
- //}
|