WebMasterUtils.java 2.4 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162
  1. //package cn.com.ctop.common.utils;
  2. //
  3. //import us.codecraft.webmagic.Page;
  4. //import us.codecraft.webmagic.ResultItems;
  5. //import us.codecraft.webmagic.Site;
  6. //import us.codecraft.webmagic.Spider;
  7. //import us.codecraft.webmagic.downloader.PhantomJSDownloader;
  8. //import us.codecraft.webmagic.pipeline.CollectorPipeline;
  9. //import us.codecraft.webmagic.pipeline.ResultItemsCollectorPipeline;
  10. //import us.codecraft.webmagic.processor.PageProcessor;
  11. //
  12. //import java.util.List;
  13. //
  14. //public class WebMasterUtils implements PageProcessor {
  15. // private Site site = Site.me()
  16. // .setDomain("live.kuaishou.com")
  17. // .setCharset("UTF-8")
  18. // .addHeader("Referer","https://live.kuaishou.com")
  19. // .setRetryTimes(1).setSleepTime(1000);
  20. //
  21. // public static void main(String[] args){
  22. // PhantomJSDownloader phantomDownloader = new PhantomJSDownloader("D:/phantomjs-2.1.1-windows/bin/phantomjs.exe","D:/phantomjs-2.1.1-windows/bin/crawler.js").setRetryNum(3);
  23. //
  24. // CollectorPipeline<ResultItems> collectorPipeline = new ResultItemsCollectorPipeline();
  25. //
  26. // Spider.create(new WebMasterUtils())
  27. // .addUrl("https://live.kuaishou.com/profile/mumu6666668")
  28. // .setDownloader(phantomDownloader)
  29. // .addPipeline(collectorPipeline)
  30. // .thread((Runtime.getRuntime().availableProcessors() - 1) << 1)
  31. // .run();
  32. //
  33. // List<ResultItems> resultItemsList = collectorPipeline.getCollected();
  34. // System.out.println(resultItemsList.get(0).get("html").toString());
  35. //
  36. // Spider.create(new WebMasterUtils())
  37. // .addUrl("https://live.kuaishou.com/profile/mumu6666668")
  38. // .setDownloader(phantomDownloader)
  39. // .addPipeline(collectorPipeline)
  40. // .thread((Runtime.getRuntime().availableProcessors() - 1) << 1)
  41. // .run();
  42. //
  43. // resultItemsList = collectorPipeline.getCollected();
  44. // ResultItems item = resultItemsList.get(0);
  45. //
  46. // System.out.println(resultItemsList.get(0).get("html").toString());
  47. //// System.out.println(resultItemsList.get(1).get("html").toString());
  48. //
  49. // }
  50. //
  51. // @Override
  52. // public void process(Page page) {
  53. // if(page.getRawText() != null) {
  54. // page.putField("html", page.getRawText());
  55. // }
  56. // }
  57. //
  58. // @Override
  59. // public Site getSite() {
  60. // return site;
  61. // }
  62. //}