Quellcode durchsuchen

爬取封面添加代理

syh vor 5 Jahren
Ursprung
Commit
0f885ba3cf

+ 0 - 12
jeecg-boot-base-common/src/main/java/org/jeecg/common/util/LoadFileUtil.java

@@ -8,18 +8,6 @@ import java.net.HttpURLConnection;
 import java.net.URL;
 
 public class LoadFileUtil {
-
-    public static void main(String[] args) {
-        try {
-            //  downLoadFromUrl("", "");
-
-            delFile("D:\\tets1\\app\\TEST视频-1565073368755.mp4");
-
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-    }
-
     /**
      * 上传文件
      *

+ 0 - 9
module-common/src/main/java/cn/com/ctop/common/module/utils/CsvUtils.java

@@ -47,13 +47,4 @@ public class CsvUtils {
         }
         wr.close();
     }
-
-    public static void main(String[] args) throws IOException {
-        String[] header = new String[]{"日期", "plan", "广告组", "花费", "曝光数", "点击数", "行为数", "点击率", "行为率", "平均千次曝光花费", "平均点击单价", "平均行为单价", "提交按钮点击", "表单提交率", "表单提交单价"};
-        String inPath = "D:\\工作文件\\360借条\\effect_unit_2019-08-02-00_2019-08-02-00.csv";
-        String outPath = "D:\\工作文件\\360借条\\effect_unit_2019-08-02-00_2019-08-02-00_bak.csv";
-        List<String[]> list = CsvUtils.readCsv(inPath, "utf8");
-        CsvUtils.writeCsv(outPath, list);
-
-    }
 }

+ 0 - 10
module-common/src/main/java/cn/com/ctop/common/module/utils/HttpUtils.java

@@ -543,16 +543,6 @@ public class HttpUtils {
         return result;
     }
 
-    public static void main(String[] args) {
-        Map<String, Object> params = new HashMap<>();
-        params.put("add", 1);
-        params.forEach((k, v) -> {
-
-        });
-
-    }
-
-
     public static String httpGetRequest(String url, Map<String, String> headers, TreeMap<String, Object> params) {
         HttpClient httpClient = createSslClientDefault();
         String strReturn = "";

+ 0 - 48
module-common/src/main/java/cn/com/ctop/common/module/utils/ImageUtils.java

@@ -140,54 +140,6 @@ public class ImageUtils {
     public int getContentLength(String content, Graphics2D graphics2D) {
         return graphics2D.getFontMetrics(graphics2D.getFont()).charsWidth(content.toCharArray(), 0, content.length());
     }
-
-    public static void main(String[] args) {
-        ImageUtils i = new ImageUtils();
-        String theme = "两件套";
-        Font font = new Font("Aa棉花糖", Font.BOLD + Font.PLAIN, 60);
-        String borderPath = "D:/border/mb/ff0099.png";
-        String coverPath = "D:/image/cover/" + theme;
-        String sourcePath = "D:/image/source/" + theme;
-        String targetPath = "D:/image/target/" + theme;
-        File coverFile = new File(coverPath);
-        if (!coverFile.exists()) {
-            coverFile.mkdirs();
-        }
-
-        int bx = 20;
-        int by = 50;
-        int cx = 70;
-        int cy = 200;
-
-        Color color = new Color(255, 0, 153, 255);
-        List<String> list = i.getContentList();
-        File sourceFile = new File(sourcePath);
-
-        String[] sourceFiles = sourceFile.list();
-        for (String str : sourceFiles) {
-            String imagePath = sourceFile.getAbsolutePath() + "/" + str;
-            new ImageUtils().resize(720, 1280, imagePath, targetPath + "/" + UUID.randomUUID() + ".jpg");
-        }
-
-        File file = new File(targetPath);
-        if (!file.exists()) {
-            file.mkdirs();
-        }
-        String[] files = file.list();
-        for (String str : files) {
-            String imagePath = file.getAbsolutePath() + "/" + str;
-            try {
-                File imageFile = new File(imagePath);
-                for (String content : list) {
-                    InputStream inputStream = new FileInputStream(imageFile);
-                    i.addTextInImage(theme, inputStream, content, color, font, borderPath, coverPath, bx, by, cx, cy);
-                }
-            } catch (IOException e) {
-                e.printStackTrace();
-            }
-        }
-    }
-
     public List<String> getContentList() {
         List<String> list = new ArrayList<String>();
         //秋装

Datei-Diff unterdrückt, da er zu groß ist
+ 14 - 5
module-crawler/src/main/java/cn/com/ctop/crawler/modules/core/service/impl/KuaishouCrawlerServiceImpl.java


+ 1 - 1
module-crawler/src/main/java/cn/com/ctop/crawler/modules/core/util/KuaimiUtil.java

@@ -10,7 +10,7 @@ import java.util.regex.Pattern;
  * 2019年11月12日10:04:00
  */
 public class KuaimiUtil {
-    public static final String KUAIMI_API_URL = "http://api.kmiyz.com/api/do.php";
+    public static final String KUAIMI_API_URL = "http://api.caihyz.com/api/do.php";
 
     public static String kuaimiLogin() throws Exception {
         String result = HttpUtils.httpPostNoParamRequest(KUAIMI_API_URL + "?action=loginIn&name=xuzuoyun&password=heaven01");

+ 12 - 10
module-crawler/src/main/java/cn/com/ctop/crawler/modules/core/util/KuaishouUtil.java

@@ -1,21 +1,19 @@
 package cn.com.ctop.crawler.modules.core.util;
 
+import cn.com.ctop.common.module.entity.IpPool;
 import cn.com.ctop.crawler.modules.core.entity.HttpBody;
 import cn.com.ctop.crawler.modules.core.entity.HttpCookie;
 import cn.com.ctop.crawler.modules.core.entity.HttpEntity;
 import cn.com.ctop.crawler.modules.core.entity.HttpHeader;
 import cn.com.ctop.crawler.modules.log.entity.CrawlerLog;
 import com.google.gson.Gson;
+import org.apache.http.HttpHost;
 import org.apache.http.HttpResponse;
-import org.apache.http.client.CookieStore;
 import org.apache.http.client.config.CookieSpecs;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
 import org.apache.http.conn.ssl.TrustStrategy;
-import org.apache.http.cookie.ClientCookie;
-import org.apache.http.cookie.Cookie;
-import org.apache.http.cookie.CookieOrigin;
 import org.apache.http.entity.StringEntity;
 import org.apache.http.impl.client.BasicCookieStore;
 import org.apache.http.impl.client.CloseableHttpClient;
@@ -30,7 +28,7 @@ import java.security.cert.X509Certificate;
 import java.util.*;
 
 public class KuaishouUtil {
-    public static CrawlerLog requestData(HttpEntity httpEntity){
+    public static CrawlerLog requestData(HttpEntity httpEntity, IpPool ipPool) {
         long beginTime = System.currentTimeMillis();
         CrawlerLog crawlerLog = new CrawlerLog();
         String salt = "382700b563f4";
@@ -64,8 +62,14 @@ public class KuaishouUtil {
                     return true;
                 }
             }).build();
-            RequestConfig globalConfig = RequestConfig.custom().setConnectTimeout(30000).setSocketTimeout(30000).setCookieSpec(CookieSpecs.STANDARD).build();
+            RequestConfig globalConfig = null;
             SSLConnectionSocketFactory sslFactory = new SSLConnectionSocketFactory(sslContext);
+            if (ipPool != null) {
+                HttpHost proxy = new HttpHost(ipPool.getIp(), ipPool.getPort(), "http");
+                globalConfig = RequestConfig.custom().setProxy(proxy).setConnectTimeout(60000).setSocketTimeout(60000).setCookieSpec(CookieSpecs.STANDARD).build();
+            } else {
+                globalConfig = RequestConfig.custom().setConnectTimeout(60000).setSocketTimeout(60000).setCookieSpec(CookieSpecs.STANDARD).build();
+            }
             CloseableHttpClient httpClient = HttpClients.custom().setDefaultCookieStore(cookieStore).setDefaultRequestConfig(globalConfig).setConnectionReuseStrategy((response, context) -> false ).setSSLSocketFactory(sslFactory).build();
             HttpPost httppost = new HttpPost(httpEntity.getUrl()+httpEntity.getPath()+"?"+staticQueryString);
             HttpHeader header = httpEntity.getHttpHeader();
@@ -106,7 +110,7 @@ public class KuaishouUtil {
         return String.valueOf(nextInt);
     }
 
-    public static CrawlerLog kuaishouAppDataFeatch(String url,String path,String urlParam,String bodyParam,String headerParam,String token){
+    public static CrawlerLog kuaishouAppDataFeatch(String url, String path, String urlParam, String bodyParam, String headerParam, String token, IpPool ipPool) {
         HttpEntity httpEntity = new HttpEntity();
         Map<String,String> staticParams = HttpParamUtil.urlParamToMap(urlParam);
         Map<String,String> dynamicParams = HttpParamUtil.urlParamToMap(bodyParam);
@@ -128,8 +132,6 @@ public class KuaishouUtil {
         httpBody.setStaticParams(staticParams);
         httpEntity.setHttpHeader(header);
         httpEntity.setHttpBody(httpBody);
-        return KuaishouUtil.requestData(httpEntity);
+        return KuaishouUtil.requestData(httpEntity, ipPool);
     }
-
-
 }

+ 2 - 8
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/graphql/service/impl/KuaishouWebInterfaceServiceImpl.java

@@ -24,7 +24,6 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.gson.Gson;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.io.FileUtils;
-import org.apache.commons.logging.LogFactory;
 import org.apache.fontbox.ttf.CmapSubtable;
 import org.apache.fontbox.ttf.GlyphData;
 import org.apache.fontbox.ttf.TTFParser;
@@ -715,6 +714,7 @@ public class KuaishouWebInterfaceServiceImpl implements IKuaishouWebInterfaceSer
         }
     }
 
+    @Override
     public void sendPreview(Integer creativeId, String kwid) {
         try {
             List<Cookie> list = HttpUtils2.cookieStore.getCookies();
@@ -744,6 +744,7 @@ public class KuaishouWebInterfaceServiceImpl implements IKuaishouWebInterfaceSer
         }
     }
 
+    @Override
     public void adekuaishouWebLogin(String phone, String password) throws IOException {
         ChromeDriverService service = new ChromeDriverService.Builder().usingDriverExecutable(new File("D:/chromedriver.exe")).usingAnyFreePort().build();
         service.start();
@@ -818,19 +819,12 @@ public class KuaishouWebInterfaceServiceImpl implements IKuaishouWebInterfaceSer
                 clientCookie.setDomain("ad.e.kuaishou.com");
                 clientCookie.setPath(cookieMap.get("path"));
                 HttpUtils2.cookieStore.addCookie(clientCookie);
-//                clientCookie.setDomain("id.kuaishou.com");
-////                HttpUtils2.cookieStore.addCookie(clientCookie);
-////                clientCookie.setDomain("uc.e.kuaishou.com");
-////                HttpUtils2.cookieStore.addCookie(clientCookie);
             }
             String result = HttpUtils2.httpGetRequest("https://uc.e.kuaishou.com/rest/web/login?sid=kuaishou.ad.dsp&followUrl=https%3A%2F%2Fad.e.kuaishou.com%2F%23%2Findex");
             System.out.println(result);
         } catch (Exception e) {
             e.printStackTrace();
         } finally {
-//            webDriver.quit();
-//            service.stop();
-//            HttpUtils.cookieStore.clear();
         }
     }