Jelajahi Sumber

修改爬虫文件

syh 4 tahun lalu
induk
melakukan
8fd5c024e9

+ 8 - 3
jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java

@@ -106,9 +106,14 @@ public class SampleTest {
 
     @Test
     public void loadKuaishouAgentData() {
-        for (int i = 0; i < 30; i++) {
-            String currentDate = DateUtils.formatDate(DateUtils.addDay(new Date(), -i));
-            kuaishouReportDailyAgentService.getReport(currentDate, DateUtils.getNowDate("yyyy-MM-dd"));
+        for(int i=0;i<20;i++){
+            String currentDate = DateUtils.formatDate(DateUtils.addDay(new Date(),-i));
+            kuaishouReportDailyAgentService.getReport(currentDate,DateUtils.getNowDate("yyyy-MM-dd"));
+        }
+
+        int currentPage = 1;
+        while (kuaishouReportDailyAgentService.getAccount(currentPage)){
+            currentPage++;
         }
     }
 }

+ 57 - 9
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/service/impl/KuaishouReportDailyAgentServiceImpl.java

@@ -1,5 +1,7 @@
 package cn.com.ctop.kuaishou.modules.report.service.impl;
 
+import cn.com.ctop.common.module.utils.Base64Utils;
+import cn.com.ctop.common.module.utils.ChaojiyingUtils;
 import cn.com.ctop.common.module.utils.HttpUtils2;
 import cn.com.ctop.kuaishou.modules.report.entity.KuaishouAgentAccount;
 import cn.com.ctop.kuaishou.modules.report.entity.KuaishouReportDailyAgent;
@@ -8,25 +10,28 @@ import cn.com.ctop.kuaishou.modules.report.mapper.KuaishouAgentAccountMapper;
 import cn.com.ctop.kuaishou.modules.report.mapper.KuaishouReportDailyAgentMapper;
 import cn.com.ctop.kuaishou.modules.report.mapper.KuaishouReportDailyAgentSumMapper;
 import cn.com.ctop.kuaishou.modules.report.service.IKuaishouReportDailyAgentService;
+import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.FileUtils;
 import org.apache.http.impl.client.BasicCookieStore;
 import org.apache.http.impl.cookie.BasicClientCookie;
 import org.jeecg.common.util.DateUtils;
-import org.openqa.selenium.By;
-import org.openqa.selenium.Cookie;
-import org.openqa.selenium.WebDriver;
-import org.openqa.selenium.WebElement;
+import org.openqa.selenium.*;
+import org.openqa.selenium.NoSuchElementException;
 import org.openqa.selenium.chrome.ChromeDriver;
 import org.openqa.selenium.chrome.ChromeOptions;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
+import javax.imageio.ImageIO;
+import java.awt.image.BufferedImage;
+import java.io.File;
 import java.math.BigDecimal;
 import java.text.SimpleDateFormat;
 import java.util.*;
@@ -60,8 +65,8 @@ public class KuaishouReportDailyAgentServiceImpl extends ServiceImpl<KuaishouRep
         System.getProperties().setProperty("webdriver.chrome.driver", chromeDriver);
         String url = "https://agent.e.kuaishou.com";
         ChromeOptions chromeOptions = new ChromeOptions();
-        chromeOptions.addArguments("--headless");
-        chromeOptions.addArguments("--no-sandbox");
+//        chromeOptions.addArguments("--headless");
+//        chromeOptions.addArguments("--no-sandbox");
         chromeOptions.addArguments("--disable-dev-shm-usage");
         chromeOptions.addArguments("--incognito");
         chromeOptions.addArguments("--disable-gpu");
@@ -86,7 +91,21 @@ public class KuaishouReportDailyAgentServiceImpl extends ServiceImpl<KuaishouRep
             //点击登录
             loginElement.click();
             Thread.sleep(3000L);
+            if (null!=webDriverGetElement(webDriver,By.tagName("iframe"))) {
+                log.error("【错误】快手登录出现验证码!");
+                File scrFile = captureElement(webDriver.findElement(By.tagName("iframe")));
+                //clickCharBg
+                String savePath = "D:/temp.png";
+                FileUtils.copyFile(scrFile, new File(savePath));
+                webDriver.switchTo().frame(webDriver.findElement(By.tagName("iframe")));
+                String result = ChaojiyingUtils.process(Base64Utils.imageToBase64(savePath),9104);
+                System.out.println(result);
+                String pointString = JSONObject.parseObject(result).getString("pic_str");
 
+                ((JavascriptExecutor) webDriver).executeScript("arguments[0].click();", webDriver.findElement(By.className("aq_icon_refresh")));
+            }
+            webDriver.get("https://agent.e.kuaishou.com/#/index");
+            Thread.sleep(3000L);
             Set<Cookie> cookies = webDriver.manage().getCookies();
             Iterator iterator = cookies.iterator();
             while (iterator.hasNext()) {
@@ -116,6 +135,35 @@ public class KuaishouReportDailyAgentServiceImpl extends ServiceImpl<KuaishouRep
         return loginStatus;
     }
 
+    private WebElement webDriverGetElement(WebDriver webDriver,By by){
+        try {
+            return webDriver.findElement(by);
+        }catch (NoSuchElementException e){
+            return null;
+        }
+    }
+
+    public File captureElement(WebElement element) throws Exception {
+        // TODO Auto-generated method stub
+        WrapsDriver wrapsDriver = (WrapsDriver) element;
+        // 截图整个页面
+        File screen = ((TakesScreenshot) wrapsDriver.getWrappedDriver()).getScreenshotAs(OutputType.FILE);
+//        ((TakesScreenshot) wrapsDriver.getWrappedDriver()).getScreenshotAs(OutputType.)
+        String tempPath = "D:/temp.png";
+        FileUtils.copyFile(screen, new File(tempPath));
+        BufferedImage img = ImageIO.read(new File(tempPath));
+        // 获得元素的高度和宽度
+        int width = element.getSize().getWidth();
+        int height = element.getSize().getHeight();
+//        // 得到元素的坐标
+        Point p = element.getLocation();
+        BufferedImage dest = img.getSubimage(p.x, p.y, width, height);
+        // 存为png格式
+        ImageIO.write(dest, "png", screen);
+        return screen;
+    }
+
+
     @Override
     public boolean getAccount(int currentPage) {
         boolean hasNextPage = false;
@@ -208,7 +256,7 @@ public class KuaishouReportDailyAgentServiceImpl extends ServiceImpl<KuaishouRep
                             kuaishouReportDailyAgentSum.setConvertCount(dataNode.get("actionbarClick").asLong());
                             kuaishouReportDailyAgentSum.setCost(dataNode.get("totalChargedInYuan").decimalValue());
                             kuaishouReportDailyAgentSum.setCostCampaignCount(dataNode.get("chargedCampaignCount").asLong());
-                            //-   kuaishouReportDailyAgentSum.setDate(sdf.parse(currentDate));
+                            kuaishouReportDailyAgentSum.setDate(currentDate);
                             kuaishouReportDailyAgentSum.setFengmianClickCount(dataNode.get("adPhotoClick").asLong());
                             kuaishouReportDailyAgentSum.setFengmianClickRate(dataNode.get("clickRatio").decimalValue());
                             kuaishouReportDailyAgentSum.setFengmianShowCount(dataNode.get("impression").asLong());
@@ -232,9 +280,9 @@ public class KuaishouReportDailyAgentServiceImpl extends ServiceImpl<KuaishouRep
                             kuaishouReportDailyAgent.setConvertCount(dataNode.get("actionbarClick").asInt());
                             kuaishouReportDailyAgent.setCost(dataNode.get("totalChargedInYuan").decimalValue());
                             kuaishouReportDailyAgent.setCostCampaignCount(dataNode.get("chargedCampaignCount").asInt());
-                            //      kuaishouReportDailyAgent.setDate(sdf.parse(dataNode.get("dateTime").asText()));
+                            kuaishouReportDailyAgent.setDate(dataNode.get("dateTime").asText());
                             kuaishouReportDailyAgent.setFengmianClickCount(dataNode.get("adPhotoClick").asInt());
-                            // kuaishouReportDailyAgent.setFengmianClickRate(dataNode.get("clickRatio").decimalValue());
+                            kuaishouReportDailyAgent.setFengmianClickRate(dataNode.get("clickRatio").asText());
                             kuaishouReportDailyAgent.setFengmianShowCount(dataNode.get("impression").asInt());
                             kuaishouReportDailyAgent.setFandianCost(dataNode.get("rebateRealChargedInYuan").decimalValue());
                             kuaishouReportDailyAgent.setJiliCost(dataNode.get("directRebateRealChargedInYuan").decimalValue());