Bladeren bron

修改穿山甲爬虫代码

syh 5 jaren geleden
bovenliggende
commit
e9e6040426

+ 48 - 0
jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java

@@ -8,8 +8,13 @@ import cn.com.ctop.common.module.utils.CtopAdConstant;
 import cn.com.ctop.crawler.modules.oceanengine.entity.HotMaterial;
 import cn.com.ctop.crawler.modules.oceanengine.service.IHotMaterialService;
 import cn.com.ctop.crawler.modules.oceanengine.service.IOceanEngineService;
+import cn.com.ctop.crawler.modules.pangolin.entity.PangolinApp;
+import cn.com.ctop.crawler.modules.pangolin.service.PangolinAppService;
+import cn.com.ctop.crawler.modules.pangolin.service.PangolinCrawlerService;
+import cn.com.ctop.crawler.modules.pangolin.service.PangolinLoginService;
 import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouWebInterfaceService;
 import cn.com.ctop.toutiao.modules.report.service.IBytedanceFundDailyService;
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.xxl.job.core.log.XxlJobLogger;
 import lombok.extern.slf4j.Slf4j;
 import org.jeecg.common.util.DateUtils;
@@ -144,6 +149,49 @@ public class SampleTest {
         }
         System.out.println("执行完成");
     }
+
+    @Autowired
+    private PangolinLoginService pangolinLoginService;
+    @Autowired
+    private PangolinCrawlerService pangolinCrawlerService;
+    @Autowired
+    private PangolinAppService pangolinAppService;
+    @Test
+    public void testPangolinLogin(){
+        QueryWrapper<BindAccountLogin> queryWrapper = new QueryWrapper<>();
+        queryWrapper.eq("login_type","pangolin");
+        queryWrapper.eq("status",1);
+        List<BindAccountLogin> list = bindAccountLoginService.list(queryWrapper);
+        if(list!=null&&!list.isEmpty()){
+            for (BindAccountLogin login:list) {
+                if(null==login.getCookie()||"".equals(login.getCookie().trim())){
+                    pangolinLoginService.pangolinLogin(login.getAccountName(), login.getPassword());
+                }
+            }
+        }
+    }
+
+    @Test
+    public void testPangolinData(){
+        QueryWrapper<BindAccountLogin> queryWrapper = new QueryWrapper<>();
+        queryWrapper.eq("login_type", "pangolin");
+        queryWrapper.eq("status", 1);
+        queryWrapper.isNotNull("cookie");
+        List<BindAccountLogin> list = bindAccountLoginService.list(queryWrapper);
+        if (list != null && !list.isEmpty()) {
+            for (BindAccountLogin bindAccountLogin : list) {
+                pangolinCrawlerService.getChannelList(bindAccountLogin);
+                pangolinCrawlerService.getActivationList(bindAccountLogin, DateUtils.getNowDate("yyyy-MM-dd"), 1);
+                pangolinCrawlerService.getCheckList(bindAccountLogin);
+                QueryWrapper<PangolinApp> queryWrapper1 = new QueryWrapper<>();
+                queryWrapper1.eq("account_name", bindAccountLogin.getAccountName());
+                List<PangolinApp> appList = pangolinAppService.list(queryWrapper1);
+                if (appList != null && !appList.isEmpty()) {
+                    appList.forEach(pangolinApp -> pangolinCrawlerService.getRealTimeList(bindAccountLogin, pangolinApp.getId()));
+                }
+            }
+        }
+    }
 }
 
 

+ 1 - 1
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/PangolinLoginService.java

@@ -1,5 +1,5 @@
 package cn.com.ctop.crawler.modules.pangolin.service;
 
 public interface PangolinLoginService {
-    public void pangolinLogin(String username,String password);
+    void pangolinLogin(String username,String password);
 }

+ 33 - 33
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinLoginServiceImpl.java

@@ -5,9 +5,8 @@ import cn.com.ctop.common.module.service.IBindAccountLoginService;
 import cn.com.ctop.common.module.utils.ChaojiyingUtils;
 import cn.com.ctop.common.module.utils.HttpUtils2;
 import cn.com.ctop.crawler.modules.pangolin.service.PangolinLoginService;
+import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.gson.Gson;
 import org.apache.http.impl.client.BasicCookieStore;
 import org.apache.http.impl.cookie.BasicClientCookie;
@@ -17,6 +16,7 @@ import org.openqa.selenium.chrome.ChromeOptions;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
+
 import java.util.*;
 
 
@@ -45,33 +45,38 @@ public class PangolinLoginServiceImpl implements PangolinLoginService {
             webDriver.manage().deleteAllCookies();
             //获取登录页面
             webDriver.get(url);
+            Thread.sleep(3000L);
+            WebElement accountElement = webDriver.findElement(By.xpath("//input[@id='username']"));
+            accountElement.sendKeys(username);
+            Thread.sleep(3000L);
+            WebElement passwordElement = webDriver.findElement(By.xpath("//input[@id='password']"));
+            passwordElement.sendKeys(password);
+            WebElement captchaElement = webDriver.findElement(By.xpath("//div[@class='ant-form-item-control ']/img"));
+            String outputType = captchaElement.getScreenshotAs(OutputType.BASE64);
+            String res = ChaojiyingUtils.process(outputType,1902);
+            JSONObject capachaObject = JSONObject.parseObject(res);
+            Integer code = capachaObject.getInteger("err_no");
+            String captcha = capachaObject.getString("pic_str");
+            WebElement captchaInputElement = webDriver.findElement(By.xpath("//input[@id='captcha']"));
+            captchaInputElement.sendKeys(captcha);
+            WebElement loginElement = webDriver.findElement(By.xpath("//button[@type='submit']"));
+            Thread.sleep(3000L);
+            //点击登录
+            loginElement.click();
+            Thread.sleep(3000L);
+            webDriver.get("https://pangolin.bytedance.com");
+            Thread.sleep(3000L);
             List<WebElement> list = webDriver.findElements(By.xpath("//input[@id='captcha']"));
-            do {
-                Thread.sleep(3000L);
-                WebElement accountElement = webDriver.findElement(By.xpath("//input[@id='username']"));
-                accountElement.sendKeys(username);
-                Thread.sleep(3000L);
-                WebElement passwordElement = webDriver.findElement(By.xpath("//input[@id='password']"));
-                passwordElement.sendKeys(password);
-                WebElement captchaElement = webDriver.findElement(By.xpath("//div[@class='ant-form-item-control ']/img"));
-                String outputType = captchaElement.getScreenshotAs(OutputType.BASE64);
-                String res = ChaojiyingUtils.process(outputType,1902);
-                ObjectMapper mapper = new ObjectMapper();
-                JsonNode n = mapper.readTree(res);
-                String captcha = n.get("pic_str").asText();
-                WebElement captchaInputElement = webDriver.findElement(By.xpath("//input[@id='captcha']"));
-                captchaInputElement.sendKeys(captcha);
-                WebElement loginElement = webDriver.findElement(By.xpath("//button[@type='submit']"));
-                Thread.sleep(3000L);
-                //点击登录
-                loginElement.click();
-                Thread.sleep(3000L);
-                webDriver.get("https://pangolin.bytedance.com");
-                list = webDriver.findElements(By.xpath("//input[@id='captcha']"));
-            }while (list != null && list.size() > 0);
-
-
-
+            QueryWrapper<BindAccountLogin> queryWrapper = new QueryWrapper<>();
+            queryWrapper.eq("account_name",username);
+            queryWrapper.eq("login_type","pangolin");
+            BindAccountLogin bindAccountLogin = bindAccountLoginService.getOne(queryWrapper);
+            if(null != code && code == 0 && null != list && !list.isEmpty()){
+                //表示验证码正确,登录不成功
+                bindAccountLogin.setStatus("2");
+                bindAccountLoginService.updateById(bindAccountLogin);
+                return ;
+            }
             Set<Cookie> cookies = webDriver.manage().getCookies();
             List<String> cookieList = new ArrayList<>();
             Iterator iterator = cookies.iterator();
@@ -92,11 +97,6 @@ public class PangolinLoginServiceImpl implements PangolinLoginService {
                 clientCookie.setPath(cookieMap.get("path"));
                 HttpUtils2.cookieStore.addCookie(clientCookie);
             }
-
-            QueryWrapper<BindAccountLogin> queryWrapper = new QueryWrapper<>();
-            queryWrapper.eq("account_name",username);
-            queryWrapper.eq("login_type","pangolin");
-            BindAccountLogin bindAccountLogin = bindAccountLoginService.getOne(queryWrapper);
             if (bindAccountLogin != null){
                 bindAccountLogin.setCookie(new Gson().toJson(cookieList));
                 bindAccountLoginService.updateById(bindAccountLogin);