Pārlūkot izejas kodu

抽取巨量创意爬虫逻辑

syh 5 gadi atpakaļ
vecāks
revīzija
3cd3b2cc14

+ 1 - 0
jeecg-boot-module-system/src/main/java/org/jeecg/config/ShiroConfig.java

@@ -148,6 +148,7 @@ public class ShiroConfig {
 		filterChainDefinitionMap.put("/ks/web/test", "anon");
         filterChainDefinitionMap.put("/ureport/**", "anon");
         filterChainDefinitionMap.put("/ctop/advertiser/**", "anon");
+		filterChainDefinitionMap.put("/pangolin/**", "anon");
 
 		// 添加自己的过滤器并且取名为jwt
 		Map<String, Filter> filterMap = new HashMap<>(1);

+ 2 - 2
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/PangolinController.java

@@ -95,8 +95,8 @@ public class PangolinController {
     @PostMapping("api/activation/list")
     public Map<String, Object> activationList(@RequestBody JSONObject data) {
         LoginUser user = (LoginUser) SecurityUtils.getSubject().getPrincipal();
-        String userId = user.getId();
-//        String userId = "6b4c821adc414dc8b7718ab63ccbfcaf";
+//        String userId = user.getId();
+        String userId = "6b4c821adc414dc8b7718ab63ccbfcaf";
         String roleCode = sysRoleService.getRoleCodeByUserId(userId);
         if(null!=roleCode&&(roleCode.equals("admin"))){
             userId = null;

+ 26 - 0
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/KuaishouAutoCommentLoginJob.java

@@ -0,0 +1,26 @@
+package org.jeecg.modules.ctop.job;
+
+import cn.com.ctop.common.module.entity.BindAccountLogin;
+import cn.com.ctop.common.module.service.IBindAccountLoginService;
+import cn.com.ctop.common.module.utils.CtopAdConstant;
+import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouWebInterfaceService;
+import org.quartz.Job;
+import org.quartz.JobExecutionContext;
+import org.quartz.JobExecutionException;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.util.List;
+
+public class KuaishouAutoCommentLoginJob implements Job {
+    @Autowired
+    private IBindAccountLoginService bindAccountLoginService;
+    @Autowired
+    private IKuaishouWebInterfaceService kuaishouWebInterfaceService;
+    @Override
+    public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
+        List<BindAccountLogin> loginList = bindAccountLoginService.getListByParams(CtopAdConstant.PLATFORM_TYPE_KUAISHOU_PY,1);
+        if(null!=loginList&&!loginList.isEmpty()){
+            loginList.forEach(login-> kuaishouWebInterfaceService.getkuaishouWebLoginCookie(login));
+        }
+    }
+}

+ 4 - 7
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/KuaishouCommentAutoDelete.java

@@ -2,8 +2,8 @@ package org.jeecg.modules.ctop.job;
 
 import cn.com.ctop.common.module.entity.BindAccountLogin;
 import cn.com.ctop.common.module.service.IBindAccountLoginService;
+import cn.com.ctop.common.module.utils.CtopAdConstant;
 import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouWebInterfaceService;
-import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import lombok.extern.slf4j.Slf4j;
 import org.quartz.Job;
 import org.quartz.JobExecutionContext;
@@ -28,12 +28,9 @@ public class KuaishouCommentAutoDelete implements Job {
         Thread thread = new Thread() {
             @Override
             public void run() {
-                QueryWrapper<BindAccountLogin> queryWrapper = new QueryWrapper<>();
-                queryWrapper.eq("login_type", "kuaishou");
-                queryWrapper.eq("status", 1);
-                List<BindAccountLogin> list = bindAccountLoginService.list(queryWrapper);
-                if (list != null && list.size() > 0) {
-                    for (BindAccountLogin account : list) {
+                List<BindAccountLogin>loginList = bindAccountLoginService.getListByParams(CtopAdConstant.PLATFORM_TYPE_KUAISHOU_PY,1);
+                if (loginList != null && !loginList.isEmpty()) {
+                    for (BindAccountLogin account : loginList) {
                         try {
                             log.info("正在删除评论:{}", account.getAccountName());
                             kuaishouWebInterfaceService.adkuaishouWebLogin(account.getAccountName(), account.getPassword());

+ 1 - 1
jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java

@@ -19,7 +19,7 @@ public class SampleTest {
         String account="3248395570@qq.com";
         String password = "Ydxq-704127411";
         oceanEngineService.login(account,password);
-        oceanEngineService.douyinHotHandler(1,1);
+//        oceanEngineService.douyinHotHandler(1,1);
         oceanEngineService.effectCaseHandler(1);
         oceanEngineService.hotMaterialHandler(1,1,"西瓜");
         oceanEngineService.hotMaterialHandler(1,3,"火山");

+ 3 - 0
module-common/src/main/java/cn/com/ctop/common/module/service/IBindAccountLoginService.java

@@ -3,6 +3,7 @@ package cn.com.ctop.common.module.service;
 import cn.com.ctop.common.module.entity.BindAccountLogin;
 import com.baomidou.mybatisplus.extension.service.IService;
 
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -21,4 +22,6 @@ public interface IBindAccountLoginService extends IService<BindAccountLogin> {
     BindAccountLogin selectByAccountId(String accountId);
 
     Map<String, Object> getListByUserId(String userId);
+
+    List<BindAccountLogin> getListByParams(String loginType,Integer status);
 }

+ 12 - 0
module-common/src/main/java/cn/com/ctop/common/module/service/impl/BindAccountLoginServiceImpl.java

@@ -48,4 +48,16 @@ public class BindAccountLoginServiceImpl extends ServiceImpl<BindAccountLoginMap
         ResultMapUtils.setResultMap(result, StatusCode.COMMON_SUCCESS);
         return result;
     }
+
+    @Override
+    public List<BindAccountLogin> getListByParams(String loginType, Integer status) {
+        QueryWrapper<BindAccountLogin> queryWrapper = new QueryWrapper<>();
+        if(null!=loginType&&!loginType.equals("")){
+            queryWrapper.eq("login_type",loginType);
+        }
+        if(status!=null){
+            queryWrapper.eq("status",status);
+        }
+        return this.list(queryWrapper);
+    }
 }

+ 2 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/service/impl/OceanEngineServiceImpl.java

@@ -67,6 +67,7 @@ public class OceanEngineServiceImpl implements IOceanEngineService {
     @Override
     public void hotMaterialHandler(int page, int appCode, String appName){
         String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_material/list?material_type=3&order_by=convert_show_rate&period_type=3&aggr_app_code="+appCode+"&page="+page+"&limit=20");
+        System.out.println(result);
         ObjectMapper mapper = new ObjectMapper();
         mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
         try {
@@ -119,6 +120,7 @@ public class OceanEngineServiceImpl implements IOceanEngineService {
     @Override
     public void effectCaseHandler(int page){
         String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/case/effect_case_list?industry=0&style=0&mode=0&page="+page+"&limit=20");
+        System.out.println(result);
         ObjectMapper mapper = new ObjectMapper();
         mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
         try {

+ 1 - 2
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinLoginServiceImpl.java

@@ -85,7 +85,7 @@ public class PangolinLoginServiceImpl implements PangolinLoginService {
                 String cookieStr = iterator.next().toString();
                 cookieList.add(cookieStr);
                 String[] cookieArray = cookieStr.split(";");
-                Map<String, String> cookieMap = new HashMap<String, String>();
+                Map<String, String> cookieMap = new HashMap<>();
                 for (String cookie : cookieArray) {
                     String[] kv = cookie.split("=");
                     if (kv.length > 1) {
@@ -111,7 +111,6 @@ public class PangolinLoginServiceImpl implements PangolinLoginService {
             e.printStackTrace();
         } finally {
             webDriver.manage().deleteAllCookies();
-//            webDriver.close();
             webDriver.quit();
         }
     }

+ 17 - 13
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/graphql/service/IKuaishouWebInterfaceService.java

@@ -1,31 +1,35 @@
 package cn.com.ctop.kuaishou.modules.graphql.service;
 
+import cn.com.ctop.common.module.entity.BindAccountLogin;
+
 import java.io.IOException;
 import java.util.Map;
 
 public interface IKuaishouWebInterfaceService {
-    public void getVideoList(String uid) throws IOException;
+    void getVideoList(String uid) throws IOException;
+
+    String checkKuaishouUser(String videoUrl);
 
-    public String checkKuaishouUser(String videoUrl);
+    void adkuaishouWebLogin(String phone, String password) throws IOException;
 
-    public void adkuaishouWebLogin(String phone, String password) throws IOException;
+    void deleteAllComment(Map<String, String> pcursorMap);
 
-    public void deleteAllComment(Map<String, String> pcursorMap);
+    String qrLogin(String taskId) throws Exception;
 
-    public String qrLogin(String taskId) throws Exception;
+    Map<String, Object> loginStatus(String taskId);
 
-    public Map<String, Object> loginStatus(String taskId);
+    Map<String, Object> commentDelete(String userId, String ksid, String photoId, Long commentId);
 
-    public Map<String, Object> commentDelete(String userId, String ksid, String photoId, Long commentId);
+    Map<String, Object> videoList(String userId, String ksid, String pcursor);
 
-    public Map<String, Object> videoList(String userId, String ksid, String pcursor);
+    Map<String, Object> commentList(String userId, String ksid, String photoId, String pcursor);
 
-    public Map<String, Object> commentList(String userId, String ksid, String photoId, String pcursor);
+    Map<String, Object> commentAdd(String userId, String ksid, String photoId, String content, Long replyToCommentId, Long replyTo);
 
-    public Map<String, Object> commentAdd(String userId, String ksid, String photoId, String content, Long replyToCommentId, Long replyTo);
+    void getkuaishouWebLoginCookie(BindAccountLogin login);
 
-    public void sendPreview(Integer creativeId, String kwid);
+    void sendPreview(Integer creativeId, String kwid);
 
-    public void adekuaishouWebLogin(String phone, String password) throws IOException;
-    public Map<String, Object> subCommentList(String userId, String ksid, String photoId, Long rootCommentId, String pcursor);
+    void adekuaishouWebLogin(String phone, String password) throws IOException;
+    Map<String, Object> subCommentList(String userId, String ksid, String photoId, Long rootCommentId, String pcursor);
 }

+ 53 - 6
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/graphql/service/impl/KuaishouWebInterfaceServiceImpl.java

@@ -1,7 +1,9 @@
 package cn.com.ctop.kuaishou.modules.graphql.service.impl;
 
+import cn.com.ctop.common.module.entity.BindAccountLogin;
 import cn.com.ctop.common.module.entity.HttpClientEntity;
 import cn.com.ctop.common.module.entity.IpPool;
+import cn.com.ctop.common.module.service.IBindAccountLoginService;
 import cn.com.ctop.common.module.service.IIpPoolService;
 import cn.com.ctop.common.module.utils.HttpClientUtils;
 import cn.com.ctop.common.module.utils.HttpUtils2;
@@ -10,8 +12,6 @@ import cn.com.ctop.kuaishou.modules.graphql.entity.KuaishouUserCookie;
 import cn.com.ctop.kuaishou.modules.graphql.entity.KuaishouWebData;
 import cn.com.ctop.kuaishou.modules.graphql.entity.KuaishouWebResult;
 import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouUserCookieService;
-import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouUserService;
-import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouVideoService;
 import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouWebInterfaceService;
 import cn.com.ctop.kuaishou.modules.graphql.vo.KuaishouAcceptVO;
 import cn.com.ctop.kuaishou.modules.graphql.vo.KuaishouQrStartVO;
@@ -74,14 +74,12 @@ import java.util.regex.Pattern;
 @Service
 public class KuaishouWebInterfaceServiceImpl implements IKuaishouWebInterfaceService {
     @Autowired
-    private IKuaishouUserService kuaishouUserService;
-    @Autowired
-    private IKuaishouVideoService kuaishouVideoService;
-    @Autowired
     private IKuaishouUserCookieService kuaishouUserCookieService;
     @Autowired
     private IIpPoolService iIpPoolService;
     @Autowired
+    private IBindAccountLoginService  bindAccountLoginService;
+    @Autowired
     private RedisUtil redisUtil;
     public static String GRAPHQL_URL = "https://live.kuaishou.com/graphql";
 
@@ -653,6 +651,55 @@ public class KuaishouWebInterfaceServiceImpl implements IKuaishouWebInterfaceSer
         }
     }
 
+    @Override
+    public void getkuaishouWebLoginCookie(BindAccountLogin login){
+        ChromeDriverService service = new ChromeDriverService.Builder().usingDriverExecutable(new File(chromeDriver)).usingAnyFreePort().build();
+        try {
+            service.start();
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+        ChromeOptions chromeOptions = new ChromeOptions();
+        chromeOptions.addArguments("--headless");
+        chromeOptions.addArguments("--no-sandbox");
+        chromeOptions.addArguments("--disable-dev-shm-usage");
+        chromeOptions.addArguments("--disable-gpu");
+        chromeOptions.addArguments("--window-size=1920,1080");
+        chromeOptions.addArguments("--user-agent=" + HttpUtils2.USER_AGENT);
+        chromeOptions.setAcceptInsecureCerts(true);
+        WebDriver webDriver = new RemoteWebDriver(service.getUrl(), chromeOptions);
+        try {
+            //全局隐式等待
+            webDriver.manage().timeouts().implicitlyWait(10, TimeUnit.SECONDS);
+            //设定网址
+            webDriver.get("https://b.e.kuaishou.com");
+            //显示等待控制对象
+            WebDriverWait webDriverWait = new WebDriverWait(webDriver, 10);
+            webDriverWait.until(ExpectedConditions.elementToBeClickable(By.cssSelector(".phone input"))).sendKeys(login.getAccountName());
+            webDriverWait.until(ExpectedConditions.elementToBeClickable(By.cssSelector(".password input"))).sendKeys(login.getPassword());
+            //点击登录
+            webDriver.findElement(By.className("foot")).click();
+            //等待2秒用于页面加载,保证Cookie响应全部获取。
+            Thread.sleep(10000);
+            webDriver.get("https://b.e.kuaishou.com/#/home/profile");
+            //获取Cookie并打印
+            Set<org.openqa.selenium.Cookie> cookies = webDriver.manage().getCookies();
+            List<String> cookieList = new ArrayList<>();
+            Iterator iterator = cookies.iterator();
+            while (iterator.hasNext()) {
+                String cookieStr = iterator.next().toString();
+                cookieList.add(cookieStr);
+            }
+
+            login.setCookie(new Gson().toJson(cookieList));
+            bindAccountLoginService.updateById(login);
+        } catch (Exception e) {
+            e.printStackTrace();
+        } finally {
+            webDriver.quit();
+        }
+    }
+
     public static void main(String[] args) {
         System.setProperty("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.SimpleLog");
         System.setProperty("org.apache.commons.logging.simplelog.showdatetime", "true");