Sfoglia il codice sorgente

修改爬虫逻辑代码

syh 5 anni fa
parent
commit
f5ecb39618

+ 22 - 0
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/ActorController.java

@@ -338,4 +338,26 @@ public class ActorController {
 
         return result;
     }
+
+    public static String longestCommonPrefix(String[] strs) {
+        if (strs.length == 0) {
+            return "";
+        }
+        String prefix = strs[0];
+        for (int i = 1; i < strs.length; i++) {
+            while (strs[i].indexOf(prefix) != 0) {
+                prefix = prefix.substring(0, prefix.length() - 1);
+                if (prefix.isEmpty()) {
+                    return "";
+                }
+            }
+        }
+        return prefix;
+    }
+
+
+    public static void main(String[] args) {
+        String[] strings = {"abcd", "adc", "ab", "a"};
+        System.out.println(longestCommonPrefix(strings));
+    }
 }

+ 0 - 17
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/service/impl/CreateInternalServiceImpl.java

@@ -207,23 +207,6 @@ public class CreateInternalServiceImpl implements ICreateInternalService {
         return resultMap;
     }
 
-    public Map<String, Object> bindAdByKuaishou() {
-        Map<String, Object> result = new HashMap<>();
-        String url = "https://ad.oceanengine.com/pages/login/index.html";
-        System.getProperties().setProperty("webdriver.chrome.driver", chromeDriver);
-        ChromeOptions chromeOptions = new ChromeOptions();
-        chromeOptions.addArguments("--headless");
-        chromeOptions.addArguments("--incognito");
-        chromeOptions.addArguments("--disable-gpu");
-        chromeOptions.addArguments("--no-sandbox");
-        chromeOptions.addArguments("--window-size=1920,1080");
-        chromeOptions.addArguments("--user-agent=" + HttpUtils2.USER_AGENT);
-        chromeOptions.setAcceptInsecureCerts(true);
-        WebDriver webDriver = new ChromeDriver(chromeOptions);
-        WebElement element = webDriver.findElement(By.xpath(""));
-        return result;
-    }
-
     @Override
     public Map<String, Object> createInternal(JSONObject requestJson) {
         Map<String, Object> resultMap = new HashMap<>();

+ 106 - 1
jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java

@@ -1,10 +1,22 @@
 package org.jeecg;
 
+import cn.com.ctop.common.module.utils.HttpUtils2;
+import cn.com.ctop.common.module.utils.StatusCode;
 import cn.com.ctop.crawler.modules.appium.service.IAppiumJobService;
+import cn.com.ctop.crawler.modules.appium.service.IAppiumTaskService;
+import com.alibaba.fastjson.JSONObject;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.http.impl.client.BasicCookieStore;
+import org.jeecg.modules.ctop.service.ICreateInternalService;
 import org.junit.Test;
 import org.junit.runner.RunWith;
+import org.openqa.selenium.By;
+import org.openqa.selenium.WebDriver;
+import org.openqa.selenium.WebElement;
+import org.openqa.selenium.chrome.ChromeDriver;
+import org.openqa.selenium.chrome.ChromeOptions;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.test.context.junit4.SpringRunner;
 
@@ -12,10 +24,103 @@ import org.springframework.test.context.junit4.SpringRunner;
 @SpringBootTest
 @Slf4j
 public class SampleTest {
+    @Value("${jeecg.path.chrome-driver}")
+    private String chromeDriver;
     @Autowired
     private IAppiumJobService jobService;
+    @Autowired
+    private ICreateInternalService createInternalService;
+    @Autowired
+    private IAppiumTaskService appiumTaskService;
     @Test
-    public void testJob() {
+    public void testGetKuaishouId() {
         jobService.loginTask(2L);
     }
+
+    @Test
+    public void testKsDown() {
+        appiumTaskService.runTask(1, 1);
+    }
+
+    @Test
+    public void testKs() throws InterruptedException {
+        String url = "https://ad.e.kuaishou.com/#/welcome?redirectUrl=https%3A%2F%2Fad.e.kuaishou.com%2F%23%2Findex";
+        System.getProperties().setProperty("webdriver.chrome.driver", chromeDriver);
+        ChromeOptions chromeOptions = new ChromeOptions();
+//        chromeOptions.addArguments("--headless");
+        chromeOptions.addArguments("--incognito");
+        chromeOptions.addArguments("--disable-gpu");
+//        chromeOptions.addArguments("--no-sandbox");
+        chromeOptions.addArguments("--window-size=1920,1080");
+        chromeOptions.addArguments("--user-agent=" + HttpUtils2.USER_AGENT);
+        chromeOptions.setAcceptInsecureCerts(true);
+        WebDriver webDriver = new ChromeDriver(chromeOptions);
+        try {
+            Thread.sleep(3000L);
+            HttpUtils2.cookieStore = new BasicCookieStore();
+            webDriver.manage().deleteAllCookies();
+            //获取登录页面
+            webDriver.get(url);
+            Thread.sleep(3000L);
+            WebElement accountElement = webDriver.findElement(By.xpath("//div[@class='phone ']/input[@type='text']"));
+            accountElement.sendKeys("19845004383");
+            Thread.sleep(3000L);
+            WebElement passwordElement = webDriver.findElement(By.xpath("//div[@class='password ']/input[@type='password']"));
+            passwordElement.sendKeys("a123456");
+            WebElement loginElement = webDriver.findElement(By.xpath("//div[@class='foot']"));
+            Thread.sleep(3000L);
+            //点击登录
+            loginElement.click();
+            Thread.sleep(3000L);
+            //获取推广按钮
+            WebElement spreadElement = webDriver.findElement(By.linkText("推广"));
+            spreadElement.click();
+            Thread.sleep(3000L);
+            //选择广告创意
+            WebElement creativeElement = webDriver.findElement(By.xpath("//div[text()='广告创意']"));
+            creativeElement.click();
+            Thread.sleep(3000L);
+            //输入创意名称,点击搜索
+            WebElement searchElement = webDriver.findElement(By.xpath("//input[@type='text']"));
+            searchElement.sendKeys("2-这个是你画的吗-设计-11.19");
+            Thread.sleep(3000L);
+            WebElement searchButton = webDriver.findElement(By.xpath("//button[@class='ant-btn ant-input-search-button ant-btn-primary']"));
+            searchButton.click();
+            Thread.sleep(3000L);
+            WebElement tiyanElement = webDriver.findElement(By.linkText("体验"));
+            tiyanElement.click();
+            Thread.sleep(2000L);
+            WebElement inputKsIdElement = webDriver.findElement(By.xpath("//textarea[@placeholder='请输入快手账号…']"));
+            inputKsIdElement.sendKeys("123456765");
+            Thread.sleep(2000L);
+            WebElement chufaElement = webDriver.findElement(By.xpath("//button[@class='creative-experience-btn-enable']"));
+            chufaElement.click();
+
+
+        } catch (Exception e) {
+            e.printStackTrace();
+            webDriver.manage().deleteAllCookies();
+            webDriver.close();
+        }
+    }
+
+    @Test
+    public void test() {
+        JSONObject object = new JSONObject();
+        createInternalService.createInternal(object);
+        StatusCode.KUAISHOU_CRAWLER_APP_SUCCESS.getCode();
+    }
+
+    @Test
+    public void testCrawler() {
+        String account = "";
+        String password = "";
+        String creativeName = "";
+        //1:获取快手账号id
+        String ksId = jobService.loginTask(2L);
+        //2:绑定体验账号
+        boolean isbind = jobService.bindCreative(account, password, creativeName, ksId);
+        //3:转化
+        appiumTaskService.runTask(1, 1);
+    }
 }

+ 2 - 2
module-common/src/main/java/cn/com/ctop/common/module/entity/MaterialInfo.java

@@ -54,8 +54,8 @@ public class MaterialInfo {
     /**
      * userId
      */
-    @Excel(name = "status", width = 15)
-    @ApiModelProperty(value = "status")
+    @Excel(name = "user_id", width = 15)
+    @ApiModelProperty(value = "user_id")
     private String userId;
 
     /**

+ 5 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/controller/AppiumJobController.java

@@ -263,4 +263,9 @@ public class AppiumJobController {
         }
         return result;
     }
+
+    @PostMapping("startTask")
+    public Map<String, Object> startTask(String account, String password, String creativeName, Integer num) {
+        return appiumJobService.startTask(account, password, creativeName, num);
+    }
 }

+ 6 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/service/IAppiumJobService.java

@@ -3,6 +3,8 @@ package cn.com.ctop.crawler.modules.appium.service;
 import cn.com.ctop.crawler.modules.appium.entity.AppiumJob;
 import com.baomidou.mybatisplus.extension.service.IService;
 
+import java.util.Map;
+
 /**
  * 爬虫调度任务
  *
@@ -15,4 +17,8 @@ public interface IAppiumJobService extends IService<AppiumJob> {
     void runTask(Long jobId, Long num);
 
     String loginTask(Long taskId);
+
+    boolean bindCreative(String account, String password, String creativeName, String ksId);
+
+    Map<String, Object> startTask(String account, String password, String creativeName, Integer num);
 }

+ 170 - 3
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/service/impl/AppiumJobServiceImpl.java

@@ -1,23 +1,33 @@
 package cn.com.ctop.crawler.modules.appium.service.impl;
 
+import cn.com.ctop.common.module.utils.HttpUtils2;
 import cn.com.ctop.common.module.utils.ResultMapUtils;
 import cn.com.ctop.common.module.utils.StatusCode;
-import cn.com.ctop.crawler.modules.appium.entity.AppiumJob;
-import cn.com.ctop.crawler.modules.appium.entity.AppiumTask;
-import cn.com.ctop.crawler.modules.appium.entity.AppiumTaskItem;
+import cn.com.ctop.crawler.modules.appium.entity.*;
 import cn.com.ctop.crawler.modules.appium.mapper.AppiumJobMapper;
 import cn.com.ctop.crawler.modules.appium.mapper.AppiumTaskItemMapper;
+import cn.com.ctop.crawler.modules.appium.mapper.AppiumTaskLogMapper;
+import cn.com.ctop.crawler.modules.appium.service.IAppiumDeviceService;
 import cn.com.ctop.crawler.modules.appium.service.IAppiumJobService;
+import cn.com.ctop.crawler.modules.appium.service.IAppiumTaskService;
 import cn.com.ctop.crawler.modules.core.util.AppiumUtil;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import io.appium.java_client.android.AndroidDriver;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.http.impl.client.BasicCookieStore;
+import org.openqa.selenium.By;
+import org.openqa.selenium.WebDriver;
+import org.openqa.selenium.WebElement;
+import org.openqa.selenium.chrome.ChromeDriver;
+import org.openqa.selenium.chrome.ChromeOptions;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Stack;
 
 /**
  * 爬虫调度任务
@@ -33,6 +43,12 @@ public class AppiumJobServiceImpl extends ServiceImpl<AppiumJobMapper, AppiumJob
     private AppiumJobMapper appiumJobMapper;
     @Autowired
     private AppiumTaskItemMapper taskItemMapper;
+    @Value("${jeecg.path.chrome-driver}")
+    private String chromeDriver;
+    @Autowired
+    private IAppiumDeviceService appiumDeviceService;
+    @Autowired
+    private AppiumTaskLogMapper appiumTaskLogMapper;
 
     @Override
     public void runTask(Long jobId, Long num) {
@@ -76,4 +92,155 @@ public class AppiumJobServiceImpl extends ServiceImpl<AppiumJobMapper, AppiumJob
         System.out.println(kuaishouId);
         return kuaishouId;
     }
+
+    @Override
+    public boolean bindCreative(String account, String password, String creativeName, String ksId) {
+        boolean isBind = false;
+        String url = "https://ad.e.kuaishou.com/#/welcome?redirectUrl=https%3A%2F%2Fad.e.kuaishou.com%2F%23%2Findex";
+        System.getProperties().setProperty("webdriver.chrome.driver", chromeDriver);
+        ChromeOptions chromeOptions = new ChromeOptions();
+        chromeOptions.addArguments("--headless");
+        chromeOptions.addArguments("--incognito");
+        chromeOptions.addArguments("--disable-gpu");
+//        chromeOptions.addArguments("--no-sandbox");
+        chromeOptions.addArguments("--window-size=1920,1080");
+        chromeOptions.addArguments("--user-agent=" + HttpUtils2.USER_AGENT);
+        chromeOptions.setAcceptInsecureCerts(true);
+        WebDriver webDriver = new ChromeDriver(chromeOptions);
+        try {
+            Thread.sleep(3000L);
+            HttpUtils2.cookieStore = new BasicCookieStore();
+            webDriver.manage().deleteAllCookies();
+            //获取登录页面
+            webDriver.get(url);
+            Thread.sleep(3000L);
+            WebElement accountElement = webDriver.findElement(By.xpath("//div[@class='phone ']/input[@type='text']"));
+            accountElement.sendKeys(account);
+            Thread.sleep(3000L);
+            WebElement passwordElement = webDriver.findElement(By.xpath("//div[@class='password ']/input[@type='password']"));
+            passwordElement.sendKeys(password);
+            WebElement loginElement = webDriver.findElement(By.xpath("//div[@class='foot']"));
+            Thread.sleep(3000L);
+            //点击登录
+            loginElement.click();
+            Thread.sleep(3000L);
+            //获取推广按钮
+            WebElement spreadElement = webDriver.findElement(By.linkText("推广"));
+            spreadElement.click();
+            Thread.sleep(3000L);
+            //选择广告创意
+            WebElement creativeElement = webDriver.findElement(By.xpath("//div[text()='广告创意']"));
+            creativeElement.click();
+            Thread.sleep(3000L);
+            //输入创意名称,点击搜索
+            WebElement searchElement = webDriver.findElement(By.xpath("//input[@type='text']"));
+            searchElement.sendKeys(creativeName);
+            Thread.sleep(3000L);
+            WebElement searchButton = webDriver.findElement(By.xpath("//button[@class='ant-btn ant-input-search-button ant-btn-primary']"));
+            searchButton.click();
+            Thread.sleep(3000L);
+            WebElement tiyanElement = webDriver.findElement(By.linkText("体验"));
+            tiyanElement.click();
+            Thread.sleep(2000L);
+            WebElement inputKsIdElement = webDriver.findElement(By.xpath("//textarea[@placeholder='请输入快手账号…']"));
+            inputKsIdElement.sendKeys(ksId);
+            Thread.sleep(2000L);
+            WebElement chufaElement = webDriver.findElement(By.xpath("//button[@class='creative-experience-btn-enable']"));
+            chufaElement.click();
+            isBind = true;
+        } catch (Exception e) {
+            e.printStackTrace();
+            log.error("绑定体验快手账号失败");
+        } finally {
+            webDriver.manage().deleteAllCookies();
+            webDriver.close();
+            return isBind;
+        }
+    }
+
+    @Autowired
+    private IAppiumTaskService appiumTaskService;
+
+    @Override
+    public Map<String, Object> startTask(String account, String password, String creativeName, Integer num) {
+        Map<String, Object> result = new HashMap<>();
+        int i = 0;
+        while (i < num) {
+            if (i > num) {
+                break;
+            }
+            //获取手机设备信息
+            AppiumDevice appiumDevice = appiumDeviceService.getById(2L);
+            appiumDevice.setStatus(2);
+            appiumDeviceService.updateById(appiumDevice);
+            AppiumTaskLog log = new AppiumTaskLog();
+            log.setDeviceId(2);
+            log.setDeviceIp(appiumDevice.getIp());
+            log.setDevicePort(appiumDevice.getPort());
+            log.setStatus(1);
+            log.setTaskId(1);
+            log.setTaskName("快手刷量任务");
+            appiumTaskLogMapper.insert(log);
+            log.setId(log.getId());
+            try {
+                String ksId = loginTask(2L);
+                if (null == ksId) {
+                    updateStatus(appiumDevice, log, -1, 1);
+                    continue;
+                }
+                boolean isBind = bindCreative(account, password, creativeName, ksId);
+                if (!isBind) {
+                    updateStatus(appiumDevice, log, -2, 1);
+                    continue;
+                }
+                appiumTaskService.runTask(1, 1);
+                i++;
+            } catch (Exception e) {
+                updateStatus(appiumDevice, log, -3, 1);
+                continue;
+            }
+        }
+        return result;
+    }
+
+    private void updateStatus(AppiumDevice appiumDevice, AppiumTaskLog log, Integer logStatus, Integer deviceStatus) {
+        log.setStatus(3);
+        appiumTaskLogMapper.updateById(log);
+        appiumDevice.setStatus(1);
+        appiumDeviceService.updateById(appiumDevice);
+    }
+
+    public static boolean isValid(String s) {
+        if (null == s || s.length() <= 0) {
+            return true;
+        }
+        if (s.length() % 2 == 1) {
+            return false;
+        }
+        Stack<Character> stack = new Stack<>();
+        for (int i = 0; i < s.length(); i++) {
+            char getchar = s.charAt(i);
+            if (getchar == '(' || getchar == '{' || getchar == '[') {
+                stack.push(getchar);
+            }
+            if (getchar == ')' || getchar == '}' || getchar == ']') {
+                if (stack.empty()) {
+                    return false;
+                }
+                char sufix = stack.pop();
+                if ((getchar == ')' && sufix != '(') || (getchar == '}' && sufix != '{') || (getchar == ']' && sufix != '[')) {
+                    stack.push(sufix);
+                }
+            }
+        }
+        if (stack.empty()) {
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    public static void main(String[] args) {
+        System.out.println(isValid("[]()[]"));
+    }
 }

+ 9 - 1
module-crawler/src/main/java/cn/com/ctop/crawler/modules/core/util/AppiumUtil.java

@@ -741,7 +741,15 @@ public class AppiumUtil {
                             } else if (appiumTaskItem.getClickType().equals("point")) {
                                 Point point = element.getLocation();
                                 System.out.println(point.getX() + ":" + point.getY());
-                                tapPoint(androidDriver, point.getX() + appiumTaskItem.getOffsiteX(), point.getY() + appiumTaskItem.getOffsiteY());
+                                Integer x = point.getX();
+                                Integer y = point.getY();
+                                if (null != appiumTaskItem.getOffsiteX()) {
+                                    x += appiumTaskItem.getOffsiteX();
+                                }
+                                if (null != appiumTaskItem.getOffsiteY()) {
+                                    y += appiumTaskItem.getOffsiteY();
+                                }
+                                tapPoint(androidDriver, x, y);
                             } else if (appiumTaskItem.getClickType().equals("send")) {
                                 String text = appiumTaskItem.getTextEqualKey();
                                 element.sendKeys(text);