yumeng пре 5 година
родитељ
комит
8244e3e96b
85 измењених фајлова са 1177 додато и 2938 уклоњено
  1. 0 12
      jeecg-boot-module-system/pom.xml
  2. 3 8
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/CrawlerController.java
  3. 1 45
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/TestMqController.java
  4. 0 61
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/KuaishouAppListJob.java
  5. 0 4
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/service/ICrawlerService.java
  6. 0 93
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/service/impl/CrawlerServiceImpl.java
  7. 2 0
      module-common/src/main/java/cn/com/ctop/common/module/entity/BindAccountLogin.java
  8. 3 2
      module-common/src/main/java/cn/com/ctop/common/module/utils/ChaojiyingUtils.java
  9. 13 5
      module-common/src/main/java/cn/com/ctop/common/module/utils/HttpUtils2.java
  10. 6 0
      module-crawler/pom.xml
  11. 0 265
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/controller/AppiumJobBindTaskInfoController.java
  12. 0 247
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/controller/AppiumJobController.java
  13. 0 258
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/controller/AppiumTaskController.java
  14. 0 247
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/controller/AppiumTaskItemController.java
  15. 0 164
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/controller/AppiumTaskLogController.java
  16. 0 62
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/entity/AppiumDevice.java
  17. 0 57
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/entity/AppiumJob.java
  18. 0 63
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/entity/AppiumJobBindTaskInfo.java
  19. 0 66
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/entity/AppiumTask.java
  20. 0 152
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/entity/AppiumTaskItem.java
  21. 0 83
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/entity/AppiumTaskLog.java
  22. 0 17
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/mapper/AppiumDeviceMapper.java
  23. 0 15
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/mapper/AppiumJobBindTaskInfoMapper.java
  24. 0 21
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/mapper/AppiumJobMapper.java
  25. 0 18
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/mapper/AppiumTaskItemMapper.java
  26. 0 18
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/mapper/AppiumTaskLogMapper.java
  27. 0 17
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/mapper/AppiumTaskMapper.java
  28. 0 5
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/mapper/xml/AppiumDeviceMapper.xml
  29. 0 5
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/mapper/xml/AppiumJobBindTaskInfoMapper.xml
  30. 0 11
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/mapper/xml/AppiumJobMapper.xml
  31. 0 22
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/mapper/xml/AppiumTaskItemMapper.xml
  32. 0 5
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/mapper/xml/AppiumTaskLogMapper.xml
  33. 0 15
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/service/IAppiumDeviceService.java
  34. 0 15
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/service/IAppiumJobBindTaskInfoService.java
  35. 0 26
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/service/IAppiumJobService.java
  36. 0 23
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/service/IAppiumTaskItemService.java
  37. 0 15
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/service/IAppiumTaskLogService.java
  38. 0 22
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/service/IAppiumTaskService.java
  39. 0 26
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/service/impl/AppiumDeviceServiceImpl.java
  40. 0 19
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/service/impl/AppiumJobBindTaskInfoServiceImpl.java
  41. 0 259
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/service/impl/AppiumJobServiceImpl.java
  42. 0 41
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/service/impl/AppiumTaskItemServiceImpl.java
  43. 0 20
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/service/impl/AppiumTaskLogServiceImpl.java
  44. 0 90
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/service/impl/AppiumTaskServiceImpl.java
  45. 0 181
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/core/util/AppiumUtil.java
  46. 15 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/entity/PangolinApp.java
  47. 17 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/entity/PangolinChannel.java
  48. 21 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/entity/PangolinReportAccountDaily.java
  49. 20 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/entity/PangolinReportAppHour.java
  50. 22 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/entity/PangolinReportBillMonth.java
  51. 22 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/entity/PangolinReportChannelDaily.java
  52. 46 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/job/PangolinDataJob.java
  53. 32 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/job/PangolinLoginJob.java
  54. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/PangolinAppMapper.java
  55. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/PangolinChannelMapper.java
  56. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/PangolinReportAccountDailyMapper.java
  57. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/PangolinReportAppHourMapper.java
  58. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/PangolinReportBillMonthMapper.java
  59. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/PangolinReportChannelDailyMapper.java
  60. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/PangolinAppService.java
  61. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/PangolinChannelService.java
  62. 11 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/PangolinCrawlerService.java
  63. 5 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/PangolinLoginService.java
  64. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/PangolinReportAccountDailyService.java
  65. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/PangolinReportAppHourService.java
  66. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/PangolinReportBillMonthService.java
  67. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/PangolinReportChannelDailyService.java
  68. 13 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinAppServiceImpl.java
  69. 11 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinChannelServiceImpl.java
  70. 353 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinCrawlerServiceImpl.java
  71. 118 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinLoginServiceImpl.java
  72. 11 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinReportAccountDailyServiceImpl.java
  73. 11 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinReportAppHourServiceImpl.java
  74. 11 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinReportBillMonthServiceImpl.java
  75. 11 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinReportChannelDailyServiceImpl.java
  76. 24 16
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/controller/BatchController.java
  77. 76 77
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/controller/AppiumDeviceController.java
  78. 113 0
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/entity/KuaishouPopulation.java
  79. 15 0
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/mapper/KuaishouPopulationMapper.java
  80. 1 1
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/mapper/xml/AppiumTaskMapper.xml
  81. 1 1
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/IKuaishouInterfaceService.java
  82. 15 0
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/IKuaishouPopulationService.java
  83. 19 26
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/impl/BatchServiceImpl.java
  84. 32 17
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/impl/KuaishouInterfaceServiceImpl.java
  85. 19 0
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/impl/KuaishouPopulationServiceImpl.java

+ 0 - 12
jeecg-boot-module-system/pom.xml

@@ -168,18 +168,6 @@
             <artifactId>ureport2-core</artifactId>
             <version>2.2.9</version>
         </dependency>
-        <!-- https://mvnrepository.com/artifact/org.apache.poi/poi-ooxml -->
-       <!-- <dependency>
-            <groupId>org.apache.poi</groupId>
-            <artifactId>poi-ooxml</artifactId>
-            <version>3.17</version>
-        </dependency>-->
-        <!--<dependency>
-            <groupId>com.test</groupId>
-            <artifactId>opencv</artifactId>
-            <scope>system</scope>
-            <systemPath>${basedir}/docs/opencv-420.jar</systemPath>
-        </dependency>-->
         <dependency>
             <groupId>cn.com.ctop</groupId>
             <artifactId>opencv</artifactId>

+ 3 - 8
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/CrawlerController.java

@@ -1,9 +1,10 @@
 package org.jeecg.modules.ctop.controller;
 
 import org.jeecg.modules.ctop.service.ICrawlerService;
-import org.jeecg.modules.ctop.vo.KuaishouAppiumJobVO;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.web.bind.annotation.*;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
 
 import java.util.Map;
 
@@ -12,12 +13,6 @@ import java.util.Map;
 public class CrawlerController {
     @Autowired
     private ICrawlerService crawlerService;
-
-    @PostMapping("startJob")
-    public Map<String, Object> runTask(@RequestBody KuaishouAppiumJobVO jobVO) {
-        return crawlerService.startJob(jobVO);
-    }
-
     @GetMapping("loadInspiration")
     public Map<String, Object> loadInspiration() {
         return crawlerService.loadInspiration();

+ 1 - 45
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/TestMqController.java

@@ -2,21 +2,12 @@ package org.jeecg.modules.ctop.controller;
 
 import cn.com.ctop.common.module.utils.ResultMapUtils;
 import cn.com.ctop.common.module.utils.StatusCode;
-import cn.com.ctop.crawler.modules.appium.entity.AppiumDevice;
-import cn.com.ctop.crawler.modules.appium.service.IAppiumDeviceService;
-import cn.com.ctop.crawler.modules.appium.service.IAppiumJobService;
-import cn.com.ctop.crawler.modules.appium.service.IAppiumTaskService;
 import org.jeecg.modules.mq.Sender;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
 
-import javax.servlet.http.HttpServletResponse;
-import java.io.BufferedInputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.OutputStream;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -28,13 +19,8 @@ import java.util.Map;
 @RequestMapping("mq")
 public class TestMqController {
     @Autowired
-    private IAppiumDeviceService deviceService;
-    @Autowired
     private Sender sender;
-    @Autowired
-    private IAppiumJobService jobService;
-    @Autowired
-    private IAppiumTaskService appiumTaskService;
+
 
     @GetMapping("hello")
     public Map<String, Object> sayHello() {
@@ -43,34 +29,4 @@ public class TestMqController {
         ResultMapUtils.setResultMap(result, StatusCode.COMMON_SUCCESS.getCode());
         return result;
     }
-
-    @GetMapping("task1")
-    public Map<String, Object> task1() {
-        Map<String, Object> result = new HashMap<>();
-        AppiumDevice device = deviceService.getById(1);
-        jobService.loginTask(2L, device);
-        ResultMapUtils.setResultMap(result, StatusCode.COMMON_SUCCESS.getCode());
-        return result;
-    }
-
-    @GetMapping("task33")
-    public void task1(HttpServletResponse response) throws Exception {
-        File f = new File("C:\\Users\\宋英豪\\Desktop\\OKR\\2.png");
-        if (!f.exists()) {
-            return;
-        }
-        BufferedInputStream br = new BufferedInputStream(new FileInputStream(f));
-        byte[] buf = new byte[1024];
-        int len = 0;
-        response.reset(); // 非常重要
-        // 纯下载方式
-        response.setContentType("application/x-msdownload");
-        response.setHeader("Content-Disposition", "attachment; filename=" + f.getName());
-        OutputStream out = response.getOutputStream();
-        while ((len = br.read(buf)) > 0) {
-            out.write(buf, 0, len);
-        }
-        br.close();
-        out.close();
-    }
 }

+ 0 - 61
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/KuaishouAppListJob.java

@@ -1,61 +0,0 @@
-package org.jeecg.modules.ctop.job;
-
-import cn.com.ctop.common.module.entity.CtopOauthToken;
-import cn.com.ctop.common.module.service.ICtopOauthTokenService;
-import cn.com.ctop.common.module.utils.Check;
-import cn.com.ctop.kuaishou.modules.batch.service.IKuaishouInterfaceService;
-import lombok.extern.slf4j.Slf4j;
-import org.quartz.Job;
-import org.quartz.JobExecutionContext;
-import org.quartz.JobExecutionException;
-import org.springframework.beans.factory.annotation.Autowired;
-
-import java.util.List;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-/**
- * 获取广告计划分时报表
- */
-@Slf4j
-public class KuaishouAppListJob implements Job {
-    @Autowired
-    private IKuaishouInterfaceService kuaishouInterfaceService;
-
-    @Autowired
-    private ICtopOauthTokenService oauthTokenService;
-
-
-    static ExecutorService executorService = Executors.newFixedThreadPool(2);
-
-    @Override
-    public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
-
-        Thread thread = new Thread() {
-            @Override
-            public void run() {
-                List<CtopOauthToken> tokens = oauthTokenService.selectKuaiShouToken();
-                for (CtopOauthToken token : tokens) {
-                    if (Check.isNull(token)) {
-                        continue;
-                    }
-                    executorService.submit(new Runnable() {
-                        @Override
-                        public void run() {
-                            try {
-                                kuaishouInterfaceService.getAppList(token.getAccountId(), token.getAccessToken());
-                            } catch (Exception e) {
-                                e.printStackTrace();
-                            }
-                        }
-                    });
-                }
-
-            }
-
-        };
-        thread.start();
-
-
-    }
-}

+ 0 - 4
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/service/ICrawlerService.java

@@ -1,11 +1,7 @@
 package org.jeecg.modules.ctop.service;
 
-import org.jeecg.modules.ctop.vo.KuaishouAppiumJobVO;
-
 import java.util.Map;
 
 public interface ICrawlerService {
-    Map<String, Object> startJob(KuaishouAppiumJobVO jobVO);
-
     Map<String, Object> loadInspiration();
 }

+ 0 - 93
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/service/impl/CrawlerServiceImpl.java

@@ -1,16 +1,7 @@
 package org.jeecg.modules.ctop.service.impl;
 
-import cn.com.ctop.common.module.entity.BindAccountLogin;
 import cn.com.ctop.common.module.service.IBindAccountLoginService;
 import cn.com.ctop.common.module.utils.HttpUtils;
-import cn.com.ctop.common.module.utils.ResultMapUtils;
-import cn.com.ctop.common.module.utils.StatusCode;
-import cn.com.ctop.crawler.modules.appium.entity.AppiumDevice;
-import cn.com.ctop.crawler.modules.appium.entity.AppiumTaskLog;
-import cn.com.ctop.crawler.modules.appium.mapper.AppiumTaskLogMapper;
-import cn.com.ctop.crawler.modules.appium.service.IAppiumDeviceService;
-import cn.com.ctop.crawler.modules.appium.service.IAppiumJobService;
-import cn.com.ctop.crawler.modules.appium.service.IAppiumTaskService;
 import cn.com.ctop.crawler.modules.kuaishoucce.entity.KuaishouEffectAdInfo;
 import cn.com.ctop.crawler.modules.kuaishoucce.entity.KuaishouHotPhotoInfo;
 import cn.com.ctop.crawler.modules.kuaishoucce.entity.KuaishouInspiredAd;
@@ -21,7 +12,6 @@ import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import lombok.extern.slf4j.Slf4j;
 import org.jeecg.modules.ctop.service.ICrawlerService;
-import org.jeecg.modules.ctop.vo.KuaishouAppiumJobVO;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
@@ -30,7 +20,6 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
 
 @Service
 @Slf4j
@@ -38,88 +27,6 @@ public class CrawlerServiceImpl implements ICrawlerService {
     static ExecutorService executorService = null;
     @Autowired
     private IBindAccountLoginService bindAccountLoginService;
-    @Autowired
-    private IAppiumJobService jobService;
-    @Autowired
-    private IAppiumTaskService taskService;
-    @Autowired
-    private IAppiumDeviceService deviceService;
-    @Autowired
-    private AppiumTaskLogMapper taskLogMapper;
-
-    @Override
-    public Map<String, Object> startJob(KuaishouAppiumJobVO jobVO) {
-        Map<String, Object> result = new HashMap<>();
-        //1:根据id查询快手账号信息
-        BindAccountLogin accountLogin = bindAccountLoginService.getById(jobVO.getAccountId());
-        if (null == accountLogin) {
-            ResultMapUtils.setResultMap(result, StatusCode.COMMON_PARAM_ERROR.getCode());
-            return result;
-        }
-        if (jobVO.getJobId() == null || jobVO.getJobId() == 0 || jobVO.getNum() == null || jobVO.getNum() == 0 || jobVO.getCreativeName() == null || "".equals(jobVO.getCreativeName()) || jobVO.getPublishName() == null || "".equals(jobVO.getPublishName().trim())) {
-            ResultMapUtils.setResultMap(result, StatusCode.COMMON_PARAM_ERROR.getCode());
-            return result;
-        }
-        Long num = jobVO.getNum();
-        //获取手机设备信息
-        AppiumDevice device = deviceService.getEnabled();
-        if (null == device || device.getStatus() != 1) {
-            ResultMapUtils.setResultMap(result, StatusCode.KUAISHOU_CRAWLER_APP_NO_ENABLED_DEVICE.getCode());
-            return result;
-        }
-        executorService = Executors.newFixedThreadPool(2);
-        executorService.submit(new Runnable() {
-            @Override
-            public void run() {
-                int i = 0;
-                while (i < num) {
-                    if (i > num) {
-                        break;
-                    }
-                    deviceService.updateById(device);
-                    AppiumTaskLog log = new AppiumTaskLog();
-                    log.setDeviceId(device.getId());
-                    log.setDeviceIp(device.getIp());
-                    log.setDevicePort(device.getPort());
-                    log.setStatus(1 + "");
-                    log.setTaskId(jobVO.getJobId());
-                    log.setTaskName("快手刷量任务");
-                    taskLogMapper.insert(log);
-                    log.setId(log.getId());
-                    try {
-                        //一键新机
-                        boolean success = jobService.refreshWebPhone();
-                        if (!success) {
-                            jobService.updateStatus(device, log, -4, 1);
-                            continue;
-                        }
-                        Thread.sleep(10000L);
-                        String ksId = jobService.loginTask(2L, device);
-                        if (null == ksId) {
-                            i++;
-                            jobService.updateStatus(device, log, -1, 1);
-                            continue;
-                        }
-                        boolean isBind = jobService.bindCreative(accountLogin.getAccountName(), accountLogin.getPassword(), jobVO.getPublishName(), ksId);
-                        if (!isBind) {
-                            i++;
-                            jobService.updateStatus(device, log, -2, 1);
-                            continue;
-                        }
-                        taskService.runTask(1L, device.getId(), jobVO.getPublishName());
-                        jobService.updateStatus(device, log, 2, 1);
-                        i++;
-                    } catch (Exception e) {
-                        jobService.updateStatus(device, log, -3, 1);
-                        i++;
-                        continue;
-                    }
-                }
-            }
-        });
-        ResultMapUtils.setResultMap(result, StatusCode.KUAISHOU_CRAWLER_APP_JOB_StART.getCode());
-        return result;
-    }
 
     @Override
     public Map<String, Object> loadInspiration() {

+ 2 - 0
module-common/src/main/java/cn/com/ctop/common/module/entity/BindAccountLogin.java

@@ -99,4 +99,6 @@ public class BindAccountLogin {
     @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss")
     @ApiModelProperty(value = "修改时间")
     private Date updateTime;
+
+    private String cookie;
 }

+ 3 - 2
module-common/src/main/java/cn/com/ctop/common/module/utils/ChaojiyingUtils.java

@@ -10,12 +10,13 @@ public class ChaojiyingUtils {
     private static String password = "heaven";
     private static Integer softId = 901055;
 
-    public static String process(String imageBase64) {
+    public static String process(String imageBase64,int code) {
         Map<String, Object> params = new HashMap<String, Object>();
         params.put("user", username);
         params.put("pass", password);
         params.put("softid", softId);
-        params.put("codetype", 9005);
+//        params.put("codetype", 9005);
+        params.put("codetype", code);
         params.put("file_base64", imageBase64);
         Map<String, String> headers = new HashMap<String, String>();
         headers.put("Content-Type", "application/json");

+ 13 - 5
module-common/src/main/java/cn/com/ctop/common/module/utils/HttpUtils2.java

@@ -144,9 +144,12 @@ public class HttpUtils2 {
             HttpPost httppost = new HttpPost(url);
             httppost.setHeader("User-Agent", USER_AGENT);
             httppost.addHeader("Content-Type", "application/json");
-            for (String key : headers.keySet()) {
-                httppost.setHeader(key, headers.get(key));
+            if(headers != null){
+                for (String key : headers.keySet()) {
+                    httppost.setHeader(key, headers.get(key));
+                }
             }
+
             httppost.setEntity(new StringEntity(new Gson().toJson(param), Charset.forName("UTF-8")));
             HttpEntity respentity;
 
@@ -237,10 +240,15 @@ public class HttpUtils2 {
             HttpPost httppost = new HttpPost(url);
             httppost.setHeader("User-Agent", USER_AGENT);
             httppost.addHeader("Content-Type", "application/json");
-            for (String key : headers.keySet()) {
-                httppost.setHeader(key, headers.get(key));
+            if(headers != null){
+                for (String key : headers.keySet()) {
+                    httppost.setHeader(key, headers.get(key));
+                }
+            }
+            if(params != null){
+                httppost.setEntity(new StringEntity(params.toJSONString(), Charset.forName("UTF-8")));
             }
-            httppost.setEntity(new StringEntity(params.toJSONString(), Charset.forName("UTF-8")));
+
             HttpEntity respentity;
 
             HttpResponse response = httpClient.execute(httppost);

+ 6 - 0
module-crawler/pom.xml

@@ -22,6 +22,12 @@
             <artifactId>jeecg-boot-base-common</artifactId>
             <version>2.0.2</version>
         </dependency>
+        <dependency>
+            <groupId>org.codehaus.jackson</groupId>
+            <artifactId>jackson-core-asl</artifactId>
+            <version>1.9.11</version>
+            <scope>compile</scope>
+        </dependency>
     </dependencies>
     <build>
         <resources>

+ 0 - 265
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/controller/AppiumJobBindTaskInfoController.java

@@ -1,265 +0,0 @@
-package cn.com.ctop.crawler.modules.appium.controller;
-
-import cn.com.ctop.common.module.utils.ResultMapUtils;
-import cn.com.ctop.common.module.utils.StatusCode;
-import cn.com.ctop.crawler.modules.appium.entity.AppiumJobBindTaskInfo;
-import cn.com.ctop.crawler.modules.appium.service.IAppiumJobBindTaskInfoService;
-import com.alibaba.fastjson.JSON;
-import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
-import com.baomidou.mybatisplus.core.metadata.IPage;
-import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiOperation;
-import lombok.extern.slf4j.Slf4j;
-import org.jeecg.common.api.vo.Result;
-import org.jeecg.common.aspect.annotation.AutoLog;
-import org.jeecg.common.system.query.QueryGenerator;
-import org.jeecg.common.util.oConvertUtils;
-import org.jeecgframework.poi.excel.ExcelImportUtil;
-import org.jeecgframework.poi.excel.def.NormalExcelConstants;
-import org.jeecgframework.poi.excel.entity.ExportParams;
-import org.jeecgframework.poi.excel.entity.ImportParams;
-import org.jeecgframework.poi.excel.view.JeecgEntityExcelView;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.web.bind.annotation.*;
-import org.springframework.web.multipart.MultipartFile;
-import org.springframework.web.multipart.MultipartHttpServletRequest;
-import org.springframework.web.servlet.ModelAndView;
-
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.net.URLDecoder;
-import java.util.*;
-
-/**
- * 爬虫调度任务绑定信息
- *
- * @author jeecg-boot
- * @version V1.0
- * @date 2019-11-14
- */
-@Slf4j
-@Api(tags = "爬虫调度任务绑定信息")
-@RestController
-@RequestMapping("/ctop/appiumJobBindTaskInfo")
-public class AppiumJobBindTaskInfoController {
-    @Autowired
-    private IAppiumJobBindTaskInfoService appiumJobBindTaskInfoService;
-
-    /**
-     * 分页列表查询
-     *
-     * @param appiumJobBindTaskInfo
-     * @param pageNo
-     * @param pageSize
-     * @param req
-     * @return
-     */
-    @AutoLog(value = "爬虫调度任务绑定信息-分页列表查询")
-    @ApiOperation(value = "爬虫调度任务绑定信息-分页列表查询", notes = "爬虫调度任务绑定信息-分页列表查询")
-    @GetMapping(value = "/list")
-    public Result<IPage<AppiumJobBindTaskInfo>> queryPageList(AppiumJobBindTaskInfo appiumJobBindTaskInfo,
-                                                              @RequestParam(name = "pageNo", defaultValue = "1") Integer pageNo,
-                                                              @RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize,
-                                                              HttpServletRequest req) {
-        Result<IPage<AppiumJobBindTaskInfo>> result = new Result<>();
-        QueryWrapper<AppiumJobBindTaskInfo> queryWrapper = QueryGenerator.initQueryWrapper(appiumJobBindTaskInfo, req.getParameterMap());
-        Page<AppiumJobBindTaskInfo> page = new Page<>(pageNo, pageSize);
-        IPage<AppiumJobBindTaskInfo> pageList = appiumJobBindTaskInfoService.page(page, queryWrapper);
-        result.setSuccess(true);
-        result.setResult(pageList);
-        return result;
-    }
-
-    /**
-     * 添加
-     *
-     * @param appiumJobBindTaskInfo
-     * @return
-     */
-    @AutoLog(value = "爬虫调度任务绑定信息-添加")
-    @ApiOperation(value = "爬虫调度任务绑定信息-添加", notes = "爬虫调度任务绑定信息-添加")
-    @PostMapping(value = "/add")
-    public Result<AppiumJobBindTaskInfo> add(@RequestBody AppiumJobBindTaskInfo appiumJobBindTaskInfo) {
-        Result<AppiumJobBindTaskInfo> result = new Result<>();
-        try {
-            appiumJobBindTaskInfoService.save(appiumJobBindTaskInfo);
-            result.success("添加成功!");
-        } catch (Exception e) {
-            log.error(e.getMessage(), e);
-            result.error500("操作失败");
-        }
-        return result;
-    }
-
-    /**
-     * 编辑
-     *
-     * @param appiumJobBindTaskInfo
-     * @return
-     */
-    @AutoLog(value = "爬虫调度任务绑定信息-编辑")
-    @ApiOperation(value = "爬虫调度任务绑定信息-编辑", notes = "爬虫调度任务绑定信息-编辑")
-    @PutMapping(value = "/edit")
-    public Result<AppiumJobBindTaskInfo> edit(@RequestBody AppiumJobBindTaskInfo appiumJobBindTaskInfo) {
-        Result<AppiumJobBindTaskInfo> result = new Result<>();
-        AppiumJobBindTaskInfo appiumJobBindTaskInfoEntity = appiumJobBindTaskInfoService.getById(appiumJobBindTaskInfo.getId());
-        if (appiumJobBindTaskInfoEntity == null) {
-            result.error500("未找到对应实体");
-        } else {
-            boolean ok = appiumJobBindTaskInfoService.updateById(appiumJobBindTaskInfo);
-            if (ok) {
-                result.success("修改成功!");
-            }
-        }
-
-        return result;
-    }
-
-    /**
-     * 通过id删除
-     *
-     * @param id
-     * @return
-     */
-    @AutoLog(value = "爬虫调度任务绑定信息-通过id删除")
-    @ApiOperation(value = "爬虫调度任务绑定信息-通过id删除", notes = "爬虫调度任务绑定信息-通过id删除")
-    @DeleteMapping(value = "/delete")
-    public Result<?> delete(@RequestParam(name = "id", required = true) String id) {
-        try {
-            appiumJobBindTaskInfoService.removeById(id);
-        } catch (Exception e) {
-            log.error("删除失败", e.getMessage());
-            return Result.error("删除失败!");
-        }
-        return Result.ok("删除成功!");
-    }
-
-    /**
-     * 批量删除
-     *
-     * @param ids
-     * @return
-     */
-    @AutoLog(value = "爬虫调度任务绑定信息-批量删除")
-    @ApiOperation(value = "爬虫调度任务绑定信息-批量删除", notes = "爬虫调度任务绑定信息-批量删除")
-    @DeleteMapping(value = "/deleteBatch")
-    public Result<AppiumJobBindTaskInfo> deleteBatch(@RequestParam(name = "ids", required = true) String ids) {
-        Result<AppiumJobBindTaskInfo> result = new Result<>();
-        if (ids == null || "".equals(ids.trim())) {
-            result.error500("参数不识别!");
-        } else {
-            this.appiumJobBindTaskInfoService.removeByIds(Arrays.asList(ids.split(",")));
-            result.success("删除成功!");
-        }
-        return result;
-    }
-
-    /**
-     * 通过id查询
-     *
-     * @param id
-     * @return
-     */
-    @AutoLog(value = "爬虫调度任务绑定信息-通过id查询")
-    @ApiOperation(value = "爬虫调度任务绑定信息-通过id查询", notes = "爬虫调度任务绑定信息-通过id查询")
-    @GetMapping(value = "/queryById")
-    public Result<AppiumJobBindTaskInfo> queryById(@RequestParam(name = "id", required = true) String id) {
-        Result<AppiumJobBindTaskInfo> result = new Result<>();
-        AppiumJobBindTaskInfo appiumJobBindTaskInfo = appiumJobBindTaskInfoService.getById(id);
-        if (appiumJobBindTaskInfo == null) {
-            result.error500("未找到对应实体");
-        } else {
-            result.setResult(appiumJobBindTaskInfo);
-            result.setSuccess(true);
-        }
-        return result;
-    }
-
-    /**
-     * 导出excel
-     *
-     * @param request
-     * @param response
-     */
-    @RequestMapping(value = "/exportXls")
-    public ModelAndView exportXls(HttpServletRequest request, HttpServletResponse response) {
-        // Step.1 组装查询条件
-        QueryWrapper<AppiumJobBindTaskInfo> queryWrapper = null;
-        try {
-            String paramsStr = request.getParameter("paramsStr");
-            if (oConvertUtils.isNotEmpty(paramsStr)) {
-                String deString = URLDecoder.decode(paramsStr, "UTF-8");
-                AppiumJobBindTaskInfo appiumJobBindTaskInfo = JSON.parseObject(deString, AppiumJobBindTaskInfo.class);
-                queryWrapper = QueryGenerator.initQueryWrapper(appiumJobBindTaskInfo, request.getParameterMap());
-            }
-        } catch (UnsupportedEncodingException e) {
-            e.printStackTrace();
-        }
-
-        //Step.2 AutoPoi 导出Excel
-        ModelAndView mv = new ModelAndView(new JeecgEntityExcelView());
-        List<AppiumJobBindTaskInfo> pageList = appiumJobBindTaskInfoService.list(queryWrapper);
-        //导出文件名称
-        mv.addObject(NormalExcelConstants.FILE_NAME, "爬虫调度任务绑定信息列表");
-        mv.addObject(NormalExcelConstants.CLASS, AppiumJobBindTaskInfo.class);
-        mv.addObject(NormalExcelConstants.PARAMS, new ExportParams("爬虫调度任务绑定信息列表数据", "导出人:Jeecg", "导出信息"));
-        mv.addObject(NormalExcelConstants.DATA_LIST, pageList);
-        return mv;
-    }
-
-    /**
-     * 通过excel导入数据
-     *
-     * @param request
-     * @param response
-     * @return
-     */
-    @RequestMapping(value = "/importExcel", method = RequestMethod.POST)
-    public Result<?> importExcel(HttpServletRequest request, HttpServletResponse response) {
-        MultipartHttpServletRequest multipartRequest = (MultipartHttpServletRequest) request;
-        Map<String, MultipartFile> fileMap = multipartRequest.getFileMap();
-        for (Map.Entry<String, MultipartFile> entity : fileMap.entrySet()) {
-            MultipartFile file = entity.getValue();
-            ImportParams params = new ImportParams();
-            params.setTitleRows(2);
-            params.setHeadRows(1);
-            params.setNeedSave(true);
-            try {
-                List<AppiumJobBindTaskInfo> listAppiumJobBindTaskInfos = ExcelImportUtil.importExcel(file.getInputStream(), AppiumJobBindTaskInfo.class, params);
-                appiumJobBindTaskInfoService.saveBatch(listAppiumJobBindTaskInfos);
-                return Result.ok("文件导入成功!数据行数:" + listAppiumJobBindTaskInfos.size());
-            } catch (Exception e) {
-                log.error(e.getMessage(), e);
-                return Result.error("文件导入失败:" + e.getMessage());
-            } finally {
-                try {
-                    file.getInputStream().close();
-                } catch (IOException e) {
-                    e.printStackTrace();
-                }
-            }
-        }
-        return Result.ok("文件导入失败!");
-    }
-
-    @PostMapping("bindInfo")
-    public Map<String, Object> bindInfo(Long jobId, Long taskId) {
-        Map<String, Object> result = new HashMap<>();
-        if (null == jobId || jobId == 0 || null == taskId || taskId == 0) {
-            ResultMapUtils.setResultMap(result, StatusCode.COMMON_PARAM_ERROR.getCode());
-            return result;
-        }
-        AppiumJobBindTaskInfo taskInfo = new AppiumJobBindTaskInfo();
-        taskInfo.setJobId(jobId);
-        taskInfo.setTaskId(taskId);
-        taskInfo.setStatus(1);
-        Date now = new Date();
-        taskInfo.setCreateTime(now);
-        taskInfo.setUpdateTime(now);
-        appiumJobBindTaskInfoService.save(taskInfo);
-        return result;
-    }
-}

+ 0 - 247
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/controller/AppiumJobController.java

@@ -1,247 +0,0 @@
-package cn.com.ctop.crawler.modules.appium.controller;
-
-import cn.com.ctop.crawler.modules.appium.entity.AppiumJob;
-import cn.com.ctop.crawler.modules.appium.service.IAppiumJobService;
-import com.alibaba.fastjson.JSON;
-import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
-import com.baomidou.mybatisplus.core.metadata.IPage;
-import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiOperation;
-import lombok.extern.slf4j.Slf4j;
-import org.jeecg.common.api.vo.Result;
-import org.jeecg.common.aspect.annotation.AutoLog;
-import org.jeecg.common.system.query.QueryGenerator;
-import org.jeecg.common.util.oConvertUtils;
-import org.jeecgframework.poi.excel.ExcelImportUtil;
-import org.jeecgframework.poi.excel.def.NormalExcelConstants;
-import org.jeecgframework.poi.excel.entity.ExportParams;
-import org.jeecgframework.poi.excel.entity.ImportParams;
-import org.jeecgframework.poi.excel.view.JeecgEntityExcelView;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.web.bind.annotation.*;
-import org.springframework.web.multipart.MultipartFile;
-import org.springframework.web.multipart.MultipartHttpServletRequest;
-import org.springframework.web.servlet.ModelAndView;
-
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.net.URLDecoder;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
-/**
- * 爬虫调度任务
- *
- * @author jeecg-boot
- * @version V1.0
- * @date 2019-11-14
- */
-@Slf4j
-@Api(tags = "爬虫调度任务")
-@RestController
-@RequestMapping("/ctop/appiumJob")
-public class AppiumJobController {
-    @Autowired
-    private IAppiumJobService appiumJobService;
-
-    /**
-     * 分页列表查询
-     *
-     * @param appiumJob
-     * @param pageNo
-     * @param pageSize
-     * @param req
-     * @return
-     */
-    @AutoLog(value = "爬虫调度任务-分页列表查询")
-    @ApiOperation(value = "爬虫调度任务-分页列表查询", notes = "爬虫调度任务-分页列表查询")
-    @GetMapping(value = "/list")
-    public Result<IPage<AppiumJob>> queryPageList(AppiumJob appiumJob,
-                                                  @RequestParam(name = "pageNo", defaultValue = "1") Integer pageNo,
-                                                  @RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize,
-                                                  HttpServletRequest req) {
-        Result<IPage<AppiumJob>> result = new Result<>();
-        QueryWrapper<AppiumJob> queryWrapper = QueryGenerator.initQueryWrapper(appiumJob, req.getParameterMap());
-        Page<AppiumJob> page = new Page<>(pageNo, pageSize);
-        IPage<AppiumJob> pageList = appiumJobService.page(page, queryWrapper);
-        result.setSuccess(true);
-        result.setResult(pageList);
-        return result;
-    }
-
-    /**
-     * 添加
-     *
-     * @param appiumJob
-     * @return
-     */
-    @AutoLog(value = "爬虫调度任务-添加")
-    @ApiOperation(value = "爬虫调度任务-添加", notes = "爬虫调度任务-添加")
-    @PostMapping(value = "/add")
-    public Result<AppiumJob> add(@RequestBody AppiumJob appiumJob) {
-        Result<AppiumJob> result = new Result<>();
-        try {
-            appiumJobService.save(appiumJob);
-            result.success("添加成功!");
-        } catch (Exception e) {
-            log.error(e.getMessage(), e);
-            result.error500("操作失败");
-        }
-        return result;
-    }
-
-    /**
-     * 编辑
-     *
-     * @param appiumJob
-     * @return
-     */
-    @AutoLog(value = "爬虫调度任务-编辑")
-    @ApiOperation(value = "爬虫调度任务-编辑", notes = "爬虫调度任务-编辑")
-    @PutMapping(value = "/edit")
-    public Result<AppiumJob> edit(@RequestBody AppiumJob appiumJob) {
-        Result<AppiumJob> result = new Result<>();
-        AppiumJob appiumJobEntity = appiumJobService.getById(appiumJob.getId());
-        if (appiumJobEntity == null) {
-            result.error500("未找到对应实体");
-        } else {
-            boolean ok = appiumJobService.updateById(appiumJob);
-            if (ok) {
-                result.success("修改成功!");
-            }
-        }
-
-        return result;
-    }
-
-    /**
-     * 通过id删除
-     *
-     * @param id
-     * @return
-     */
-    @AutoLog(value = "爬虫调度任务-通过id删除")
-    @ApiOperation(value = "爬虫调度任务-通过id删除", notes = "爬虫调度任务-通过id删除")
-    @DeleteMapping(value = "/delete")
-    public Result<?> delete(@RequestParam(name = "id", required = true) String id) {
-        try {
-            appiumJobService.removeById(id);
-        } catch (Exception e) {
-            log.error("删除失败", e.getMessage());
-            return Result.error("删除失败!");
-        }
-        return Result.ok("删除成功!");
-    }
-
-    /**
-     * 批量删除
-     *
-     * @param ids
-     * @return
-     */
-    @AutoLog(value = "爬虫调度任务-批量删除")
-    @ApiOperation(value = "爬虫调度任务-批量删除", notes = "爬虫调度任务-批量删除")
-    @DeleteMapping(value = "/deleteBatch")
-    public Result<AppiumJob> deleteBatch(@RequestParam(name = "ids", required = true) String ids) {
-        Result<AppiumJob> result = new Result<>();
-        if (ids == null || "".equals(ids.trim())) {
-            result.error500("参数不识别!");
-        } else {
-            this.appiumJobService.removeByIds(Arrays.asList(ids.split(",")));
-            result.success("删除成功!");
-        }
-        return result;
-    }
-
-    /**
-     * 通过id查询
-     *
-     * @param id
-     * @return
-     */
-    @AutoLog(value = "爬虫调度任务-通过id查询")
-    @ApiOperation(value = "爬虫调度任务-通过id查询", notes = "爬虫调度任务-通过id查询")
-    @GetMapping(value = "/queryById")
-    public Result<AppiumJob> queryById(@RequestParam(name = "id", required = true) String id) {
-        Result<AppiumJob> result = new Result<>();
-        AppiumJob appiumJob = appiumJobService.getById(id);
-        if (appiumJob == null) {
-            result.error500("未找到对应实体");
-        } else {
-            result.setResult(appiumJob);
-            result.setSuccess(true);
-        }
-        return result;
-    }
-
-    /**
-     * 导出excel
-     *
-     * @param request
-     * @param response
-     */
-    @RequestMapping(value = "/exportXls")
-    public ModelAndView exportXls(HttpServletRequest request, HttpServletResponse response) {
-        // Step.1 组装查询条件
-        QueryWrapper<AppiumJob> queryWrapper = null;
-        try {
-            String paramsStr = request.getParameter("paramsStr");
-            if (oConvertUtils.isNotEmpty(paramsStr)) {
-                String deString = URLDecoder.decode(paramsStr, "UTF-8");
-                AppiumJob appiumJob = JSON.parseObject(deString, AppiumJob.class);
-                queryWrapper = QueryGenerator.initQueryWrapper(appiumJob, request.getParameterMap());
-            }
-        } catch (UnsupportedEncodingException e) {
-            e.printStackTrace();
-        }
-
-        //Step.2 AutoPoi 导出Excel
-        ModelAndView mv = new ModelAndView(new JeecgEntityExcelView());
-        List<AppiumJob> pageList = appiumJobService.list(queryWrapper);
-        //导出文件名称
-        mv.addObject(NormalExcelConstants.FILE_NAME, "爬虫调度任务列表");
-        mv.addObject(NormalExcelConstants.CLASS, AppiumJob.class);
-        mv.addObject(NormalExcelConstants.PARAMS, new ExportParams("爬虫调度任务列表数据", "导出人:Jeecg", "导出信息"));
-        mv.addObject(NormalExcelConstants.DATA_LIST, pageList);
-        return mv;
-    }
-
-    /**
-     * 通过excel导入数据
-     *
-     * @param request
-     * @param response
-     * @return
-     */
-    @RequestMapping(value = "/importExcel", method = RequestMethod.POST)
-    public Result<?> importExcel(HttpServletRequest request, HttpServletResponse response) {
-        MultipartHttpServletRequest multipartRequest = (MultipartHttpServletRequest) request;
-        Map<String, MultipartFile> fileMap = multipartRequest.getFileMap();
-        for (Map.Entry<String, MultipartFile> entity : fileMap.entrySet()) {
-            MultipartFile file = entity.getValue();
-            ImportParams params = new ImportParams();
-            params.setTitleRows(2);
-            params.setHeadRows(1);
-            params.setNeedSave(true);
-            try {
-                List<AppiumJob> listAppiumJobs = ExcelImportUtil.importExcel(file.getInputStream(), AppiumJob.class, params);
-                appiumJobService.saveBatch(listAppiumJobs);
-                return Result.ok("文件导入成功!数据行数:" + listAppiumJobs.size());
-            } catch (Exception e) {
-                log.error(e.getMessage(), e);
-                return Result.error("文件导入失败:" + e.getMessage());
-            } finally {
-                try {
-                    file.getInputStream().close();
-                } catch (IOException e) {
-                    e.printStackTrace();
-                }
-            }
-        }
-        return Result.ok("文件导入失败!");
-    }
-}

+ 0 - 258
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/controller/AppiumTaskController.java

@@ -1,258 +0,0 @@
-package cn.com.ctop.crawler.modules.appium.controller;
-
-import cn.com.ctop.crawler.modules.appium.entity.AppiumTask;
-import cn.com.ctop.crawler.modules.appium.service.IAppiumTaskService;
-import com.alibaba.fastjson.JSON;
-import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
-import com.baomidou.mybatisplus.core.metadata.IPage;
-import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiOperation;
-import lombok.extern.slf4j.Slf4j;
-import org.jeecg.common.api.vo.Result;
-import org.jeecg.common.aspect.annotation.AutoLog;
-import org.jeecg.common.system.query.QueryGenerator;
-import org.jeecg.common.util.oConvertUtils;
-import org.jeecgframework.poi.excel.ExcelImportUtil;
-import org.jeecgframework.poi.excel.def.NormalExcelConstants;
-import org.jeecgframework.poi.excel.entity.ExportParams;
-import org.jeecgframework.poi.excel.entity.ImportParams;
-import org.jeecgframework.poi.excel.view.JeecgEntityExcelView;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.web.bind.annotation.*;
-import org.springframework.web.multipart.MultipartFile;
-import org.springframework.web.multipart.MultipartHttpServletRequest;
-import org.springframework.web.servlet.ModelAndView;
-
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.net.URLDecoder;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
-/**
- * 任务表
- * @author jeecg-boot
- * @date 2019-10-31
- * @version V1.0
- */
-@Slf4j
-@Api(tags = "任务表")
-@RestController
-@RequestMapping("/appium/appiumTask")
-public class AppiumTaskController {
-    @Autowired
-    private IAppiumTaskService appiumTaskService;
-
-    /**
-     * 分页列表查询
-     *
-     * @param appiumTask
-     * @param pageNo
-     * @param pageSize
-     * @param req
-     * @return
-     */
-    @AutoLog(value = "任务表-分页列表查询")
-    @ApiOperation(value = "任务表-分页列表查询", notes = "任务表-分页列表查询")
-    @GetMapping(value = "/list")
-    public Result<IPage<AppiumTask>> queryPageList(AppiumTask appiumTask,
-                                                   @RequestParam(name = "pageNo", defaultValue = "1") Integer pageNo,
-                                                   @RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize,
-                                                   HttpServletRequest req) {
-        Result<IPage<AppiumTask>> result = new Result<IPage<AppiumTask>>();
-        QueryWrapper<AppiumTask> queryWrapper = QueryGenerator.initQueryWrapper(appiumTask, req.getParameterMap());
-        Page<AppiumTask> page = new Page<AppiumTask>(pageNo, pageSize);
-        IPage<AppiumTask> pageList = appiumTaskService.page(page, queryWrapper);
-        result.setSuccess(true);
-        result.setResult(pageList);
-        return result;
-    }
-
-    /**
-     * 添加
-     *
-     * @param appiumTask
-     * @return
-     */
-    @AutoLog(value = "任务表-添加")
-    @ApiOperation(value = "任务表-添加", notes = "任务表-添加")
-    @PostMapping(value = "/add")
-    public Result<AppiumTask> add(@RequestBody AppiumTask appiumTask) {
-        Result<AppiumTask> result = new Result<AppiumTask>();
-        try {
-            appiumTaskService.save(appiumTask);
-            result.success("添加成功!");
-        } catch (Exception e) {
-            log.error(e.getMessage(), e);
-            result.error500("操作失败");
-        }
-        return result;
-    }
-
-    /**
-     * 编辑
-     *
-     * @param appiumTask
-     * @return
-     */
-    @AutoLog(value = "任务表-编辑")
-    @ApiOperation(value = "任务表-编辑", notes = "任务表-编辑")
-    @PutMapping(value = "/edit")
-    public Result<AppiumTask> edit(@RequestBody AppiumTask appiumTask) {
-        Result<AppiumTask> result = new Result<AppiumTask>();
-        AppiumTask appiumTaskEntity = appiumTaskService.getById(appiumTask.getId());
-        if (appiumTaskEntity == null) {
-            result.error500("未找到对应实体");
-        } else {
-            boolean ok = appiumTaskService.updateById(appiumTask);
-            if (ok) {
-                result.success("修改成功!");
-            }
-        }
-
-        return result;
-    }
-
-    /**
-     * 通过id删除
-     *
-     * @param id
-     * @return
-     */
-    @AutoLog(value = "任务表-通过id删除")
-    @ApiOperation(value = "任务表-通过id删除", notes = "任务表-通过id删除")
-    @DeleteMapping(value = "/delete")
-    public Result<?> delete(@RequestParam(name = "id", required = true) String id) {
-        try {
-            appiumTaskService.removeById(id);
-        } catch (Exception e) {
-            log.error("删除失败", e.getMessage());
-            return Result.error("删除失败!");
-        }
-        return Result.ok("删除成功!");
-    }
-
-    /**
-     * 批量删除
-     *
-     * @param ids
-     * @return
-     */
-    @AutoLog(value = "任务表-批量删除")
-    @ApiOperation(value = "任务表-批量删除", notes = "任务表-批量删除")
-    @DeleteMapping(value = "/deleteBatch")
-    public Result<AppiumTask> deleteBatch(@RequestParam(name = "ids", required = true) String ids) {
-        Result<AppiumTask> result = new Result<AppiumTask>();
-        if (ids == null || "".equals(ids.trim())) {
-            result.error500("参数不识别!");
-        } else {
-            this.appiumTaskService.removeByIds(Arrays.asList(ids.split(",")));
-            result.success("删除成功!");
-        }
-        return result;
-    }
-
-    /**
-     * 通过id查询
-     *
-     * @param id
-     * @return
-     */
-    @AutoLog(value = "任务表-通过id查询")
-    @ApiOperation(value = "任务表-通过id查询", notes = "任务表-通过id查询")
-    @GetMapping(value = "/queryById")
-    public Result<AppiumTask> queryById(@RequestParam(name = "id", required = true) String id) {
-        Result<AppiumTask> result = new Result<AppiumTask>();
-        AppiumTask appiumTask = appiumTaskService.getById(id);
-        if (appiumTask == null) {
-            result.error500("未找到对应实体");
-        } else {
-            result.setResult(appiumTask);
-            result.setSuccess(true);
-        }
-        return result;
-    }
-
-    /**
-     * 导出excel
-     *
-     * @param request
-     * @param response
-     */
-    @RequestMapping(value = "/exportXls")
-    public ModelAndView exportXls(HttpServletRequest request, HttpServletResponse response) {
-        // Step.1 组装查询条件
-        QueryWrapper<AppiumTask> queryWrapper = null;
-        try {
-            String paramsStr = request.getParameter("paramsStr");
-            if (oConvertUtils.isNotEmpty(paramsStr)) {
-                String deString = URLDecoder.decode(paramsStr, "UTF-8");
-                AppiumTask appiumTask = JSON.parseObject(deString, AppiumTask.class);
-                queryWrapper = QueryGenerator.initQueryWrapper(appiumTask, request.getParameterMap());
-            }
-        } catch (UnsupportedEncodingException e) {
-            e.printStackTrace();
-        }
-
-        //Step.2 AutoPoi 导出Excel
-        ModelAndView mv = new ModelAndView(new JeecgEntityExcelView());
-        List<AppiumTask> pageList = appiumTaskService.list(queryWrapper);
-        //导出文件名称
-        mv.addObject(NormalExcelConstants.FILE_NAME, "任务表列表");
-        mv.addObject(NormalExcelConstants.CLASS, AppiumTask.class);
-        mv.addObject(NormalExcelConstants.PARAMS, new ExportParams("任务表列表数据", "导出人:Jeecg", "导出信息"));
-        mv.addObject(NormalExcelConstants.DATA_LIST, pageList);
-        return mv;
-    }
-
-    @RequestMapping(value = "/run/login", method = RequestMethod.POST)
-    public Result<?> runLogin(HttpServletRequest request, HttpServletResponse response, Integer taskId, Integer deviceId) {
-        Thread thread = new Thread() {
-            @Override
-            public void run() {
-                appiumTaskService.runKuaishouLoginTask();
-            }
-        };
-        thread.start();
-        return Result.ok("任务已启动");
-    }
-    /**
-     * 通过excel导入数据
-     *
-     * @param request
-     * @param response
-     * @return
-     */
-    @RequestMapping(value = "/importExcel", method = RequestMethod.POST)
-    public Result<?> importExcel(HttpServletRequest request, HttpServletResponse response) {
-        MultipartHttpServletRequest multipartRequest = (MultipartHttpServletRequest) request;
-        Map<String, MultipartFile> fileMap = multipartRequest.getFileMap();
-        for (Map.Entry<String, MultipartFile> entity : fileMap.entrySet()) {
-            MultipartFile file = entity.getValue();
-            ImportParams params = new ImportParams();
-            params.setTitleRows(2);
-            params.setHeadRows(1);
-            params.setNeedSave(true);
-            try {
-                List<AppiumTask> listAppiumTasks = ExcelImportUtil.importExcel(file.getInputStream(), AppiumTask.class, params);
-                appiumTaskService.saveBatch(listAppiumTasks);
-                return Result.ok("文件导入成功!数据行数:" + listAppiumTasks.size());
-            } catch (Exception e) {
-                log.error(e.getMessage(), e);
-                return Result.error("文件导入失败:" + e.getMessage());
-            } finally {
-                try {
-                    file.getInputStream().close();
-                } catch (IOException e) {
-                    e.printStackTrace();
-                }
-            }
-        }
-        return Result.ok("文件导入失败!");
-    }
-
-}

+ 0 - 247
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/controller/AppiumTaskItemController.java

@@ -1,247 +0,0 @@
-package cn.com.ctop.crawler.modules.appium.controller;
-
-import cn.com.ctop.crawler.modules.appium.entity.AppiumTaskItem;
-import cn.com.ctop.crawler.modules.appium.service.IAppiumTaskItemService;
-import com.alibaba.fastjson.JSON;
-import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
-import com.baomidou.mybatisplus.core.metadata.IPage;
-import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiOperation;
-import lombok.extern.slf4j.Slf4j;
-import org.jeecg.common.api.vo.Result;
-import org.jeecg.common.aspect.annotation.AutoLog;
-import org.jeecg.common.system.query.QueryGenerator;
-import org.jeecg.common.util.oConvertUtils;
-import org.jeecgframework.poi.excel.ExcelImportUtil;
-import org.jeecgframework.poi.excel.def.NormalExcelConstants;
-import org.jeecgframework.poi.excel.entity.ExportParams;
-import org.jeecgframework.poi.excel.entity.ImportParams;
-import org.jeecgframework.poi.excel.view.JeecgEntityExcelView;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.web.bind.annotation.*;
-import org.springframework.web.multipart.MultipartFile;
-import org.springframework.web.multipart.MultipartHttpServletRequest;
-import org.springframework.web.servlet.ModelAndView;
-
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.net.URLDecoder;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
-/**
- * 任务内容表
- * @author jeecg-boot
- * @date 2019-10-31
- * @version V1.0
- */
-@Slf4j
-@Api(tags = "任务内容表")
-@RestController
-@RequestMapping("/appium/appiumTaskItem")
-public class AppiumTaskItemController {
-    @Autowired
-    private IAppiumTaskItemService appiumTaskItemService;
-
-    /**
-     * 分页列表查询
-     *
-     * @param appiumTaskItem
-     * @param pageNo
-     * @param pageSize
-     * @param req
-     * @return
-     */
-    @AutoLog(value = "任务内容表-分页列表查询")
-    @ApiOperation(value = "任务内容表-分页列表查询", notes = "任务内容表-分页列表查询")
-    @GetMapping(value = "/list")
-    public Result<IPage<AppiumTaskItem>> queryPageList(AppiumTaskItem appiumTaskItem,
-                                                       @RequestParam(name = "pageNo", defaultValue = "1") Integer pageNo,
-                                                       @RequestParam(name = "pageSize", defaultValue = "100") Integer pageSize,
-                                                       HttpServletRequest req) {
-        Result<IPage<AppiumTaskItem>> result = new Result<IPage<AppiumTaskItem>>();
-        QueryWrapper<AppiumTaskItem> queryWrapper = QueryGenerator.initQueryWrapper(appiumTaskItem, req.getParameterMap());
-        Page<AppiumTaskItem> page = new Page<AppiumTaskItem>(pageNo, pageSize);
-        IPage<AppiumTaskItem> pageList = appiumTaskItemService.page(page, queryWrapper);
-        result.setSuccess(true);
-        result.setResult(pageList);
-        return result;
-    }
-
-    /**
-     * 添加
-     *
-     * @param appiumTaskItem
-     * @return
-     */
-    @AutoLog(value = "任务内容表-添加")
-    @ApiOperation(value = "任务内容表-添加", notes = "任务内容表-添加")
-    @PostMapping(value = "/add")
-    public Result<AppiumTaskItem> add(@RequestBody AppiumTaskItem appiumTaskItem) {
-        Result<AppiumTaskItem> result = new Result<AppiumTaskItem>();
-        try {
-            appiumTaskItemService.save(appiumTaskItem);
-            result.success("添加成功!");
-        } catch (Exception e) {
-            log.error(e.getMessage(), e);
-            result.error500("操作失败");
-        }
-        return result;
-    }
-
-    /**
-     * 编辑
-     *
-     * @param appiumTaskItem
-     * @return
-     */
-    @AutoLog(value = "任务内容表-编辑")
-    @ApiOperation(value = "任务内容表-编辑", notes = "任务内容表-编辑")
-    @PutMapping(value = "/edit")
-    public Result<AppiumTaskItem> edit(@RequestBody AppiumTaskItem appiumTaskItem) {
-        Result<AppiumTaskItem> result = new Result<AppiumTaskItem>();
-        AppiumTaskItem appiumTaskItemEntity = appiumTaskItemService.getById(appiumTaskItem.getId());
-        if (appiumTaskItemEntity == null) {
-            result.error500("未找到对应实体");
-        } else {
-            boolean ok = appiumTaskItemService.updateById(appiumTaskItem);
-            if (ok) {
-                result.success("修改成功!");
-            }
-        }
-
-        return result;
-    }
-
-    /**
-     * 通过id删除
-     *
-     * @param id
-     * @return
-     */
-    @AutoLog(value = "任务内容表-通过id删除")
-    @ApiOperation(value = "任务内容表-通过id删除", notes = "任务内容表-通过id删除")
-    @DeleteMapping(value = "/delete")
-    public Result<?> delete(@RequestParam(name = "id", required = true) String id) {
-        try {
-            appiumTaskItemService.removeById(id);
-        } catch (Exception e) {
-            log.error("删除失败", e.getMessage());
-            return Result.error("删除失败!");
-        }
-        return Result.ok("删除成功!");
-    }
-
-    /**
-     * 批量删除
-     *
-     * @param ids
-     * @return
-     */
-    @AutoLog(value = "任务内容表-批量删除")
-    @ApiOperation(value = "任务内容表-批量删除", notes = "任务内容表-批量删除")
-    @DeleteMapping(value = "/deleteBatch")
-    public Result<AppiumTaskItem> deleteBatch(@RequestParam(name = "ids", required = true) String ids) {
-        Result<AppiumTaskItem> result = new Result<AppiumTaskItem>();
-        if (ids == null || "".equals(ids.trim())) {
-            result.error500("参数不识别!");
-        } else {
-            this.appiumTaskItemService.removeByIds(Arrays.asList(ids.split(",")));
-            result.success("删除成功!");
-        }
-        return result;
-    }
-
-    /**
-     * 通过id查询
-     *
-     * @param id
-     * @return
-     */
-    @AutoLog(value = "任务内容表-通过id查询")
-    @ApiOperation(value = "任务内容表-通过id查询", notes = "任务内容表-通过id查询")
-    @GetMapping(value = "/queryById")
-    public Result<AppiumTaskItem> queryById(@RequestParam(name = "id", required = true) String id) {
-        Result<AppiumTaskItem> result = new Result<AppiumTaskItem>();
-        AppiumTaskItem appiumTaskItem = appiumTaskItemService.getById(id);
-        if (appiumTaskItem == null) {
-            result.error500("未找到对应实体");
-        } else {
-            result.setResult(appiumTaskItem);
-            result.setSuccess(true);
-        }
-        return result;
-    }
-
-    /**
-     * 导出excel
-     *
-     * @param request
-     * @param response
-     */
-    @RequestMapping(value = "/exportXls")
-    public ModelAndView exportXls(HttpServletRequest request, HttpServletResponse response) {
-        // Step.1 组装查询条件
-        QueryWrapper<AppiumTaskItem> queryWrapper = null;
-        try {
-            String paramsStr = request.getParameter("paramsStr");
-            if (oConvertUtils.isNotEmpty(paramsStr)) {
-                String deString = URLDecoder.decode(paramsStr, "UTF-8");
-                AppiumTaskItem appiumTaskItem = JSON.parseObject(deString, AppiumTaskItem.class);
-                queryWrapper = QueryGenerator.initQueryWrapper(appiumTaskItem, request.getParameterMap());
-            }
-        } catch (UnsupportedEncodingException e) {
-            e.printStackTrace();
-        }
-
-        //Step.2 AutoPoi 导出Excel
-        ModelAndView mv = new ModelAndView(new JeecgEntityExcelView());
-        List<AppiumTaskItem> pageList = appiumTaskItemService.list(queryWrapper);
-        //导出文件名称
-        mv.addObject(NormalExcelConstants.FILE_NAME, "任务内容表列表");
-        mv.addObject(NormalExcelConstants.CLASS, AppiumTaskItem.class);
-        mv.addObject(NormalExcelConstants.PARAMS, new ExportParams("任务内容表列表数据", "导出人:Jeecg", "导出信息"));
-        mv.addObject(NormalExcelConstants.DATA_LIST, pageList);
-        return mv;
-    }
-
-    /**
-     * 通过excel导入数据
-     *
-     * @param request
-     * @param response
-     * @return
-     */
-    @RequestMapping(value = "/importExcel", method = RequestMethod.POST)
-    public Result<?> importExcel(HttpServletRequest request, HttpServletResponse response) {
-        MultipartHttpServletRequest multipartRequest = (MultipartHttpServletRequest) request;
-        Map<String, MultipartFile> fileMap = multipartRequest.getFileMap();
-        for (Map.Entry<String, MultipartFile> entity : fileMap.entrySet()) {
-            MultipartFile file = entity.getValue();
-            ImportParams params = new ImportParams();
-            params.setTitleRows(2);
-            params.setHeadRows(1);
-            params.setNeedSave(true);
-            try {
-                List<AppiumTaskItem> listAppiumTaskItems = ExcelImportUtil.importExcel(file.getInputStream(), AppiumTaskItem.class, params);
-                appiumTaskItemService.saveBatch(listAppiumTaskItems);
-                return Result.ok("文件导入成功!数据行数:" + listAppiumTaskItems.size());
-            } catch (Exception e) {
-                log.error(e.getMessage(), e);
-                return Result.error("文件导入失败:" + e.getMessage());
-            } finally {
-                try {
-                    file.getInputStream().close();
-                } catch (IOException e) {
-                    e.printStackTrace();
-                }
-            }
-        }
-        return Result.ok("文件导入失败!");
-    }
-
-}

+ 0 - 164
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/controller/AppiumTaskLogController.java

@@ -1,164 +0,0 @@
-package cn.com.ctop.crawler.modules.appium.controller;
-
-import cn.com.ctop.crawler.modules.appium.entity.AppiumTaskLog;
-import cn.com.ctop.crawler.modules.appium.service.IAppiumTaskLogService;
-import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
-import com.baomidou.mybatisplus.core.metadata.IPage;
-import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiOperation;
-import lombok.extern.slf4j.Slf4j;
-import org.jeecg.common.api.vo.Result;
-import org.jeecg.common.aspect.annotation.AutoLog;
-import org.jeecg.common.system.query.QueryGenerator;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.web.bind.annotation.*;
-
-import javax.servlet.http.HttpServletRequest;
-import java.util.Arrays;
-
-/**
- * 任务日志表
- *
- * @author jeecg-boot
- * @version V1.0
- * @date 2019-11-01
- */
-@Slf4j
-@Api(tags = "任务日志表")
-@RestController
-@RequestMapping("/appium/appiumTaskLog")
-public class AppiumTaskLogController {
-    @Autowired
-    private IAppiumTaskLogService appiumTaskLogService;
-
-    /**
-     * 分页列表查询
-     *
-     * @param appiumTaskLog
-     * @param pageNo
-     * @param pageSize
-     * @param req
-     * @return
-     */
-    @AutoLog(value = "任务日志表-分页列表查询")
-    @ApiOperation(value = "任务日志表-分页列表查询", notes = "任务日志表-分页列表查询")
-    @GetMapping(value = "/list")
-    public Result<IPage<AppiumTaskLog>> queryPageList(AppiumTaskLog appiumTaskLog,
-                                                      @RequestParam(name = "pageNo", defaultValue = "1") Integer pageNo,
-                                                      @RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize,
-                                                      HttpServletRequest req) {
-        Result<IPage<AppiumTaskLog>> result = new Result<>();
-        QueryWrapper<AppiumTaskLog> queryWrapper = QueryGenerator.initQueryWrapper(appiumTaskLog, req.getParameterMap());
-        Page<AppiumTaskLog> page = new Page<>(pageNo, pageSize);
-        IPage<AppiumTaskLog> pageList = appiumTaskLogService.page(page, queryWrapper);
-        result.setSuccess(true);
-        result.setResult(pageList);
-        return result;
-    }
-
-    /**
-     * 添加
-     *
-     * @param appiumTaskLog
-     * @return
-     */
-    @AutoLog(value = "任务日志表-添加")
-    @ApiOperation(value = "任务日志表-添加", notes = "任务日志表-添加")
-    @PostMapping(value = "/add")
-    public Result<AppiumTaskLog> add(@RequestBody AppiumTaskLog appiumTaskLog) {
-        Result<AppiumTaskLog> result = new Result<>();
-        try {
-            appiumTaskLogService.save(appiumTaskLog);
-            result.success("添加成功!");
-        } catch (Exception e) {
-            log.error(e.getMessage(), e);
-            result.error500("操作失败");
-        }
-        return result;
-    }
-
-    /**
-     * 编辑
-     *
-     * @param appiumTaskLog
-     * @return
-     */
-    @AutoLog(value = "任务日志表-编辑")
-    @ApiOperation(value = "任务日志表-编辑", notes = "任务日志表-编辑")
-    @PutMapping(value = "/edit")
-    public Result<AppiumTaskLog> edit(@RequestBody AppiumTaskLog appiumTaskLog) {
-        Result<AppiumTaskLog> result = new Result<>();
-        AppiumTaskLog appiumTaskLogEntity = appiumTaskLogService.getById(appiumTaskLog.getId());
-        if (appiumTaskLogEntity == null) {
-            result.error500("未找到对应实体");
-        } else {
-            boolean ok = appiumTaskLogService.updateById(appiumTaskLog);
-            if (ok) {
-                result.success("修改成功!");
-            }
-        }
-
-        return result;
-    }
-
-    /**
-     * 通过id删除
-     *
-     * @param id
-     * @return
-     */
-    @AutoLog(value = "任务日志表-通过id删除")
-    @ApiOperation(value = "任务日志表-通过id删除", notes = "任务日志表-通过id删除")
-    @DeleteMapping(value = "/delete")
-    public Result<?> delete(@RequestParam(name = "id", required = true) String id) {
-        try {
-            appiumTaskLogService.removeById(id);
-        } catch (Exception e) {
-            log.error("删除失败", e.getMessage());
-            return Result.error("删除失败!");
-        }
-        return Result.ok("删除成功!");
-    }
-
-    /**
-     * 批量删除
-     *
-     * @param ids
-     * @return
-     */
-    @AutoLog(value = "任务日志表-批量删除")
-    @ApiOperation(value = "任务日志表-批量删除", notes = "任务日志表-批量删除")
-    @DeleteMapping(value = "/deleteBatch")
-    public Result<AppiumTaskLog> deleteBatch(@RequestParam(name = "ids", required = true) String ids) {
-        Result<AppiumTaskLog> result = new Result<>();
-        if (ids == null || "".equals(ids.trim())) {
-            result.error500("参数不识别!");
-        } else {
-            this.appiumTaskLogService.removeByIds(Arrays.asList(ids.split(",")));
-            result.success("删除成功!");
-        }
-        return result;
-    }
-
-    /**
-     * 通过id查询
-     *
-     * @param id
-     * @return
-     */
-    @AutoLog(value = "任务日志表-通过id查询")
-    @ApiOperation(value = "任务日志表-通过id查询", notes = "任务日志表-通过id查询")
-    @GetMapping(value = "/queryById")
-    public Result<AppiumTaskLog> queryById(@RequestParam(name = "id", required = true) String id) {
-        Result<AppiumTaskLog> result = new Result<>();
-        AppiumTaskLog appiumTaskLog = appiumTaskLogService.getById(id);
-        if (appiumTaskLog == null) {
-            result.error500("未找到对应实体");
-        } else {
-            result.setResult(appiumTaskLog);
-            result.setSuccess(true);
-        }
-        return result;
-    }
-}

+ 0 - 62
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/entity/AppiumDevice.java

@@ -1,62 +0,0 @@
-package cn.com.ctop.crawler.modules.appium.entity;
-
-import com.baomidou.mybatisplus.annotation.IdType;
-import com.baomidou.mybatisplus.annotation.TableId;
-import com.baomidou.mybatisplus.annotation.TableName;
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.experimental.Accessors;
-import org.jeecgframework.poi.excel.annotation.Excel;
-
-import java.util.Date;
-
-/**
- * 手机设备表
- * @author jeecg-boot
- * @date 2019-10-31
- * @version V1.0
- */
-@Data
-@TableName("ctop_appium_device")
-@EqualsAndHashCode(callSuper = false)
-@Accessors(chain = true)
-@ApiModel(value = "ctop_appium_device对象", description = "手机设备表")
-public class AppiumDevice {
-
-    /**
-     * 主键
-     */
-    @TableId(type = IdType.AUTO)
-    @ApiModelProperty(value = "主键")
-    private Integer id;
-    /**
-     * IP
-     */
-    @Excel(name = "IP", width = 15)
-    @ApiModelProperty(value = "IP")
-    private String ip;
-    /**
-     * 端口号
-     */
-    @Excel(name = "端口号", width = 15)
-    @ApiModelProperty(value = "端口号")
-    private String port;
-    /**
-     * 状态
-     */
-    @Excel(name = "状态", width = 15)
-    @ApiModelProperty(value = "状态")
-    private Integer status;
-    /**
-     * createTime
-     */
-    @ApiModelProperty(value = "createTime")
-    private Date createTime;
-    /**
-     * updateTime
-     */
-    @ApiModelProperty(value = "updateTime")
-    private Date updateTime;
-}

+ 0 - 57
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/entity/AppiumJob.java

@@ -1,57 +0,0 @@
-package cn.com.ctop.crawler.modules.appium.entity;
-
-import com.baomidou.mybatisplus.annotation.IdType;
-import com.baomidou.mybatisplus.annotation.TableId;
-import com.baomidou.mybatisplus.annotation.TableName;
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.experimental.Accessors;
-import org.jeecgframework.poi.excel.annotation.Excel;
-
-import java.util.Date;
-
-/**
- * 爬虫调度任务
- *
- * @author jeecg-boot
- * @version V1.0
- * @date 2019-11-14
- */
-@Data
-@TableName("ctop_appium_job")
-@EqualsAndHashCode(callSuper = false)
-@Accessors(chain = true)
-@ApiModel(value = "ctop_appium_job对象", description = "爬虫调度任务")
-public class AppiumJob {
-
-    /**
-     * id
-     */
-    @TableId(type = IdType.AUTO)
-    @ApiModelProperty(value = "id")
-    private Long id;
-    /**
-     * 调度任务名称
-     */
-    @Excel(name = "调度任务名称", width = 15)
-    @ApiModelProperty(value = "调度任务名称")
-    private String jobName;
-    /**
-     * status
-     */
-    @Excel(name = "status", width = 15)
-    @ApiModelProperty(value = "status")
-    private Integer status;
-    /**
-     * createTime
-     */
-    @ApiModelProperty(value = "createTime")
-    private Date createTime;
-    /**
-     * updateTime
-     */
-    @ApiModelProperty(value = "updateTime")
-    private Date updateTime;
-}

+ 0 - 63
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/entity/AppiumJobBindTaskInfo.java

@@ -1,63 +0,0 @@
-package cn.com.ctop.crawler.modules.appium.entity;
-
-import com.baomidou.mybatisplus.annotation.IdType;
-import com.baomidou.mybatisplus.annotation.TableId;
-import com.baomidou.mybatisplus.annotation.TableName;
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.experimental.Accessors;
-import org.jeecgframework.poi.excel.annotation.Excel;
-
-import java.util.Date;
-
-/**
- * 爬虫调度任务绑定信息
- *
- * @author jeecg-boot
- * @version V1.0
- * @date 2019-11-14
- */
-@Data
-@TableName("ctop_appium_job_bind_task_info")
-@EqualsAndHashCode(callSuper = false)
-@Accessors(chain = true)
-@ApiModel(value = "ctop_appium_job_bind_task_info对象", description = "爬虫调度任务绑定信息")
-public class AppiumJobBindTaskInfo {
-
-    /**
-     * id
-     */
-    @TableId(type = IdType.AUTO)
-    @ApiModelProperty(value = "id")
-    private Long id;
-    /**
-     * 任务号
-     */
-    @Excel(name = "任务号", width = 15)
-    @ApiModelProperty(value = "任务号")
-    private Long jobId;
-    /**
-     * 执行任务id
-     */
-    @Excel(name = "执行任务id", width = 15)
-    @ApiModelProperty(value = "执行任务id")
-    private Long taskId;
-    /**
-     * 任务绑定状态
-     */
-    @Excel(name = "任务绑定状态", width = 15)
-    @ApiModelProperty(value = "任务绑定状态")
-    private Integer status;
-    /**
-     * createTime
-     */
-    @ApiModelProperty(value = "createTime")
-    private Date createTime;
-    /**
-     * updateTime
-     */
-    @ApiModelProperty(value = "updateTime")
-    private Date updateTime;
-}

+ 0 - 66
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/entity/AppiumTask.java

@@ -1,66 +0,0 @@
-package cn.com.ctop.crawler.modules.appium.entity;
-
-import com.baomidou.mybatisplus.annotation.IdType;
-import com.baomidou.mybatisplus.annotation.TableField;
-import com.baomidou.mybatisplus.annotation.TableId;
-import com.baomidou.mybatisplus.annotation.TableName;
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.experimental.Accessors;
-import org.jeecgframework.poi.excel.annotation.Excel;
-
-import java.util.Date;
-import java.util.List;
-
-/**
- * 任务表
- * @author jeecg-boot
- * @date 2019-10-31
- * @version V1.0
- */
-@Data
-@TableName("ctop_appium_task")
-@EqualsAndHashCode(callSuper = false)
-@Accessors(chain = true)
-@ApiModel(value = "ctop_appium_task对象", description = "任务表")
-public class AppiumTask {
-
-    /**
-     * 主键
-     */
-    @TableId(type = IdType.AUTO)
-    @ApiModelProperty(value = "主键")
-    private Integer id;
-    /**
-     * 任务名称
-     */
-    @Excel(name = "任务名称", width = 15)
-    @ApiModelProperty(value = "任务名称")
-    private String taskName;
-    /**
-     * APP包名
-     */
-    @Excel(name = "APP包名", width = 15)
-    @ApiModelProperty(value = "APP包名")
-    private String appPackage;
-    /**
-     * APP界面名
-     */
-    @Excel(name = "APP界面名", width = 15)
-    @ApiModelProperty(value = "APP界面名")
-    private String appActivity;
-    /**
-     * createTime
-     */
-    @ApiModelProperty(value = "createTime")
-    private Date createTime;
-    /**
-     * updateTime
-     */
-    @ApiModelProperty(value = "updateTime")
-    private Date updateTime;
-    @TableField(exist = false)
-    private List<AppiumTask> subList;
-}

+ 0 - 152
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/entity/AppiumTaskItem.java

@@ -1,152 +0,0 @@
-package cn.com.ctop.crawler.modules.appium.entity;
-
-import com.baomidou.mybatisplus.annotation.IdType;
-import com.baomidou.mybatisplus.annotation.TableField;
-import com.baomidou.mybatisplus.annotation.TableId;
-import com.baomidou.mybatisplus.annotation.TableName;
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.experimental.Accessors;
-import org.jeecgframework.poi.excel.annotation.Excel;
-
-import java.util.Date;
-import java.util.List;
-
-/**
- * 任务内容表
- * @author jeecg-boot
- * @date 2019-10-31
- * @version V1.0
- */
-@Data
-@TableName("ctop_appium_task_item")
-@EqualsAndHashCode(callSuper = false)
-@Accessors(chain = true)
-@ApiModel(value = "ctop_appium_task_item对象", description = "任务内容表")
-public class AppiumTaskItem {
-
-	/**
-	 * 主键
-	 */
-	@TableId(type = IdType.AUTO)
-	@ApiModelProperty(value = "主键")
-	private Integer id;
-	/**
-	 * 任务ID
-	 */
-	@Excel(name = "任务ID", width = 15)
-	@ApiModelProperty(value = "任务ID")
-	private Long taskId;
-	/**
-	 * 父ID
-	 */
-	@Excel(name = "父ID", width = 15)
-	@ApiModelProperty(value = "父ID")
-	private Integer parentId;
-	/**
-	 * 顺序
-	 */
-	@Excel(name = "顺序", width = 15)
-	@ApiModelProperty(value = "顺序")
-	private Integer seq;
-	/**
-	 * 查找类型
-	 */
-	@Excel(name = "查找类型", width = 15)
-	@ApiModelProperty(value = "查找类型")
-	private String findType;
-
-    @Excel(name = "名称", width = 15)
-    @ApiModelProperty(value = "名称")
-    private String name;
-	/**
-	 * 查找关键字
-	 */
-	@Excel(name = "查找关键字", width = 15)
-	@ApiModelProperty(value = "查找关键字")
-	private String findKey;
-	/**
-	 * 点击类型
-	 */
-	@Excel(name = "点击类型", width = 15)
-	@ApiModelProperty(value = "点击类型")
-	private String clickType;
-	/**
-	 * 偏移量X
-	 */
-	@Excel(name = "偏移量X", width = 15)
-	@ApiModelProperty(value = "偏移量X")
-	private Integer offsiteX;
-	/**
-	 * 偏移量Y
-	 */
-	@Excel(name = "偏移量Y", width = 15)
-	@ApiModelProperty(value = "偏移量Y")
-	private Integer offsiteY;
-	/**
-	 * 循环间隔
-	 */
-	@Excel(name = "循环间隔", width = 15)
-	@ApiModelProperty(value = "循环间隔")
-	private Integer loopRate;
-	/**
-	 * 循环类型
-	 */
-	@Excel(name = "循环类型", width = 15)
-	@ApiModelProperty(value = "循环类型")
-	private String loopType;
-	/**
-	 * 是否必经执行
-	 */
-	@Excel(name = "是否必经执行", width = 15)
-	@ApiModelProperty(value = "是否必经执行")
-	private Integer isMust;
-	/**
-	 * 文本查找类型
-	 */
-	@Excel(name = "文本查找类型", width = 15)
-	@ApiModelProperty(value = "文本查找类型")
-	private String textEqualType;
-	/**
-	 * 文本查找关键字
-	 */
-	@Excel(name = "文本查找关键字", width = 15)
-	@ApiModelProperty(value = "文本查找关键字")
-	private String textEqualKey;
-	/**
-	 * 滑动类型
-	 */
-	@Excel(name = "滑动类型", width = 15)
-	@ApiModelProperty(value = "滑动类型")
-	private String swapType;
-	/**
-	 * 延迟时长
-	 */
-	@Excel(name = "延迟时长", width = 15)
-	@ApiModelProperty(value = "延迟时长")
-	private Integer waitTime;
-	/**
-	 * 是否执行完关闭
-	 */
-	@Excel(name = "是否执行完关闭", width = 15)
-	@ApiModelProperty(value = "是否执行完关闭")
-	private Integer isClose;
-	/**
-	 * createTime
-	 */
-    @ApiModelProperty(value = "createTime")
-	private Date createTime;
-	/**updateTime*/
-    @ApiModelProperty(value = "updateTime")
-	private Date updateTime;
-
-    @ApiModelProperty(value = "appPackage")
-    private String appPackage;
-    @ApiModelProperty(value = "appActivity")
-    private String appActivity;
-
-    @TableField(exist = false)
-    private List<AppiumTaskItem> subList;
-}

+ 0 - 83
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/entity/AppiumTaskLog.java

@@ -1,83 +0,0 @@
-package cn.com.ctop.crawler.modules.appium.entity;
-
-import com.baomidou.mybatisplus.annotation.IdType;
-import com.baomidou.mybatisplus.annotation.TableId;
-import com.baomidou.mybatisplus.annotation.TableName;
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.experimental.Accessors;
-import org.jeecg.common.aspect.annotation.Dict;
-import org.jeecgframework.poi.excel.annotation.Excel;
-
-import java.util.Date;
-
-/**
- * 任务日志表
- *
- * @author jeecg-boot
- * @version V1.0
- * @date 2019-11-01
- */
-@Data
-@TableName("ctop_appium_task_log")
-@EqualsAndHashCode(callSuper = false)
-@Accessors(chain = true)
-@ApiModel(value = "ctop_appium_task_log对象", description = "任务日志表")
-public class AppiumTaskLog {
-
-    /**
-     * 主键
-     */
-    @TableId(type = IdType.AUTO)
-    @ApiModelProperty(value = "主键")
-    private Long id;
-    /**
-     * 任务ID
-     */
-    @Excel(name = "任务ID", width = 15)
-    @ApiModelProperty(value = "任务ID")
-    private Long taskId;
-    /**
-     * 设备ID
-     */
-    @Excel(name = "设备ID", width = 15)
-    @ApiModelProperty(value = "设备ID")
-    private Integer deviceId;
-    /**
-     * 任务名称
-     */
-    @Excel(name = "任务名称", width = 15)
-    @ApiModelProperty(value = "任务名称")
-    private String taskName;
-    /**
-     * 设备IP
-     */
-    @Excel(name = "设备IP", width = 15)
-    @ApiModelProperty(value = "设备IP")
-    private String deviceIp;
-    /**
-     * 设备端口
-     */
-    @Excel(name = "设备端口", width = 15)
-    @ApiModelProperty(value = "设备端口")
-    private String devicePort;
-    /**
-     * 状态
-     */
-    @Excel(name = "状态", width = 15)
-    @Dict(dicCode = "appium_ks_log_status")
-    @ApiModelProperty(value = "状态")
-    private String status;
-    /**
-     * 创建时间
-     */
-    @ApiModelProperty(value = "创建时间")
-    private Date createTime;
-    /**
-     * 修改时间
-     */
-    @ApiModelProperty(value = "修改时间")
-    private Date updateTime;
-}

+ 0 - 17
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/mapper/AppiumDeviceMapper.java

@@ -1,17 +0,0 @@
-package cn.com.ctop.crawler.modules.appium.mapper;
-
-import java.util.List;
-
-import org.apache.ibatis.annotations.Param;
-import cn.com.ctop.crawler.modules.appium.entity.AppiumDevice;
-import com.baomidou.mybatisplus.core.mapper.BaseMapper;
-
-/**
- * 手机设备表
- * @author: jeecg-boot
- * @date: 2019-10-31
- * @cersion: V1.0
- */
-public interface AppiumDeviceMapper extends BaseMapper<AppiumDevice> {
-
-}

+ 0 - 15
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/mapper/AppiumJobBindTaskInfoMapper.java

@@ -1,15 +0,0 @@
-package cn.com.ctop.crawler.modules.appium.mapper;
-
-import cn.com.ctop.crawler.modules.appium.entity.AppiumJobBindTaskInfo;
-import com.baomidou.mybatisplus.core.mapper.BaseMapper;
-
-/**
- * 爬虫调度任务绑定信息
- *
- * @author: jeecg-boot
- * @date: 2019-11-14
- * @cersion: V1.0
- */
-public interface AppiumJobBindTaskInfoMapper extends BaseMapper<AppiumJobBindTaskInfo> {
-
-}

+ 0 - 21
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/mapper/AppiumJobMapper.java

@@ -1,21 +0,0 @@
-package cn.com.ctop.crawler.modules.appium.mapper;
-
-import cn.com.ctop.crawler.modules.appium.entity.AppiumJob;
-import cn.com.ctop.crawler.modules.appium.entity.AppiumTask;
-import com.baomidou.mybatisplus.core.mapper.BaseMapper;
-import org.apache.ibatis.annotations.Param;
-
-import java.util.List;
-
-/**
- * 爬虫调度任务
- *
- * @author: jeecg-boot
- * @date: 2019-11-14
- * @cersion: V1.0
- */
-public interface AppiumJobMapper extends BaseMapper<AppiumJob> {
-
-    List<AppiumTask> getTaskByJobId(@Param(value = "jobId") Long jobId);
-
-}

+ 0 - 18
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/mapper/AppiumTaskItemMapper.java

@@ -1,18 +0,0 @@
-package cn.com.ctop.crawler.modules.appium.mapper;
-
-import cn.com.ctop.crawler.modules.appium.entity.AppiumTaskItem;
-import com.baomidou.mybatisplus.core.mapper.BaseMapper;
-import org.apache.ibatis.annotations.Param;
-
-import java.util.List;
-
-/**
- * 任务内容表
- * @author: jeecg-boot
- * @date: 2019-10-31
- * @cersion: V1.0
- */
-public interface AppiumTaskItemMapper extends BaseMapper<AppiumTaskItem> {
-
-    List<AppiumTaskItem> selectByParams(@Param(value = "package") String packageName, @Param(value = "activity") String activityName, @Param(value = "taskId") Long taskId);
-}

+ 0 - 18
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/mapper/AppiumTaskLogMapper.java

@@ -1,18 +0,0 @@
-package cn.com.ctop.crawler.modules.appium.mapper;
-
-import java.util.List;
-
-import org.apache.ibatis.annotations.Param;
-import cn.com.ctop.crawler.modules.appium.entity.AppiumTaskLog;
-import com.baomidou.mybatisplus.core.mapper.BaseMapper;
-
-/**
- * 任务日志表
- *
- * @author: jeecg-boot
- * @date: 2019-11-01
- * @cersion: V1.0
- */
-public interface AppiumTaskLogMapper extends BaseMapper<AppiumTaskLog> {
-
-}

+ 0 - 17
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/mapper/AppiumTaskMapper.java

@@ -1,17 +0,0 @@
-package cn.com.ctop.crawler.modules.appium.mapper;
-
-import java.util.List;
-
-import org.apache.ibatis.annotations.Param;
-import cn.com.ctop.crawler.modules.appium.entity.AppiumTask;
-import com.baomidou.mybatisplus.core.mapper.BaseMapper;
-
-/**
- * 任务表
- * @author: jeecg-boot
- * @date: 2019-10-31
- * @cersion: V1.0
- */
-public interface AppiumTaskMapper extends BaseMapper<AppiumTask> {
-
-}

+ 0 - 5
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/mapper/xml/AppiumDeviceMapper.xml

@@ -1,5 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
-<mapper namespace="cn.com.ctop.crawler.modules.appium.mapper.AppiumDeviceMapper">
-
-</mapper>

+ 0 - 5
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/mapper/xml/AppiumJobBindTaskInfoMapper.xml

@@ -1,5 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
-<mapper namespace="cn.com.ctop.crawler.modules.appium.mapper.AppiumJobBindTaskInfoMapper">
-
-</mapper>

+ 0 - 11
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/mapper/xml/AppiumJobMapper.xml

@@ -1,11 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
-<mapper namespace="cn.com.ctop.crawler.modules.appium.mapper.AppiumJobMapper">
-    <select id="getTaskByJobId" resultType="cn.com.ctop.crawler.modules.appium.entity.AppiumTask">
-        select
-            *
-        from
-         ctop_appium_task task
-        left join ctop_appium_job_bind_task_info bindinfo on task.job_id = bindinfo.job_id
-    </select>
-</mapper>

+ 0 - 22
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/mapper/xml/AppiumTaskItemMapper.xml

@@ -1,22 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
-<mapper namespace="cn.com.ctop.crawler.modules.appium.mapper.AppiumTaskItemMapper">
-    <select id="selectByParams" resultType="cn.com.ctop.crawler.modules.appium.entity.AppiumTaskItem">
-        select
-        *
-        from
-        ctop_appium_task_item
-        where
-        1=1
-        <if test="null!=package">
-            and app_package = #{package}
-        </if>
-        <if test="null!=activity">
-            and app_activity = #{activity}
-        </if>
-        <if test="null!=taskId">
-            and task_id = #{taskId}
-        </if>
-        order by id desc
-    </select>
-</mapper>

+ 0 - 5
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/mapper/xml/AppiumTaskLogMapper.xml

@@ -1,5 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
-<mapper namespace="cn.com.ctop.crawler.modules.appium.mapper.AppiumTaskLogMapper">
-
-</mapper>

+ 0 - 15
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/service/IAppiumDeviceService.java

@@ -1,15 +0,0 @@
-package cn.com.ctop.crawler.modules.appium.service;
-
-import cn.com.ctop.crawler.modules.appium.entity.AppiumDevice;
-import com.baomidou.mybatisplus.extension.service.IService;
-
-/**
- * 手机设备表
- * @author jeecg-boot
- * @date 2019-10-31
- * @version V1.0
- */
-public interface IAppiumDeviceService extends IService<AppiumDevice> {
-
-    AppiumDevice getEnabled();
-}

+ 0 - 15
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/service/IAppiumJobBindTaskInfoService.java

@@ -1,15 +0,0 @@
-package cn.com.ctop.crawler.modules.appium.service;
-
-import cn.com.ctop.crawler.modules.appium.entity.AppiumJobBindTaskInfo;
-import com.baomidou.mybatisplus.extension.service.IService;
-
-/**
- * 爬虫调度任务绑定信息
- *
- * @author jeecg-boot
- * @version V1.0
- * @date 2019-11-14
- */
-public interface IAppiumJobBindTaskInfoService extends IService<AppiumJobBindTaskInfo> {
-
-}

+ 0 - 26
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/service/IAppiumJobService.java

@@ -1,26 +0,0 @@
-package cn.com.ctop.crawler.modules.appium.service;
-
-import cn.com.ctop.crawler.modules.appium.entity.AppiumDevice;
-import cn.com.ctop.crawler.modules.appium.entity.AppiumJob;
-import cn.com.ctop.crawler.modules.appium.entity.AppiumTaskLog;
-import com.baomidou.mybatisplus.extension.service.IService;
-
-/**
- * 爬虫调度任务
- *
- * @author jeecg-boot
- * @version V1.0
- * @date 2019-11-14
- */
-public interface IAppiumJobService extends IService<AppiumJob> {
-
-    String loginTask(Long taskId, AppiumDevice device);
-
-    boolean bindCreative(String account, String password, String creativeName, String ksId);
-
-    void loadTantanData();
-
-    boolean refreshWebPhone();
-
-    void updateStatus(AppiumDevice appiumDevice, AppiumTaskLog log, Integer logStatus, Integer deviceStatus);
-}

+ 0 - 23
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/service/IAppiumTaskItemService.java

@@ -1,23 +0,0 @@
-package cn.com.ctop.crawler.modules.appium.service;
-
-import cn.com.ctop.crawler.modules.appium.entity.AppiumTaskItem;
-import com.baomidou.mybatisplus.extension.service.IService;
-
-import java.util.List;
-
-/**
- * 任务内容表
- * @author jeecg-boot
- * @date 2019-10-31
- * @version V1.0
- */
-public interface IAppiumTaskItemService extends IService<AppiumTaskItem> {
-    /**
-     * 查询任务列表
-     *
-     * @param taskId
-     * @param parentId
-     * @return
-     */
-    List<AppiumTaskItem> initTaskItemList(Long taskId, Integer parentId);
-}

+ 0 - 15
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/service/IAppiumTaskLogService.java

@@ -1,15 +0,0 @@
-package cn.com.ctop.crawler.modules.appium.service;
-
-import cn.com.ctop.crawler.modules.appium.entity.AppiumTaskLog;
-import com.baomidou.mybatisplus.extension.service.IService;
-
-/**
- * 任务日志表
- *
- * @author jeecg-boot
- * @version V1.0
- * @date 2019-11-01
- */
-public interface IAppiumTaskLogService extends IService<AppiumTaskLog> {
-
-}

+ 0 - 22
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/service/IAppiumTaskService.java

@@ -1,22 +0,0 @@
-package cn.com.ctop.crawler.modules.appium.service;
-
-import cn.com.ctop.crawler.modules.appium.entity.AppiumTask;
-import com.baomidou.mybatisplus.extension.service.IService;
-
-/**
- * 任务表
- * @author jeecg-boot
- * @date 2019-10-31
- * @version V1.0
- */
-public interface IAppiumTaskService extends IService<AppiumTask> {
-    /**
-     * 执行任务
-     *
-     * @param taskId
-     * @param deviceId
-     */
-    public void runTask(Long taskId, Integer deviceId, String publishName);
-
-    public void runKuaishouLoginTask();
-}

+ 0 - 26
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/service/impl/AppiumDeviceServiceImpl.java

@@ -1,26 +0,0 @@
-package cn.com.ctop.crawler.modules.appium.service.impl;
-
-import cn.com.ctop.crawler.modules.appium.entity.AppiumDevice;
-import cn.com.ctop.crawler.modules.appium.mapper.AppiumDeviceMapper;
-import cn.com.ctop.crawler.modules.appium.service.IAppiumDeviceService;
-import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
-import org.springframework.stereotype.Service;
-
-import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
-
-/**
- * 手机设备表
- * @author jeecg-boot
- * @date 2019-10-31
- * @version V1.0
- */
-@Service
-public class AppiumDeviceServiceImpl extends ServiceImpl<AppiumDeviceMapper, AppiumDevice> implements IAppiumDeviceService {
-
-    @Override
-    public AppiumDevice getEnabled() {
-        QueryWrapper<AppiumDevice> queryWrapper = new QueryWrapper<>();
-        queryWrapper.eq("status", 1).orderByDesc("id").last("limit 1");
-        return getOne(queryWrapper);
-    }
-}

+ 0 - 19
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/service/impl/AppiumJobBindTaskInfoServiceImpl.java

@@ -1,19 +0,0 @@
-package cn.com.ctop.crawler.modules.appium.service.impl;
-
-import cn.com.ctop.crawler.modules.appium.entity.AppiumJobBindTaskInfo;
-import cn.com.ctop.crawler.modules.appium.mapper.AppiumJobBindTaskInfoMapper;
-import cn.com.ctop.crawler.modules.appium.service.IAppiumJobBindTaskInfoService;
-import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
-import org.springframework.stereotype.Service;
-
-/**
- * 爬虫调度任务绑定信息
- *
- * @author jeecg-boot
- * @version V1.0
- * @date 2019-11-14
- */
-@Service
-public class AppiumJobBindTaskInfoServiceImpl extends ServiceImpl<AppiumJobBindTaskInfoMapper, AppiumJobBindTaskInfo> implements IAppiumJobBindTaskInfoService {
-
-}

+ 0 - 259
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/service/impl/AppiumJobServiceImpl.java

@@ -1,259 +0,0 @@
-package cn.com.ctop.crawler.modules.appium.service.impl;
-
-import cn.com.ctop.common.module.utils.HttpUtils2;
-import cn.com.ctop.common.module.utils.ResultMapUtils;
-import cn.com.ctop.common.module.utils.StatusCode;
-import cn.com.ctop.crawler.modules.appium.entity.*;
-import cn.com.ctop.crawler.modules.appium.mapper.AppiumJobMapper;
-import cn.com.ctop.crawler.modules.appium.mapper.AppiumTaskItemMapper;
-import cn.com.ctop.crawler.modules.appium.mapper.AppiumTaskLogMapper;
-import cn.com.ctop.crawler.modules.appium.mapper.AppiumTaskMapper;
-import cn.com.ctop.crawler.modules.appium.service.IAppiumDeviceService;
-import cn.com.ctop.crawler.modules.appium.service.IAppiumJobService;
-import cn.com.ctop.crawler.modules.core.util.AppiumUtil;
-import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
-import io.appium.java_client.android.AndroidDriver;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.http.impl.client.BasicCookieStore;
-import org.jeecg.common.util.DateUtils;
-import org.openqa.selenium.By;
-import org.openqa.selenium.WebDriver;
-import org.openqa.selenium.WebElement;
-import org.openqa.selenium.chrome.ChromeDriver;
-import org.openqa.selenium.chrome.ChromeOptions;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Service;
-
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * 爬虫调度任务
- *
- * @author jeecg-boot
- * @version V1.0
- * @date 2019-11-14
- */
-@Slf4j
-@Service
-public class AppiumJobServiceImpl extends ServiceImpl<AppiumJobMapper, AppiumJob> implements IAppiumJobService {
-    @Autowired
-    private AppiumTaskMapper taskMapper;
-    @Autowired
-    private AppiumTaskItemMapper taskItemMapper;
-    @Value("${jeecg.path.chrome-driver}")
-    private String chromeDriver;
-    @Autowired
-    private IAppiumDeviceService appiumDeviceService;
-    @Autowired
-    private AppiumTaskLogMapper appiumTaskLogMapper;
-
-    @Override
-    public String loginTask(Long taskId, AppiumDevice device) {
-        String kuaishouId = null;
-        try {
-            AppiumTask task = taskMapper.selectById(taskId);
-            AppiumUtil appiumUtil = new AppiumUtil();
-            AndroidDriver androidDriver = appiumUtil.initAndroidDriver("http://39.106.184.70:4723/wd/hub", device.getIp() + ":" + device.getPort(), task.getAppPackage(), task.getAppActivity());
-            Map<String, Object> result = new HashMap<>();
-            ResultMapUtils.setResultMap(result, StatusCode.KUAISHOU_CRAWLER_APP_SUCCESS.getCode());
-            boolean loop = true;
-            while (loop) {
-                Integer code = (Integer) result.get("code");
-                if (null != code && code != StatusCode.KUAISHOU_CRAWLER_APP_SUCCESS.getCode()) {
-                    kuaishouId = (String) result.get("kuaishouId");
-                    androidDriver.quit();
-                    break;
-                }
-                Thread.sleep(3000L);
-                String packageName = androidDriver.getCurrentPackage();
-                String activityName = androidDriver.currentActivity();
-                List<AppiumTaskItem> taskItems = taskItemMapper.selectByParams(packageName, activityName, taskId);
-                if (null != taskItems || taskItems.size() > 0) {
-                    for (AppiumTaskItem item : taskItems) {
-                        result = appiumUtil.runLoginTask(androidDriver, item, result);
-                    }
-                }
-                result = appiumUtil.checkMethod(androidDriver, activityName, result);
-            }
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-        return kuaishouId;
-    }
-
-    @Override
-    public boolean bindCreative(String account, String password, String creativeName, String ksId) {
-        boolean isBind = false;
-        String url = "https://ad.e.kuaishou.com/#/welcome?redirectUrl=https%3A%2F%2Fad.e.kuaishou.com%2F%23%2Findex";
-        System.getProperties().setProperty("webdriver.chrome.driver", chromeDriver);
-        ChromeOptions chromeOptions = new ChromeOptions();
-        chromeOptions.addArguments("--headless");
-        chromeOptions.addArguments("--incognito");
-        chromeOptions.addArguments("--disable-gpu");
-//        chromeOptions.addArguments("--no-sandbox");
-        chromeOptions.addArguments("--window-size=1920,1080");
-        chromeOptions.addArguments("--user-agent=" + HttpUtils2.USER_AGENT);
-        chromeOptions.setAcceptInsecureCerts(true);
-        WebDriver webDriver = new ChromeDriver(chromeOptions);
-        try {
-            Thread.sleep(3000L);
-            HttpUtils2.cookieStore = new BasicCookieStore();
-            webDriver.manage().deleteAllCookies();
-            //获取登录页面
-            webDriver.get(url);
-            Thread.sleep(3000L);
-            WebElement accountElement = webDriver.findElement(By.xpath("//div[@class='phone ']/input[@type='text']"));
-            accountElement.sendKeys(account);
-            Thread.sleep(3000L);
-            WebElement passwordElement = webDriver.findElement(By.xpath("//div[@class='password ']/input[@type='password']"));
-            passwordElement.sendKeys(password);
-            WebElement loginElement = webDriver.findElement(By.xpath("//div[@class='foot']"));
-            Thread.sleep(3000L);
-            //点击登录
-            loginElement.click();
-            Thread.sleep(3000L);
-            //获取推广按钮
-            WebElement spreadElement = webDriver.findElement(By.linkText("推广"));
-            spreadElement.click();
-            Thread.sleep(3000L);
-            //选择广告创意
-            WebElement creativeElement = webDriver.findElement(By.xpath("//div[text()='广告创意']"));
-            creativeElement.click();
-            Thread.sleep(3000L);
-            //输入创意名称,点击搜索
-            WebElement searchElement = webDriver.findElement(By.xpath("//input[@type='text']"));
-            searchElement.sendKeys(creativeName);
-            Thread.sleep(3000L);
-            WebElement searchButton = webDriver.findElement(By.xpath("//button[@class='ant-btn ant-input-search-button ant-btn-primary']"));
-            searchButton.click();
-            Thread.sleep(3000L);
-            WebElement tiyanElement = webDriver.findElement(By.linkText("体验"));
-            tiyanElement.click();
-            Thread.sleep(2000L);
-            WebElement inputKsIdElement = webDriver.findElement(By.xpath("//textarea[@placeholder='请输入快手账号…']"));
-            inputKsIdElement.sendKeys(ksId);
-            Thread.sleep(2000L);
-            WebElement chufaElement = webDriver.findElement(By.xpath("//button[@class='creative-experience-btn-enable']"));
-            chufaElement.click();
-            isBind = true;
-        } catch (Exception e) {
-            e.printStackTrace();
-            log.error("绑定体验快手账号失败");
-            isBind = false;
-        } finally {
-            webDriver.manage().deleteAllCookies();
-            webDriver.close();
-        }
-        return isBind;
-    }
-
-    public static final String GET_AGENT_DATA_URL = "https://marketing.tantanapp.com/platform/ad_statistic/channel_report/table?level=agent&interval=hour&";
-
-    @Override
-    public void loadTantanData() {
-        String url = "https://marketing.tantanapp.com/login";
-        System.getProperties().setProperty("webdriver.chrome.driver", chromeDriver);
-        ChromeOptions chromeOptions = new ChromeOptions();
-//        chromeOptions.addArguments("--headless");
-        chromeOptions.addArguments("--incognito");
-        chromeOptions.addArguments("--disable-gpu");
-        chromeOptions.addArguments("--no-sandbox");
-        chromeOptions.addArguments("--window-size=1920,1080");
-        chromeOptions.addArguments("--user-agent=" + HttpUtils2.USER_AGENT);
-        chromeOptions.setAcceptInsecureCerts(true);
-        WebDriver webDriver = new ChromeDriver(chromeOptions);
-        try {
-            HttpUtils2.cookieStore = new BasicCookieStore();
-            webDriver.manage().deleteAllCookies();
-            //获取登录页面
-            webDriver.get(url);
-            Thread.sleep(1000L);
-            WebElement accountElement = webDriver.findElement(By.xpath("//input[@class='form-control user_name']"));
-            accountElement.sendKeys("huichuang");
-            Thread.sleep(1000L);
-            WebElement passwordElement = webDriver.findElement(By.xpath("//input[@class='form-control user_password']"));
-            passwordElement.sendKeys("2PitTUSFG7AgBWVfbnIC");
-            WebElement loginElement = webDriver.findElement(By.xpath("//button[@class='btn btn-primary block full-width m-b user_button']"));
-            Thread.sleep(2000L);
-            //点击登录
-            loginElement.click();
-            Thread.sleep(2000L);
-            String nowDate = DateUtils.formatDate(new Date());
-            String stDate = nowDate + " 00:00:00";
-            String etDate = nowDate + " 23:59:59";
-            webDriver.get(GET_AGENT_DATA_URL + "st=" + stDate + "&et=" + etDate);
-            Thread.sleep(1000L);
-            String getData = webDriver.findElement(By.xpath("//pre[@style='word-wrap: break-word; white-space: pre-wrap;']")).getText();
-        } catch (Exception e) {
-            e.printStackTrace();
-            log.error("爬取代理商数据异常");
-        } finally {
-            webDriver.manage().deleteAllCookies();
-            webDriver.close();
-        }
-    }
-
-    @Override
-    public boolean refreshWebPhone() {
-        boolean success = false;
-        String url = "https://lcloud.longene.com.cn/";
-        System.getProperties().setProperty("webdriver.chrome.driver", chromeDriver);
-        ChromeOptions chromeOptions = new ChromeOptions();
-        chromeOptions.addArguments("--headless");
-        chromeOptions.addArguments("--incognito");
-        chromeOptions.addArguments("--disable-gpu");
-//        chromeOptions.addArguments("--no-sandbox");
-        chromeOptions.addArguments("--window-size=1920,1080");
-        chromeOptions.addArguments("--user-agent=" + HttpUtils2.USER_AGENT);
-        chromeOptions.setAcceptInsecureCerts(true);
-        WebDriver webDriver = new ChromeDriver(chromeOptions);
-        try {
-            Thread.sleep(3000L);
-            HttpUtils2.cookieStore = new BasicCookieStore();
-            webDriver.manage().deleteAllCookies();
-            //获取登录页面
-            webDriver.get(url);
-            Thread.sleep(3000L);
-            WebElement accountElement = webDriver.findElement(By.id("username"));
-            accountElement.sendKeys("18600471989");
-            Thread.sleep(3000L);
-            WebElement passwordElement = webDriver.findElement(By.id("userpwd"));
-            passwordElement.sendKeys("0759125184xu");
-            WebElement loginElement = webDriver.findElement(By.xpath("//input[@class='quc-submit quc-button quc-button-primary quc-button-sign-in']"));
-            Thread.sleep(3000L);
-            //点击登录
-            loginElement.click();
-            Thread.sleep(3000L);
-            //选择华东二站
-            WebElement spreadElement = webDriver.findElement(By.xpath("//a[@href='javascript:gotobranch(2)']"));
-            spreadElement.click();
-            Thread.sleep(3000L);
-            //
-            webDriver.get("https://ecsite.longene.com.cn/userWeb!batchipswitch?ids=4293&groupid=8729&mode=5&ipbind=0&region=0&ipareas=");
-            Thread.sleep(5000L);
-            webDriver.get("https://ecsite.longene.com.cn/userWeb!batchnewphone?groupid=8729&pkgnames=com.smile.gifmaker&ids=4293");
-            Thread.sleep(5000L);
-            success = true;
-        } catch (Exception e) {
-            success = false;
-            e.printStackTrace();
-            log.error("一键新机操作失败");
-        } finally {
-            webDriver.manage().deleteAllCookies();
-            webDriver.close();
-        }
-        return success;
-    }
-
-    @Override
-    public void updateStatus(AppiumDevice appiumDevice, AppiumTaskLog log, Integer logStatus, Integer deviceStatus) {
-        log.setStatus(logStatus + "");
-        appiumTaskLogMapper.updateById(log);
-        appiumDevice.setStatus(deviceStatus);
-        appiumDeviceService.updateById(appiumDevice);
-    }
-}

+ 0 - 41
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/service/impl/AppiumTaskItemServiceImpl.java

@@ -1,41 +0,0 @@
-package cn.com.ctop.crawler.modules.appium.service.impl;
-
-import cn.com.ctop.crawler.modules.appium.entity.AppiumTaskItem;
-import cn.com.ctop.crawler.modules.appium.mapper.AppiumTaskItemMapper;
-import cn.com.ctop.crawler.modules.appium.service.IAppiumTaskItemService;
-import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
-import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
-import org.springframework.stereotype.Service;
-
-import java.util.List;
-
-/**
- * 任务内容表
- * @author jeecg-boot
- * @date 2019-10-31
- * @version V1.0
- */
-@Service
-public class AppiumTaskItemServiceImpl extends ServiceImpl<AppiumTaskItemMapper, AppiumTaskItem> implements IAppiumTaskItemService {
-    @Override
-    public List<AppiumTaskItem> initTaskItemList(Long taskId, Integer parentId) {
-        QueryWrapper<AppiumTaskItem> queryWrapper = new QueryWrapper<>();
-        queryWrapper.eq("task_id", taskId);
-        if (parentId == null) {
-            queryWrapper.isNull("parent_id");
-        } else {
-            queryWrapper.eq("parent_id", parentId);
-        }
-
-        queryWrapper.orderByAsc("seq");
-        List<AppiumTaskItem> list = this.list(queryWrapper);
-        if (list != null && list.size() > 0) {
-            for (int i = 0; i < list.size(); i++) {
-                AppiumTaskItem taskItem = list.get(i);
-                taskItem.setSubList(initTaskItemList(taskItem.getTaskId(), taskItem.getId()));
-                list.set(i, taskItem);
-            }
-        }
-        return list;
-    }
-}

+ 0 - 20
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/service/impl/AppiumTaskLogServiceImpl.java

@@ -1,20 +0,0 @@
-package cn.com.ctop.crawler.modules.appium.service.impl;
-
-import cn.com.ctop.crawler.modules.appium.entity.AppiumTaskLog;
-import cn.com.ctop.crawler.modules.appium.mapper.AppiumTaskLogMapper;
-import cn.com.ctop.crawler.modules.appium.service.IAppiumTaskLogService;
-import org.springframework.stereotype.Service;
-
-import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
-
-/**
- * 任务日志表
- *
- * @author jeecg-boot
- * @version V1.0
- * @date 2019-11-01
- */
-@Service
-public class AppiumTaskLogServiceImpl extends ServiceImpl<AppiumTaskLogMapper, AppiumTaskLog> implements IAppiumTaskLogService {
-
-}

+ 0 - 90
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/service/impl/AppiumTaskServiceImpl.java

@@ -1,90 +0,0 @@
-package cn.com.ctop.crawler.modules.appium.service.impl;
-
-import cn.com.ctop.crawler.modules.appium.entity.AppiumDevice;
-import cn.com.ctop.crawler.modules.appium.entity.AppiumTask;
-import cn.com.ctop.crawler.modules.appium.entity.AppiumTaskItem;
-import cn.com.ctop.crawler.modules.appium.entity.AppiumTaskLog;
-import cn.com.ctop.crawler.modules.appium.mapper.AppiumTaskLogMapper;
-import cn.com.ctop.crawler.modules.appium.mapper.AppiumTaskMapper;
-import cn.com.ctop.crawler.modules.appium.service.IAppiumDeviceService;
-import cn.com.ctop.crawler.modules.appium.service.IAppiumTaskItemService;
-import cn.com.ctop.crawler.modules.appium.service.IAppiumTaskService;
-import cn.com.ctop.crawler.modules.core.util.AppiumUtil;
-import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
-import io.appium.java_client.android.AndroidDriver;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Service;
-
-import java.util.List;
-
-/**
- * 任务表
- * @author jeecg-boot
- * @date 2019-10-31
- * @version V1.0
- */
-@Service
-public class AppiumTaskServiceImpl extends ServiceImpl<AppiumTaskMapper, AppiumTask> implements IAppiumTaskService {
-    @Value("${kuaishou.group-control.path}")
-    private String groupControlPath;
-    @Value("${kuaishou.group-control.port}")
-    private String groupControlPort;
-    @Autowired
-    private IAppiumTaskItemService iAppiumTaskItemService;
-    @Autowired
-    private IAppiumDeviceService iAppiumDeviceService;
-    @Autowired
-    private AppiumTaskLogMapper appiumTaskLogMapper;
-
-    @Override
-    public void runKuaishouLoginTask() {
-        AppiumUtil appiumUtil = new AppiumUtil();
-        try {
-            AndroidDriver androidDriver = appiumUtil.initAndroidDriver("http://39.106.184.70:4723/wd/hub", "183.131.157.66:10005", "com.smile.gifmaker", "com.yxcorp.gifshow.HomeActivity");
-            appiumUtil.loginTask(androidDriver, null, null);
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-    }
-
-    @Override
-    public void runTask(Long taskId, Integer deviceId, String publishName) {
-        //获取当前需要执行的任务信息
-        AppiumTask task = this.getById(taskId);
-        //获取手机设备信息
-        AppiumDevice appiumDevice = iAppiumDeviceService.getById(deviceId);
-        appiumDevice.setStatus(2);
-        iAppiumDeviceService.updateById(appiumDevice);
-        AppiumTaskLog log = new AppiumTaskLog();
-        log.setDeviceId(deviceId);
-        log.setDeviceIp(appiumDevice.getIp());
-        log.setDevicePort(appiumDevice.getPort());
-        log.setStatus(1 + "");
-        log.setTaskId(taskId);
-        log.setTaskName(task.getTaskName());
-        appiumTaskLogMapper.insert(log);
-        log.setId(log.getId());
-        //查询所需运行的任务列表
-        List<AppiumTaskItem> taskItemList = iAppiumTaskItemService.initTaskItemList(taskId, null);
-        AppiumUtil appiumUtil = new AppiumUtil();
-        try {
-            AndroidDriver androidDriver = appiumUtil.initAndroidDriver(groupControlPath + ":" + groupControlPort + "/wd/hub", appiumDevice.getIp() + ":" + appiumDevice.getPort(), task.getAppPackage(), task.getAppActivity());
-            if (taskItemList != null && taskItemList.size() > 0) {
-                for (AppiumTaskItem appiumTaskItem : taskItemList) {
-                    appiumUtil.runTask(androidDriver, appiumTaskItem, null, publishName);
-                }
-                log.setStatus(2 + "");
-                appiumTaskLogMapper.updateById(log);
-                appiumDevice.setStatus(1);
-                iAppiumDeviceService.updateById(appiumDevice);
-            }
-        } catch (Exception e) {
-            e.printStackTrace();
-            log.setStatus(3 + "");
-            appiumTaskLogMapper.updateById(log);
-            appiumDevice.setStatus(1);
-            iAppiumDeviceService.updateById(appiumDevice);
-        }
-    }
-}

+ 0 - 181
module-crawler/src/main/java/cn/com/ctop/crawler/modules/core/util/AppiumUtil.java

@@ -2,14 +2,11 @@ package cn.com.ctop.crawler.modules.core.util;
 
 import cn.com.ctop.common.module.utils.ResultMapUtils;
 import cn.com.ctop.common.module.utils.StatusCode;
-import cn.com.ctop.crawler.modules.appium.entity.AppiumTaskItem;
-import com.google.gson.Gson;
 import io.appium.java_client.TouchAction;
 import io.appium.java_client.android.AndroidDriver;
 import io.appium.java_client.touch.WaitOptions;
 import io.appium.java_client.touch.offset.PointOption;
 import lombok.extern.slf4j.Slf4j;
-import org.openqa.selenium.Point;
 import org.openqa.selenium.WebElement;
 import org.openqa.selenium.remote.DesiredCapabilities;
 
@@ -623,184 +620,6 @@ public class AppiumUtil {
         }
     }
 
-    public Map<String, Object> runLoginTask(AndroidDriver androidDriver, AppiumTaskItem appiumTaskItem, Map<String, Object> result) throws Exception {
-        Integer code = (Integer) result.get("code");
-        if (null != code && code != StatusCode.KUAISHOU_CRAWLER_APP_SUCCESS.getCode()) {
-            return result;
-        }
-        Thread.sleep(1000L);
-        String findKey = appiumTaskItem.getFindKey();
-        if ("exit".equals(findKey)) {
-            androidDriver.quit();
-            ResultMapUtils.setResultMap(result, StatusCode.KUAISHOU_CRAWLER_APP_EXIT.getCode());
-            log.error(StatusCode.KUAISHOU_CRAWLER_APP_EXIT.getDesc());
-            return result;
-        }
-        //1:获取元素信息
-        WebElement element = getElementByType(androidDriver, appiumTaskItem.getFindType(), appiumTaskItem.getFindKey());
-        if (null == element) {
-            ResultMapUtils.setResultMap(result, StatusCode.KUAISHOU_CRAWLER_APP_ELEMENT_IS_NULL.getCode());
-            log.error("元素获取异常==>activity:{},key:{}", appiumTaskItem.getAppActivity(), "com.smile.gifmaker:id/confirm_btn");
-            return result;
-        }
-        //(1):滑动效果
-        String swipeType = appiumTaskItem.getSwapType();
-        if (null != swipeType && !"".equals(swipeType.trim())) {
-            swipeByType(androidDriver, swipeType);
-            ResultMapUtils.setResultMap(result, StatusCode.KUAISHOU_CRAWLER_APP_SUCCESS.getCode());
-            return result;
-        }
-        //点击效果
-        String clickType = appiumTaskItem.getClickType();
-        if (null != clickType && !"".equals(clickType)) {
-            String getText = oprateElementByType(element, clickType, androidDriver, appiumTaskItem);
-            if (null != getText && !"".equals(getText)) {
-                ResultMapUtils.setResultMap(result, StatusCode.KUAISHOU_CRAWLER_APP_TASK_END.getCode());
-                result.put("kuaishouId", getText.replace("快手 ID:", ""));
-            } else {
-                ResultMapUtils.setResultMap(result, StatusCode.KUAISHOU_CRAWLER_APP_SUCCESS.getCode());
-            }
-            return result;
-        }
-        ResultMapUtils.setResultMap(result, StatusCode.KUAISHOU_CRAWLER_APP_SUCCESS.getCode());
-        return result;
-    }
-
-    private String oprateElementByType(WebElement element, String clickType, AndroidDriver androidDriver, AppiumTaskItem taskItem) {
-        String getText = null;
-        if ("element".equals(clickType)) {
-            element.click();
-        } else if ("point".equals(clickType)) {
-            Point point = element.getLocation();
-            Integer x = point.getX();
-            Integer y = point.getY();
-            if (null != taskItem.getOffsiteX()) {
-                x = taskItem.getOffsiteX();
-            }
-            if (null != taskItem.getOffsiteY()) {
-                y = taskItem.getOffsiteY();
-            }
-            tapPoint(androidDriver, x, y);
-        } else if ("send".equals(clickType)) {
-            String text = taskItem.getTextEqualKey();
-            element.sendKeys(text);
-        } else if ("text".equals(clickType)) {
-            getText = element.getText();
-        }
-        return getText;
-    }
-
-    public String runTask(AndroidDriver androidDriver, AppiumTaskItem appiumTaskItem, WebElement loopElement, String publishName) throws Exception {
-        String getText = null;
-        if (appiumTaskItem != null) {
-            if (appiumTaskItem.getWaitTime() != null && appiumTaskItem.getWaitTime() > 0) {
-                Thread.sleep(appiumTaskItem.getWaitTime());
-            }
-            checkActivity(androidDriver);
-            if (appiumTaskItem.getLoopType() != null && "noloop".equals(appiumTaskItem.getLoopType())) {
-                //非循环类型任务
-                if (appiumTaskItem.getFindType() != null) {
-                    WebElement element = null;
-                    if ("loop".equals(appiumTaskItem.getFindType())) {
-                        element = loopElement;
-                    } else {
-                        element = getElementByType(androidDriver, appiumTaskItem.getFindType(), appiumTaskItem.getFindKey());
-                    }
-                    if (element != null) {
-                        if (appiumTaskItem.getTextEqualType() != null) {
-                            if ("eq".equals(appiumTaskItem.getTextEqualType())) {
-                                if (!element.getText().equals(appiumTaskItem.getTextEqualKey())) {
-                                    return getText;
-                                }
-                            } else if ("neq".equals(appiumTaskItem.getTextEqualType())) {
-                                if (element.getText() == null || element.getText().equals(appiumTaskItem.getTextEqualKey())) {
-                                    return getText;
-                                }
-                            } else if ("start".equals(appiumTaskItem.getTextEqualType())) {
-                                boolean check = checkAdvertiser(androidDriver, publishName);
-                                if (!check) {
-                                    return getText;
-                                }
-                            }
-                        }
-                        if (appiumTaskItem.getClickType() != null) {
-                            if ("element".equals(appiumTaskItem.getClickType())) {
-                                element.click();
-                            } else if ("point".equals(appiumTaskItem.getClickType())) {
-                                Point point = element.getLocation();
-                                Integer x = point.getX();
-                                Integer y = point.getY();
-                                if (null != appiumTaskItem.getOffsiteX()) {
-                                    x += appiumTaskItem.getOffsiteX();
-                                }
-                                if (null != appiumTaskItem.getOffsiteY()) {
-                                    y += appiumTaskItem.getOffsiteY();
-                                }
-                                tapPoint(androidDriver, x, y);
-                            } else if ("send".equals(appiumTaskItem.getClickType())) {
-                                String text = appiumTaskItem.getTextEqualKey();
-                                element.sendKeys(text);
-                            } else if ("text".equals(appiumTaskItem.getClickType())) {
-                                getText = element.getText();
-                            }
-                        }
-                        List<AppiumTaskItem> subTask = appiumTaskItem.getSubList();
-                        if (subTask != null && subTask.size() > 0) {
-                            for (AppiumTaskItem task : subTask) {
-                                runTask(androidDriver, task, element, publishName);
-                            }
-                        }
-                    } else {
-                        //必须执行
-                        if (appiumTaskItem.getIsMust() != null && appiumTaskItem.getIsMust() == 1) {
-                            if (appiumTaskItem.getLoopRate() != null && appiumTaskItem.getLoopRate() > 0) {
-                                Thread.sleep(appiumTaskItem.getLoopRate());
-                            }
-                            runTask(androidDriver, appiumTaskItem, element, publishName);
-                        }
-                        //非必须执行情况下
-                        if (appiumTaskItem.getClickType() != null && "point".equals(appiumTaskItem.getClickType())) {
-                            tapPoint(androidDriver, appiumTaskItem.getOffsiteX(), appiumTaskItem.getOffsiteY());
-                        }
-                    }
-                }
-                if (appiumTaskItem.getSwapType() != null) {
-                    swipeByType(androidDriver, appiumTaskItem.getSwapType());
-                }
-            } else if (appiumTaskItem.getLoopType() != null && "loop".equals(appiumTaskItem.getLoopType())) {
-                while (isLoop) {
-                    if (appiumTaskItem.getFindType() != null) {
-                        List<WebElement> list = getElementsByType(androidDriver, appiumTaskItem.getFindType(), appiumTaskItem.getFindKey());
-                        if (list != null && list.size() > 0) {
-                            for (WebElement element : list) {
-                                if (!isLoop) {
-                                    break;
-                                }
-                                if (appiumTaskItem.getLoopRate() != null && appiumTaskItem.getLoopRate() > 0) {
-                                    Thread.sleep(appiumTaskItem.getLoopRate());
-                                }
-                                List<AppiumTaskItem> subTask = appiumTaskItem.getSubList();
-                                if (subTask != null && subTask.size() > 0) {
-                                    for (AppiumTaskItem task : subTask) {
-                                        runTask(androidDriver, task, element, publishName);
-                                    }
-                                }
-                            }
-                        }
-                    }
-                    if (appiumTaskItem.getSwapType() != null) {
-                        swipeByType(androidDriver, appiumTaskItem.getSwapType());
-                    }
-                }
-
-            }
-            if (appiumTaskItem.getIsClose() != null && appiumTaskItem.getIsClose() == 1) {
-                this.isLoop = false;
-            }
-        }
-        return getText;
-    }
-
     public void backToMain(AndroidDriver androidDriver) {
         WebElement element = getElementByType(androidDriver, FIND_ELEMENT_TYPE_ID, "com.smile.gifmaker:id/back_btn");
         if (null != element) {

+ 15 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/entity/PangolinApp.java

@@ -0,0 +1,15 @@
+package cn.com.ctop.crawler.modules.pangolin.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+@Data
+@TableName("ctop_pangolin_app")
+public class PangolinApp {
+    @TableId(value = "id", type = IdType.INPUT)
+    private Long id;
+    private String name;
+    private String accountName;
+}

+ 17 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/entity/PangolinChannel.java

@@ -0,0 +1,17 @@
+package cn.com.ctop.crawler.modules.pangolin.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+@Data
+@TableName("ctop_pangolin_channel")
+public class PangolinChannel {
+    @TableId(value = "id", type = IdType.INPUT)
+    private Long id;
+    private Long appId;
+    private String name;
+    private String accountName;
+
+}

+ 21 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/entity/PangolinReportAccountDaily.java

@@ -0,0 +1,21 @@
+package cn.com.ctop.crawler.modules.pangolin.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+import java.math.BigDecimal;
+import java.util.Date;
+
+@Data
+@TableName("ctop_pangolin_report_account_daily")
+public class PangolinReportAccountDaily {
+    private Date date;
+    private String accountName;
+    private String appName;
+    private Integer externalActivation;
+    private BigDecimal retentionRatio;
+    private BigDecimal totalPrice;
+}

+ 20 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/entity/PangolinReportAppHour.java

@@ -0,0 +1,20 @@
+package cn.com.ctop.crawler.modules.pangolin.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+import java.math.BigDecimal;
+import java.util.Date;
+
+@Data
+@TableName("ctop_pangolin_report_app_hour")
+public class PangolinReportAppHour {
+    private Date date;
+    private Integer hour;
+    private Long appId;
+    private String type;
+    private BigDecimal num;
+}

+ 22 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/entity/PangolinReportBillMonth.java

@@ -0,0 +1,22 @@
+package cn.com.ctop.crawler.modules.pangolin.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+import java.math.BigDecimal;
+
+@Data
+@TableName("ctop_pangolin_report_bill_month")
+public class PangolinReportBillMonth {
+    private Integer year;
+    private Integer month;
+    private Long appId;
+    private String os;
+    private Integer sumInternalActivations;
+    private BigDecimal actualTotalPrice;
+    private BigDecimal estimatedTotalPrice;
+    private BigDecimal accountQualityScore;
+}

+ 22 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/entity/PangolinReportChannelDaily.java

@@ -0,0 +1,22 @@
+package cn.com.ctop.crawler.modules.pangolin.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+import java.math.BigDecimal;
+import java.util.Date;
+
+@Data
+@TableName("ctop_pangolin_report_channel_daily")
+public class PangolinReportChannelDaily {
+    private Date date;
+    private String channelName;
+    private String appName;
+    private String os;
+    private Integer externalActivation;
+    private BigDecimal retentionRatio;
+    private BigDecimal totalPrice;
+}

+ 46 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/job/PangolinDataJob.java

@@ -0,0 +1,46 @@
+package cn.com.ctop.crawler.modules.pangolin.job;
+
+import cn.com.ctop.common.module.entity.BindAccountLogin;
+import cn.com.ctop.common.module.service.IBindAccountLoginService;
+import cn.com.ctop.crawler.modules.pangolin.entity.PangolinApp;
+import cn.com.ctop.crawler.modules.pangolin.service.*;
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import org.jeecg.common.util.DateUtils;
+import org.quartz.Job;
+import org.quartz.JobExecutionContext;
+import org.quartz.JobExecutionException;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.util.List;
+
+public class PangolinDataJob implements Job {
+    @Autowired
+    private IBindAccountLoginService bindAccountLoginService;
+    @Autowired
+    private PangolinAppService pangolinAppService;
+    @Autowired
+    private PangolinCrawlerService pangolinCrawlerService;
+    @Override
+    public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
+        QueryWrapper<BindAccountLogin> queryWrapper = new QueryWrapper<>();
+        queryWrapper.eq("login_type","pangolin");
+        queryWrapper.eq("status",1);
+        queryWrapper.isNotNull("cookie");
+        List<BindAccountLogin> list = bindAccountLoginService.list(queryWrapper);
+        if (list != null && list.size() > 0){
+            for (BindAccountLogin bindAccountLogin:list){
+                pangolinCrawlerService.getChannelList(bindAccountLogin);
+                pangolinCrawlerService.getActivationList(bindAccountLogin, DateUtils.getNowDate("yyyy-MM-dd"),1);
+                pangolinCrawlerService.getCheckList(bindAccountLogin);
+                QueryWrapper<PangolinApp> queryWrapper1 = new QueryWrapper<>();
+                queryWrapper1.eq("account_name",bindAccountLogin.getAccountName());
+                List<PangolinApp> appList = pangolinAppService.list(queryWrapper1);
+                if(appList != null && appList.size() > 0){
+                    for (PangolinApp app : appList){
+                        pangolinCrawlerService.getRealTimeList(bindAccountLogin,app.getId());
+                    }
+                }
+            }
+        }
+    }
+}

+ 32 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/job/PangolinLoginJob.java

@@ -0,0 +1,32 @@
+package cn.com.ctop.crawler.modules.pangolin.job;
+
+import cn.com.ctop.common.module.entity.BindAccountLogin;
+import cn.com.ctop.common.module.service.IBindAccountLoginService;
+import cn.com.ctop.crawler.modules.pangolin.service.PangolinLoginService;
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import org.quartz.Job;
+import org.quartz.JobExecutionContext;
+import org.quartz.JobExecutionException;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.util.List;
+
+public class PangolinLoginJob implements Job {
+    @Autowired
+    private IBindAccountLoginService bindAccountLoginService;
+    @Autowired
+    private PangolinLoginService pangolinLoginService;
+    @Override
+    public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
+        QueryWrapper<BindAccountLogin> queryWrapper = new QueryWrapper<>();
+        queryWrapper.eq("login_type","pangolin");
+        queryWrapper.eq("status",1);
+        queryWrapper.isNull("cookie");
+        List<BindAccountLogin> list = bindAccountLoginService.list(queryWrapper);
+        if (list != null && list.size() > 0){
+            for (BindAccountLogin bindAccountLogin:list){
+                pangolinLoginService.pangolinLogin(bindAccountLogin.getAccountName(),bindAccountLogin.getPassword());
+            }
+        }
+    }
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/PangolinAppMapper.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.pangolin.mapper;
+
+import cn.com.ctop.crawler.modules.pangolin.entity.PangolinApp;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+public interface PangolinAppMapper extends BaseMapper<PangolinApp> {
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/PangolinChannelMapper.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.pangolin.mapper;
+
+import cn.com.ctop.crawler.modules.pangolin.entity.PangolinChannel;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+public interface PangolinChannelMapper extends BaseMapper<PangolinChannel> {
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/PangolinReportAccountDailyMapper.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.pangolin.mapper;
+
+import cn.com.ctop.crawler.modules.pangolin.entity.PangolinReportAccountDaily;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+public interface PangolinReportAccountDailyMapper extends BaseMapper<PangolinReportAccountDaily> {
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/PangolinReportAppHourMapper.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.pangolin.mapper;
+
+import cn.com.ctop.crawler.modules.pangolin.entity.PangolinReportAppHour;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+public interface PangolinReportAppHourMapper extends BaseMapper<PangolinReportAppHour> {
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/PangolinReportBillMonthMapper.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.pangolin.mapper;
+
+import cn.com.ctop.crawler.modules.pangolin.entity.PangolinReportBillMonth;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+public interface PangolinReportBillMonthMapper extends BaseMapper<PangolinReportBillMonth> {
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/PangolinReportChannelDailyMapper.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.pangolin.mapper;
+
+import cn.com.ctop.crawler.modules.pangolin.entity.PangolinReportChannelDaily;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+public interface PangolinReportChannelDailyMapper extends BaseMapper<PangolinReportChannelDaily> {
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/PangolinAppService.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.pangolin.service;
+
+import cn.com.ctop.crawler.modules.pangolin.entity.PangolinApp;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+public interface PangolinAppService extends IService<PangolinApp> {
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/PangolinChannelService.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.pangolin.service;
+
+import cn.com.ctop.crawler.modules.pangolin.entity.PangolinChannel;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+public interface PangolinChannelService extends IService<PangolinChannel> {
+}

+ 11 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/PangolinCrawlerService.java

@@ -0,0 +1,11 @@
+package cn.com.ctop.crawler.modules.pangolin.service;
+
+import cn.com.ctop.common.module.entity.BindAccountLogin;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+public interface PangolinCrawlerService {
+    public void getCheckList(BindAccountLogin bindAccountLogin);
+    public void getChannelList(BindAccountLogin bindAccountLogin);
+    public void getActivationList(BindAccountLogin bindAccountLogin,String date,int page);
+    public void getRealTimeList(BindAccountLogin bindAccountLogin,Long appId);
+}

+ 5 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/PangolinLoginService.java

@@ -0,0 +1,5 @@
+package cn.com.ctop.crawler.modules.pangolin.service;
+
+public interface PangolinLoginService {
+    public void pangolinLogin(String username,String password);
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/PangolinReportAccountDailyService.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.pangolin.service;
+
+import cn.com.ctop.crawler.modules.pangolin.entity.PangolinReportAccountDaily;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+public interface PangolinReportAccountDailyService extends IService<PangolinReportAccountDaily> {
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/PangolinReportAppHourService.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.pangolin.service;
+
+import cn.com.ctop.crawler.modules.pangolin.entity.PangolinReportAppHour;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+public interface PangolinReportAppHourService extends IService<PangolinReportAppHour> {
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/PangolinReportBillMonthService.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.pangolin.service;
+
+import cn.com.ctop.crawler.modules.pangolin.entity.PangolinReportBillMonth;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+public interface PangolinReportBillMonthService extends IService<PangolinReportBillMonth> {
+}

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/PangolinReportChannelDailyService.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.pangolin.service;
+
+import cn.com.ctop.crawler.modules.pangolin.entity.PangolinReportChannelDaily;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+public interface PangolinReportChannelDailyService extends IService<PangolinReportChannelDaily> {
+}

+ 13 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinAppServiceImpl.java

@@ -0,0 +1,13 @@
+package cn.com.ctop.crawler.modules.pangolin.service.impl;
+
+import cn.com.ctop.crawler.modules.pangolin.entity.PangolinApp;
+import cn.com.ctop.crawler.modules.pangolin.mapper.PangolinAppMapper;
+import cn.com.ctop.crawler.modules.pangolin.mapper.PangolinChannelMapper;
+import cn.com.ctop.crawler.modules.pangolin.service.PangolinAppService;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.springframework.stereotype.Service;
+
+@Service
+public class PangolinAppServiceImpl extends ServiceImpl<PangolinAppMapper, PangolinApp> implements PangolinAppService {
+
+}

+ 11 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinChannelServiceImpl.java

@@ -0,0 +1,11 @@
+package cn.com.ctop.crawler.modules.pangolin.service.impl;
+
+import cn.com.ctop.crawler.modules.pangolin.entity.PangolinChannel;
+import cn.com.ctop.crawler.modules.pangolin.mapper.PangolinChannelMapper;
+import cn.com.ctop.crawler.modules.pangolin.service.PangolinChannelService;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.springframework.stereotype.Service;
+
+@Service
+public class PangolinChannelServiceImpl extends ServiceImpl<PangolinChannelMapper, PangolinChannel> implements PangolinChannelService {
+}

+ 353 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinCrawlerServiceImpl.java

@@ -0,0 +1,353 @@
+package cn.com.ctop.crawler.modules.pangolin.service.impl;
+
+import cn.com.ctop.common.module.entity.BindAccountLogin;
+import cn.com.ctop.common.module.service.IBindAccountLoginService;
+import cn.com.ctop.common.module.utils.HttpUtils2;
+import cn.com.ctop.crawler.modules.pangolin.entity.*;
+import cn.com.ctop.crawler.modules.pangolin.service.*;
+import cn.hutool.core.date.DateUtil;
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.gson.Gson;
+import com.google.gson.JsonObject;
+import org.apache.http.impl.cookie.BasicClientCookie;
+import org.jeecg.common.util.DateUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.util.*;
+
+@Service
+public class PangolinCrawlerServiceImpl implements PangolinCrawlerService {
+    @Autowired
+    private IBindAccountLoginService bindAccountLoginService;
+    @Autowired
+    private PangolinAppService pangolinAppService;
+    @Autowired
+    private PangolinChannelService pangolinChannelService;
+    @Autowired
+    private PangolinReportBillMonthService pangolinReportBillMonthService;
+    @Autowired
+    private PangolinReportChannelDailyService pangolinReportChannelDailyService;
+    @Autowired
+    private PangolinReportAppHourService pangolinReportAppHourService;
+
+    public void getCheckList(BindAccountLogin bindAccountLogin){
+        String url = "https://pangolin.bytedance.com/api/check_apps/list";
+        ObjectMapper mapper = new ObjectMapper();
+        try {
+            List<String> cookieList = mapper.readValue(bindAccountLogin.getCookie(), new TypeReference<List<String>>() {});
+            if(cookieList != null && cookieList.size() > 0){
+                for (String cookieStr : cookieList){
+                    String[] cookieArray = cookieStr.split(";");
+                    Map<String, String> cookieMap = new HashMap<String, String>();
+                    for (String cookie : cookieArray) {
+                        String[] kv = cookie.split("=");
+                        if (kv.length > 1) {
+                            cookieMap.put(kv[0].trim(), kv[1].trim());
+                        }
+                    }
+                    BasicClientCookie clientCookie = new BasicClientCookie(cookieArray[0].split("=")[0],
+                            cookieArray[0].split("=")[1]);
+                    clientCookie.setDomain("pangolin.bytedance.com");
+                    clientCookie.setPath(cookieMap.get("path"));
+                    HttpUtils2.cookieStore.addCookie(clientCookie);
+                }
+            }
+            String result = HttpUtils2.httpPostRequest(url,null,null);
+            System.out.println(result);
+            if(checkLogin(bindAccountLogin,result)){
+                JsonNode resultNode = mapper.readTree(result);
+                Iterator<JsonNode> appIterator = resultNode.get("data").get("List").elements();
+                while (appIterator.hasNext()){
+                    JsonNode checkNode = appIterator.next();
+                    Long appId = checkNode.get("ApplicationId").asLong();
+                    String date = checkNode.get("Date").asText();
+                    String os = checkNode.get("Os").asText();
+                    Integer sumInternalActivations = checkNode.get("SumInternalActivations").asInt();
+                    BigDecimal actualTotalPrice = checkNode.get("ActualTotalPrice").decimalValue();
+                    BigDecimal estimatedTotalPrice = checkNode.get("EstimatedTotalPrice").decimalValue();
+                    BigDecimal accountQualityScore = checkNode.get("AccountQualityScore").decimalValue();
+                    String appName = checkNode.get("AppName").asText();
+                    PangolinReportBillMonth pangolinReportBillMonth = new PangolinReportBillMonth();
+                    pangolinReportBillMonth.setAccountQualityScore(accountQualityScore);
+                    pangolinReportBillMonth.setActualTotalPrice(actualTotalPrice);
+                    pangolinReportBillMonth.setAppId(appId);
+                    pangolinReportBillMonth.setEstimatedTotalPrice(estimatedTotalPrice);
+                    pangolinReportBillMonth.setOs(os);
+                    pangolinReportBillMonth.setSumInternalActivations(sumInternalActivations);
+                    pangolinReportBillMonth.setYear(Integer.parseInt(date.split("-")[0]));
+                    pangolinReportBillMonth.setMonth(Integer.parseInt(date.split("-")[1]));
+                    QueryWrapper<PangolinReportBillMonth> queryWrapper = new QueryWrapper<>();
+                    queryWrapper.eq("os",pangolinReportBillMonth.getOs());
+                    queryWrapper.eq("app_id",pangolinReportBillMonth.getAppId());
+                    queryWrapper.eq("month",pangolinReportBillMonth.getMonth());
+                    queryWrapper.eq("year",pangolinReportBillMonth.getYear());
+                    if(pangolinReportBillMonthService.getOne(queryWrapper) == null){
+                        pangolinReportBillMonthService.save(pangolinReportBillMonth);
+                    }else {
+                        pangolinReportBillMonthService.update(pangolinReportBillMonth,queryWrapper);
+                    }
+                }
+            }
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+
+    public void getChannelList(BindAccountLogin bindAccountLogin){
+        String url = "https://pangolin.bytedance.com/api/search/external_default";
+        ObjectMapper mapper = new ObjectMapper();
+        try {
+            List<String> cookieList = mapper.readValue(bindAccountLogin.getCookie(), new TypeReference<List<String>>() {});
+            if(cookieList != null && cookieList.size() > 0){
+                for (String cookieStr : cookieList){
+                    String[] cookieArray = cookieStr.split(";");
+                    Map<String, String> cookieMap = new HashMap<String, String>();
+                    for (String cookie : cookieArray) {
+                        String[] kv = cookie.split("=");
+                        if (kv.length > 1) {
+                            cookieMap.put(kv[0].trim(), kv[1].trim());
+                        }
+                    }
+                    BasicClientCookie clientCookie = new BasicClientCookie(cookieArray[0].split("=")[0],
+                            cookieArray[0].split("=")[1]);
+                    clientCookie.setDomain("pangolin.bytedance.com");
+                    clientCookie.setPath(cookieMap.get("path"));
+                    HttpUtils2.cookieStore.addCookie(clientCookie);
+                }
+            }
+            String result = HttpUtils2.httpGetRequest(url);
+            System.out.println(result);
+            if(checkLogin(bindAccountLogin,result)){
+                JsonNode resultNode = mapper.readTree(result);
+                Iterator<JsonNode> appIterator = resultNode.get("data").get("AppNameList").elements();
+                while (appIterator.hasNext()){
+                    JsonNode appNode = appIterator.next();
+                    Long id = appNode.get("Id").asLong();
+                    String name = appNode.get("Name").asText();
+                    PangolinApp app = new PangolinApp();
+                    app.setAccountName(bindAccountLogin.getAccountName());
+                    app.setId(id);
+                    app.setName(name);
+                    pangolinAppService.saveOrUpdate(app);
+                }
+                Iterator<JsonNode> channelIterator = resultNode.get("data").get("ChannelNameList").elements();
+                while (channelIterator.hasNext()){
+                    JsonNode channelNode = channelIterator.next();
+                    Long id = channelNode.get("Id").asLong();
+                    String name = channelNode.get("Name").asText();
+                    PangolinChannel channel = new PangolinChannel();
+                    channel.setAccountName(bindAccountLogin.getAccountName());
+                    channel.setId(id);
+                    channel.setName(name);
+                    pangolinChannelService.saveOrUpdate(channel);
+                }
+            }
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+
+    public void getActivationList(BindAccountLogin bindAccountLogin,String date,int page){
+        String url = "https://pangolin.bytedance.com/api/activation/list";
+        ObjectMapper mapper = new ObjectMapper();
+        try {
+            List<String> cookieList = mapper.readValue(bindAccountLogin.getCookie(), new TypeReference<List<String>>() {});
+            if(cookieList != null && cookieList.size() > 0){
+                for (String cookieStr : cookieList){
+                    String[] cookieArray = cookieStr.split(";");
+                    Map<String, String> cookieMap = new HashMap<String, String>();
+                    for (String cookie : cookieArray) {
+                        String[] kv = cookie.split("=");
+                        if (kv.length > 1) {
+                            cookieMap.put(kv[0].trim(), kv[1].trim());
+                        }
+                    }
+                    BasicClientCookie clientCookie = new BasicClientCookie(cookieArray[0].split("=")[0],
+                            cookieArray[0].split("=")[1]);
+                    clientCookie.setDomain("pangolin.bytedance.com");
+                    clientCookie.setPath(cookieMap.get("path"));
+                    HttpUtils2.cookieStore.addCookie(clientCookie);
+                }
+            }
+            Map<String,Object> paramMap = new HashMap<>();
+            Map<String,Object> searchMap = new HashMap<>();
+            searchMap.put("DateStart",DateUtils.addDay(date,-2));
+            searchMap.put("DateEnd",date);
+            searchMap.put("AppId","");
+            searchMap.put("ChannelName","");
+            searchMap.put("OS","");
+            Map<String,Object> pageMap = new HashMap<>();
+            pageMap.put("p",page);
+            paramMap.put("search",searchMap);
+            paramMap.put("page",pageMap);
+            String result = HttpUtils2.httpPostRequest(url,paramMap,null);
+            System.out.println(result);
+            if(checkLogin(bindAccountLogin,result)){
+                JsonNode resultNode = mapper.readTree(result);
+                Iterator<JsonNode> listIterator = resultNode.get("data").get("list").elements();
+                while (listIterator.hasNext()){
+                    JsonNode dataNode = listIterator.next();
+                    String startDate = dataNode.get("StatsDate").asText();
+                    String channelName = dataNode.get("ChannelName").asText();
+                    String appName = dataNode.get("AppVerboseName").asText();
+                    String os = dataNode.get("OS").asText();
+                    Integer externalActivation = dataNode.get("ExternalActivation").asInt();
+                    BigDecimal retentionRatio = dataNode.get("RetentionRatio").decimalValue();
+                    BigDecimal totalPrice = dataNode.get("TotalPrice").decimalValue();
+                    PangolinReportChannelDaily pangolinReportChannelDaily = new PangolinReportChannelDaily();
+                    pangolinReportChannelDaily.setAppName(appName);
+                    pangolinReportChannelDaily.setChannelName(channelName);
+                    pangolinReportChannelDaily.setDate(DateUtils.parseDate(startDate,"yyyy-MM-dd"));
+                    pangolinReportChannelDaily.setExternalActivation(externalActivation);
+                    pangolinReportChannelDaily.setOs(os);
+                    pangolinReportChannelDaily.setRetentionRatio(retentionRatio);
+                    pangolinReportChannelDaily.setTotalPrice(totalPrice);
+                    QueryWrapper<PangolinReportChannelDaily> queryWrapper = new QueryWrapper<>();
+                    queryWrapper.eq("date",pangolinReportChannelDaily.getDate());
+                    queryWrapper.eq("channel_name",pangolinReportChannelDaily.getChannelName());
+                    if(pangolinReportChannelDailyService.getOne(queryWrapper) == null){
+                        pangolinReportChannelDailyService.save(pangolinReportChannelDaily);
+                    }else {
+                        pangolinReportChannelDailyService.update(pangolinReportChannelDaily,queryWrapper);
+                    }
+                }
+                Integer maxPage = resultNode.get("data").get("page").get("MaxPage").asInt();
+                if(page < maxPage){
+                    page++;
+                    getActivationList(bindAccountLogin,date,page);
+                }
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    public void getRealTimeList(BindAccountLogin bindAccountLogin,Long appId){
+        String url = "https://pangolin.bytedance.com/api/real_time/";
+        ObjectMapper mapper = new ObjectMapper();
+        try {
+            List<String> cookieList = mapper.readValue(bindAccountLogin.getCookie(), new TypeReference<List<String>>() {});
+            if(cookieList != null && cookieList.size() > 0){
+                for (String cookieStr : cookieList){
+                    String[] cookieArray = cookieStr.split(";");
+                    Map<String, String> cookieMap = new HashMap<String, String>();
+                    for (String cookie : cookieArray) {
+                        String[] kv = cookie.split("=");
+                        if (kv.length > 1) {
+                            cookieMap.put(kv[0].trim(), kv[1].trim());
+                        }
+                    }
+                    BasicClientCookie clientCookie = new BasicClientCookie(cookieArray[0].split("=")[0],
+                            cookieArray[0].split("=")[1]);
+                    clientCookie.setDomain("pangolin.bytedance.com");
+                    clientCookie.setPath(cookieMap.get("path"));
+                    HttpUtils2.cookieStore.addCookie(clientCookie);
+                }
+            }
+            Map<String,Object> paramMap = new HashMap<>();
+            List<Long> appIds = new ArrayList<>();
+            appIds.add(appId);
+            paramMap.put("isInternal",0);
+            paramMap.put("channelName","");
+            paramMap.put("appId",appIds);
+            String result = HttpUtils2.httpPostRequest(url,paramMap,null);
+            System.out.println(result);
+            if(checkLogin(bindAccountLogin,result)){
+                JsonNode resultNode = mapper.readTree(result);
+                Iterator<String> clickIterator = resultNode.get("data").get("ChartData").get("click_num").fieldNames();
+                while (clickIterator.hasNext()){
+                    String date = clickIterator.next();
+                    JsonNode clickNode = resultNode.get("data").get("ChartData").get("click_num").get(date);
+                    int size = clickNode.size();
+                    for(int i = 0; i < size; i++){
+                        BigDecimal clickNum = clickNode.get(i).decimalValue();
+                        PangolinReportAppHour pangolinReportAppHour = new PangolinReportAppHour();
+                        pangolinReportAppHour.setAppId(appId);
+                        pangolinReportAppHour.setDate(DateUtils.parseDate(date,"yyyy-MM-dd"));
+                        pangolinReportAppHour.setNum(clickNum);
+                        pangolinReportAppHour.setType("click");
+                        pangolinReportAppHour.setHour(i);
+                        QueryWrapper<PangolinReportAppHour> queryWrapper = new QueryWrapper<>();
+                        queryWrapper.eq("date",pangolinReportAppHour.getDate());
+                        queryWrapper.eq("app_id",pangolinReportAppHour.getAppId());
+                        queryWrapper.eq("type",pangolinReportAppHour.getType());
+                        queryWrapper.eq("hour",pangolinReportAppHour.getHour());
+                        if(pangolinReportAppHourService.getOne(queryWrapper) == null){
+                            pangolinReportAppHourService.save(pangolinReportAppHour);
+                        }else {
+                            pangolinReportAppHourService.update(pangolinReportAppHour,queryWrapper);
+                        }
+//                        pangolinReportAppHourService.saveOrUpdate(pangolinReportAppHour);
+                    }
+                }
+                Iterator<String> activeIterator = resultNode.get("data").get("ChartData").get("active_num").fieldNames();
+                while (activeIterator.hasNext()){
+                    String date = activeIterator.next();
+                    JsonNode activeNode = resultNode.get("data").get("ChartData").get("active_num").get(date);
+                    int size = activeNode.size();
+                    for(int i = 0; i < size; i++){
+                        BigDecimal activeNum = activeNode.get(i).decimalValue();
+                        PangolinReportAppHour pangolinReportAppHour = new PangolinReportAppHour();
+                        pangolinReportAppHour.setAppId(appId);
+                        pangolinReportAppHour.setDate(DateUtils.parseDate(date,"yyyy-MM-dd"));
+                        pangolinReportAppHour.setNum(activeNum);
+                        pangolinReportAppHour.setType("active");
+                        pangolinReportAppHour.setHour(i);
+                        QueryWrapper<PangolinReportAppHour> queryWrapper = new QueryWrapper<>();
+                        queryWrapper.eq("date",pangolinReportAppHour.getDate());
+                        queryWrapper.eq("app_id",pangolinReportAppHour.getAppId());
+                        queryWrapper.eq("type",pangolinReportAppHour.getType());
+                        queryWrapper.eq("hour",pangolinReportAppHour.getHour());
+                        if(pangolinReportAppHourService.getOne(queryWrapper) == null){
+                            pangolinReportAppHourService.save(pangolinReportAppHour);
+                        }else {
+                            pangolinReportAppHourService.update(pangolinReportAppHour,queryWrapper);
+                        }
+                    }
+                }
+                Iterator<String> rateIterator = resultNode.get("data").get("ChartData").get("ratio").fieldNames();
+                while (rateIterator.hasNext()){
+                    String date = rateIterator.next();
+                    JsonNode rateNode = resultNode.get("data").get("ChartData").get("ratio").get(date);
+                    int size = rateNode.size();
+                    for(int i = 0; i < size; i++){
+                        BigDecimal clickNum = rateNode.get(i).decimalValue();
+                        PangolinReportAppHour pangolinReportAppHour = new PangolinReportAppHour();
+                        pangolinReportAppHour.setAppId(appId);
+                        pangolinReportAppHour.setDate(DateUtils.parseDate(date,"yyyy-MM-dd"));
+                        pangolinReportAppHour.setNum(clickNum);
+                        pangolinReportAppHour.setType("rate");
+                        pangolinReportAppHour.setHour(i);
+                        QueryWrapper<PangolinReportAppHour> queryWrapper = new QueryWrapper<>();
+                        queryWrapper.eq("date",pangolinReportAppHour.getDate());
+                        queryWrapper.eq("app_id",pangolinReportAppHour.getAppId());
+                        queryWrapper.eq("type",pangolinReportAppHour.getType());
+                        queryWrapper.eq("hour",pangolinReportAppHour.getHour());
+                        if(pangolinReportAppHourService.getOne(queryWrapper) == null){
+                            pangolinReportAppHourService.save(pangolinReportAppHour);
+                        }else {
+                            pangolinReportAppHourService.update(pangolinReportAppHour,queryWrapper);
+                        }
+                    }
+                }
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    public boolean checkLogin(BindAccountLogin bindAccountLogin,String response){
+        if(response.contains("<html>")){
+            bindAccountLogin.setCookie(null);
+            bindAccountLoginService.updateById(bindAccountLogin);
+            return false;
+        }
+        return true;
+    }
+}

+ 118 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinLoginServiceImpl.java

@@ -0,0 +1,118 @@
+package cn.com.ctop.crawler.modules.pangolin.service.impl;
+
+import cn.com.ctop.common.module.entity.BindAccountLogin;
+import cn.com.ctop.common.module.service.IBindAccountLoginService;
+import cn.com.ctop.common.module.utils.ChaojiyingUtils;
+import cn.com.ctop.common.module.utils.HttpUtils2;
+import cn.com.ctop.crawler.modules.pangolin.service.PangolinLoginService;
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.gson.Gson;
+import org.apache.http.impl.client.BasicCookieStore;
+import org.apache.http.impl.cookie.BasicClientCookie;
+import org.openqa.selenium.*;
+import org.openqa.selenium.chrome.ChromeDriver;
+import org.openqa.selenium.chrome.ChromeOptions;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+import java.util.*;
+
+
+@Service
+public class PangolinLoginServiceImpl implements PangolinLoginService {
+    @Autowired
+    private IBindAccountLoginService bindAccountLoginService;
+    @Value("${jeecg.path.chrome-driver}")
+    private String chromeDriver;
+
+    public static void main(String[] args){
+        PangolinLoginServiceImpl loginService = new PangolinLoginServiceImpl();
+        loginService.pangolinLogin("douyin_novel60","BpGT7afFWU");
+        String result = HttpUtils2.httpGetRequest("https://pangolin.bytedance.com/api/search/external_default");
+        System.out.println(result);
+    }
+    public void pangolinLogin(String username,String password) {
+        System.getProperties().setProperty("webdriver.chrome.driver", chromeDriver);
+        String url = "https://pangolin.bytedance.com/auth/login";
+        ChromeOptions chromeOptions = new ChromeOptions();
+        chromeOptions.addArguments("--headless");
+        chromeOptions.addArguments("--incognito");
+        chromeOptions.addArguments("--disable-gpu");
+        chromeOptions.addArguments("--window-size=1920,1080");
+        chromeOptions.addArguments("--user-agent=" + HttpUtils2.USER_AGENT);
+        chromeOptions.setAcceptInsecureCerts(true);
+        WebDriver webDriver = new ChromeDriver(chromeOptions);
+        try {
+            Thread.sleep(3000L);
+            HttpUtils2.cookieStore = new BasicCookieStore();
+            webDriver.manage().deleteAllCookies();
+            //获取登录页面
+            webDriver.get(url);
+            List<WebElement> list = webDriver.findElements(By.xpath("//input[@id='captcha']"));
+            do {
+                Thread.sleep(3000L);
+                WebElement accountElement = webDriver.findElement(By.xpath("//input[@id='username']"));
+                accountElement.sendKeys(username);
+                Thread.sleep(3000L);
+                WebElement passwordElement = webDriver.findElement(By.xpath("//input[@id='password']"));
+                passwordElement.sendKeys(password);
+                WebElement captchaElement = webDriver.findElement(By.xpath("//div[@class='ant-form-item-control ']/img"));
+                String outputType = captchaElement.getScreenshotAs(OutputType.BASE64);
+                String res = ChaojiyingUtils.process(outputType,1902);
+                ObjectMapper mapper = new ObjectMapper();
+                JsonNode n = mapper.readTree(res);
+                String captcha = n.get("pic_str").asText();
+                WebElement captchaInputElement = webDriver.findElement(By.xpath("//input[@id='captcha']"));
+                captchaInputElement.sendKeys(captcha);
+                WebElement loginElement = webDriver.findElement(By.xpath("//button[@type='submit']"));
+                Thread.sleep(3000L);
+                //点击登录
+                loginElement.click();
+                Thread.sleep(3000L);
+                webDriver.get("https://pangolin.bytedance.com");
+                list = webDriver.findElements(By.xpath("//input[@id='captcha']"));
+            }while (list != null && list.size() > 0);
+
+
+
+            Set<Cookie> cookies = webDriver.manage().getCookies();
+            List<String> cookieList = new ArrayList<>();
+            Iterator iterator = cookies.iterator();
+            while (iterator.hasNext()) {
+                String cookieStr = iterator.next().toString();
+                cookieList.add(cookieStr);
+                String[] cookieArray = cookieStr.split(";");
+                Map<String, String> cookieMap = new HashMap<String, String>();
+                for (String cookie : cookieArray) {
+                    String[] kv = cookie.split("=");
+                    if (kv.length > 1) {
+                        cookieMap.put(kv[0].trim(), kv[1].trim());
+                    }
+                }
+                BasicClientCookie clientCookie = new BasicClientCookie(cookieArray[0].split("=")[0],
+                        cookieArray[0].split("=")[1]);
+                clientCookie.setDomain("pangolin.bytedance.com");
+                clientCookie.setPath(cookieMap.get("path"));
+                HttpUtils2.cookieStore.addCookie(clientCookie);
+            }
+
+            QueryWrapper<BindAccountLogin> queryWrapper = new QueryWrapper<>();
+            queryWrapper.eq("account_name",username);
+            queryWrapper.eq("login_type","pangolin");
+            BindAccountLogin bindAccountLogin = bindAccountLoginService.getOne(queryWrapper);
+            if (bindAccountLogin != null){
+                bindAccountLogin.setCookie(new Gson().toJson(cookieList));
+                bindAccountLoginService.updateById(bindAccountLogin);
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        } finally {
+            webDriver.manage().deleteAllCookies();
+//            webDriver.close();
+            webDriver.quit();
+        }
+    }
+
+}

+ 11 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinReportAccountDailyServiceImpl.java

@@ -0,0 +1,11 @@
+package cn.com.ctop.crawler.modules.pangolin.service.impl;
+
+import cn.com.ctop.crawler.modules.pangolin.entity.PangolinReportAccountDaily;
+import cn.com.ctop.crawler.modules.pangolin.mapper.PangolinReportAccountDailyMapper;
+import cn.com.ctop.crawler.modules.pangolin.service.PangolinReportAccountDailyService;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.springframework.stereotype.Service;
+
+@Service
+public class PangolinReportAccountDailyServiceImpl extends ServiceImpl<PangolinReportAccountDailyMapper, PangolinReportAccountDaily> implements PangolinReportAccountDailyService {
+}

+ 11 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinReportAppHourServiceImpl.java

@@ -0,0 +1,11 @@
+package cn.com.ctop.crawler.modules.pangolin.service.impl;
+
+import cn.com.ctop.crawler.modules.pangolin.entity.PangolinReportAppHour;
+import cn.com.ctop.crawler.modules.pangolin.mapper.PangolinReportAppHourMapper;
+import cn.com.ctop.crawler.modules.pangolin.service.PangolinReportAppHourService;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.springframework.stereotype.Service;
+
+@Service
+public class PangolinReportAppHourServiceImpl extends ServiceImpl<PangolinReportAppHourMapper, PangolinReportAppHour> implements PangolinReportAppHourService {
+}

+ 11 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinReportBillMonthServiceImpl.java

@@ -0,0 +1,11 @@
+package cn.com.ctop.crawler.modules.pangolin.service.impl;
+
+import cn.com.ctop.crawler.modules.pangolin.entity.PangolinReportBillMonth;
+import cn.com.ctop.crawler.modules.pangolin.mapper.PangolinReportBillMonthMapper;
+import cn.com.ctop.crawler.modules.pangolin.service.PangolinReportBillMonthService;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.springframework.stereotype.Service;
+
+@Service
+public class PangolinReportBillMonthServiceImpl extends ServiceImpl<PangolinReportBillMonthMapper, PangolinReportBillMonth> implements PangolinReportBillMonthService {
+}

+ 11 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinReportChannelDailyServiceImpl.java

@@ -0,0 +1,11 @@
+package cn.com.ctop.crawler.modules.pangolin.service.impl;
+
+import cn.com.ctop.crawler.modules.pangolin.entity.PangolinReportChannelDaily;
+import cn.com.ctop.crawler.modules.pangolin.mapper.PangolinReportChannelDailyMapper;
+import cn.com.ctop.crawler.modules.pangolin.service.PangolinReportChannelDailyService;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.springframework.stereotype.Service;
+
+@Service
+public class PangolinReportChannelDailyServiceImpl extends ServiceImpl<PangolinReportChannelDailyMapper, PangolinReportChannelDaily> implements PangolinReportChannelDailyService {
+}

+ 24 - 16
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/controller/BatchController.java

@@ -78,6 +78,8 @@ public class BatchController {
     private IKuaiShouImageGetService imageGetService;
     @Autowired
     private IKuaiShouVideoGetService videoGetService;
+    @Autowired
+    private IKuaishouPopulationService populationService;
 
 
     /**
@@ -97,6 +99,8 @@ public class BatchController {
             }
             SpendVo spendVo = new SpendVo();
             JSONObject fundJson = iKuaishouInterfaceService.fundGet(oauthToken);
+
+
             if (!Check.isNull(fundJson)) {
                 spendVo.setBalance(fundJson.getBigDecimal("balance")); // 余额
             }
@@ -108,6 +112,15 @@ public class BatchController {
             result.setSuccess(true);
             result.setResult(spendVo);
 
+            Thread thread = new Thread() {
+                @Override
+                public void run() {
+                    iKuaishouInterfaceService.getAppList(oauthToken.getAccountId(), oauthToken.getAccessToken());
+                    iKuaishouInterfaceService.getPopulationList(accountId, oauthToken.getAccessToken());
+                }
+            };
+            thread.start();
+
         } catch (Exception e) {
             e.printStackTrace();
             result.setSuccess(false);
@@ -437,26 +450,21 @@ public class BatchController {
 
 
     /**
-     * 获取人群包接口
+     * 获取人群包
      *
      * @param accountId
-     * @returnBatch
+     * @return
      */
     @GetMapping(value = "/getPopulationList")
-    public Result<JSONArray> getAppList(Long accountId) {
-        Result<JSONArray> result = new Result<>();
-        try {
-            CtopOauthToken oauthToken = oauthTokenService.getTokenByAccountId(accountId);
-            if (Check.isNull(oauthToken)) {
-                throw new Exception("未获取到账户信息");
-            }
-            JSONArray jsonArray = iKuaishouInterfaceService.getPopulationList(accountId, oauthToken.getAccessToken());
-            result.setSuccess(true);
-            result.setResult(jsonArray);
-        } catch (Exception e) {
-            e.printStackTrace();
-            result.setSuccess(false);
-        }
+    public Result<List<KuaishouPopulation>> queryPageList(Long accountId) {
+        Result<List<KuaishouPopulation>> result = new Result();
+        QueryWrapper<KuaishouPopulation> queryWrapper = new QueryWrapper<>();
+        queryWrapper.eq("account_id", accountId);
+        queryWrapper.eq("status", 4);
+        queryWrapper.orderByDesc("put_time");
+        List<KuaishouPopulation> list = populationService.list(queryWrapper);
+        result.setSuccess(true);
+        result.setResult(list);
         return result;
     }
 

+ 76 - 77
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/controller/AppiumDeviceController.java

@@ -1,74 +1,73 @@
-package cn.com.ctop.crawler.modules.appium.controller;
+package cn.com.ctop.kuaishou.modules.batch.controller;
 
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.net.URLDecoder;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import org.jeecg.common.api.vo.Result;
-import org.jeecg.common.system.query.QueryGenerator;
-import org.jeecg.common.aspect.annotation.AutoLog;
-import org.jeecg.common.util.oConvertUtils;
-import cn.com.ctop.crawler.modules.appium.entity.AppiumDevice;
-import cn.com.ctop.crawler.modules.appium.service.IAppiumDeviceService;
-import java.util.Date;
+import cn.com.ctop.kuaishou.modules.batch.entity.KuaishouPopulation;
+import cn.com.ctop.kuaishou.modules.batch.service.IKuaishouPopulationService;
+import com.alibaba.fastjson.JSON;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.baomidou.mybatisplus.core.metadata.IPage;
 import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
 import lombok.extern.slf4j.Slf4j;
-
+import org.jeecg.common.api.vo.Result;
+import org.jeecg.common.aspect.annotation.AutoLog;
+import org.jeecg.common.system.query.QueryGenerator;
+import org.jeecg.common.util.oConvertUtils;
 import org.jeecgframework.poi.excel.ExcelImportUtil;
 import org.jeecgframework.poi.excel.def.NormalExcelConstants;
 import org.jeecgframework.poi.excel.entity.ExportParams;
 import org.jeecgframework.poi.excel.entity.ImportParams;
 import org.jeecgframework.poi.excel.view.JeecgEntityExcelView;
-
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.*;
 import org.springframework.web.multipart.MultipartFile;
 import org.springframework.web.multipart.MultipartHttpServletRequest;
 import org.springframework.web.servlet.ModelAndView;
-import com.alibaba.fastjson.JSON;
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiOperation;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URLDecoder;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
 
 /**
- * 手机设备表
+ * 人群包列表
+ *
  * @author jeecg-boot
- * @date 2019-10-31
  * @version V1.0
+ * @date 2020-05-06
  */
 @Slf4j
-@Api(tags = "手机设备表")
+@Api(tags = "人群包列表")
 @RestController
-@RequestMapping("/appium/appiumDevice")
-public class AppiumDeviceController {
+@RequestMapping("/abc/kuaishouPopulation")
+public class KuaishouPopulationController {
     @Autowired
-    private IAppiumDeviceService appiumDeviceService;
+    private IKuaishouPopulationService kuaishouPopulationService;
 
     /**
      * 分页列表查询
      *
-     * @param appiumDevice
+     * @param kuaishouPopulation
      * @param pageNo
      * @param pageSize
      * @param req
      * @return
      */
-    @AutoLog(value = "手机设备表-分页列表查询")
-    @ApiOperation(value = "手机设备表-分页列表查询", notes = "手机设备表-分页列表查询")
+    @AutoLog(value = "人群包列表-分页列表查询")
+    @ApiOperation(value = "人群包列表-分页列表查询", notes = "人群包列表-分页列表查询")
     @GetMapping(value = "/list")
-    public Result<IPage<AppiumDevice>> queryPageList(AppiumDevice appiumDevice,
-                                                     @RequestParam(name = "pageNo", defaultValue = "1") Integer pageNo,
-                                                     @RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize,
-                                                     HttpServletRequest req) {
-        Result<IPage<AppiumDevice>> result = new Result<IPage<AppiumDevice>>();
-        QueryWrapper<AppiumDevice> queryWrapper = QueryGenerator.initQueryWrapper(appiumDevice, req.getParameterMap());
-        Page<AppiumDevice> page = new Page<AppiumDevice>(pageNo, pageSize);
-        IPage<AppiumDevice> pageList = appiumDeviceService.page(page, queryWrapper);
+    public Result<IPage<KuaishouPopulation>> queryPageList(KuaishouPopulation kuaishouPopulation,
+                                                           @RequestParam(name = "pageNo", defaultValue = "1") Integer pageNo,
+                                                           @RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize,
+                                                           HttpServletRequest req) {
+        Result<IPage<KuaishouPopulation>> result = new Result<IPage<KuaishouPopulation>>();
+        QueryWrapper<KuaishouPopulation> queryWrapper = QueryGenerator.initQueryWrapper(kuaishouPopulation, req.getParameterMap());
+        Page<KuaishouPopulation> page = new Page<KuaishouPopulation>(pageNo, pageSize);
+        IPage<KuaishouPopulation> pageList = kuaishouPopulationService.page(page, queryWrapper);
         result.setSuccess(true);
         result.setResult(pageList);
         return result;
@@ -77,16 +76,16 @@ public class AppiumDeviceController {
     /**
      * 添加
      *
-     * @param appiumDevice
+     * @param kuaishouPopulation
      * @return
      */
-    @AutoLog(value = "手机设备表-添加")
-    @ApiOperation(value = "手机设备表-添加", notes = "手机设备表-添加")
+    @AutoLog(value = "人群包列表-添加")
+    @ApiOperation(value = "人群包列表-添加", notes = "人群包列表-添加")
     @PostMapping(value = "/add")
-    public Result<AppiumDevice> add(@RequestBody AppiumDevice appiumDevice) {
-        Result<AppiumDevice> result = new Result<AppiumDevice>();
+    public Result<KuaishouPopulation> add(@RequestBody KuaishouPopulation kuaishouPopulation) {
+        Result<KuaishouPopulation> result = new Result<KuaishouPopulation>();
         try {
-            appiumDeviceService.save(appiumDevice);
+            kuaishouPopulationService.save(kuaishouPopulation);
             result.success("添加成功!");
         } catch (Exception e) {
             log.error(e.getMessage(), e);
@@ -98,19 +97,19 @@ public class AppiumDeviceController {
     /**
      * 编辑
      *
-     * @param appiumDevice
+     * @param kuaishouPopulation
      * @return
      */
-    @AutoLog(value = "手机设备表-编辑")
-    @ApiOperation(value = "手机设备表-编辑", notes = "手机设备表-编辑")
+    @AutoLog(value = "人群包列表-编辑")
+    @ApiOperation(value = "人群包列表-编辑", notes = "人群包列表-编辑")
     @PutMapping(value = "/edit")
-    public Result<AppiumDevice> edit(@RequestBody AppiumDevice appiumDevice) {
-        Result<AppiumDevice> result = new Result<AppiumDevice>();
-        AppiumDevice appiumDeviceEntity = appiumDeviceService.getById(appiumDevice.getId());
-        if (appiumDeviceEntity == null) {
+    public Result<KuaishouPopulation> edit(@RequestBody KuaishouPopulation kuaishouPopulation) {
+        Result<KuaishouPopulation> result = new Result<KuaishouPopulation>();
+        KuaishouPopulation kuaishouPopulationEntity = kuaishouPopulationService.getById(kuaishouPopulation.getId());
+        if (kuaishouPopulationEntity == null) {
             result.error500("未找到对应实体");
         } else {
-            boolean ok = appiumDeviceService.updateById(appiumDevice);
+            boolean ok = kuaishouPopulationService.updateById(kuaishouPopulation);
             if (ok) {
                 result.success("修改成功!");
             }
@@ -125,12 +124,12 @@ public class AppiumDeviceController {
      * @param id
      * @return
      */
-    @AutoLog(value = "手机设备表-通过id删除")
-    @ApiOperation(value = "手机设备表-通过id删除", notes = "手机设备表-通过id删除")
+    @AutoLog(value = "人群包列表-通过id删除")
+    @ApiOperation(value = "人群包列表-通过id删除", notes = "人群包列表-通过id删除")
     @DeleteMapping(value = "/delete")
     public Result<?> delete(@RequestParam(name = "id", required = true) String id) {
         try {
-            appiumDeviceService.removeById(id);
+            kuaishouPopulationService.removeById(id);
         } catch (Exception e) {
             log.error("删除失败", e.getMessage());
             return Result.error("删除失败!");
@@ -144,15 +143,15 @@ public class AppiumDeviceController {
      * @param ids
      * @return
      */
-    @AutoLog(value = "手机设备表-批量删除")
-    @ApiOperation(value = "手机设备表-批量删除", notes = "手机设备表-批量删除")
+    @AutoLog(value = "人群包列表-批量删除")
+    @ApiOperation(value = "人群包列表-批量删除", notes = "人群包列表-批量删除")
     @DeleteMapping(value = "/deleteBatch")
-    public Result<AppiumDevice> deleteBatch(@RequestParam(name = "ids", required = true) String ids) {
-        Result<AppiumDevice> result = new Result<AppiumDevice>();
+    public Result<KuaishouPopulation> deleteBatch(@RequestParam(name = "ids", required = true) String ids) {
+        Result<KuaishouPopulation> result = new Result<KuaishouPopulation>();
         if (ids == null || "".equals(ids.trim())) {
             result.error500("参数不识别!");
         } else {
-            this.appiumDeviceService.removeByIds(Arrays.asList(ids.split(",")));
+            this.kuaishouPopulationService.removeByIds(Arrays.asList(ids.split(",")));
             result.success("删除成功!");
         }
         return result;
@@ -164,16 +163,16 @@ public class AppiumDeviceController {
      * @param id
      * @return
      */
-    @AutoLog(value = "手机设备表-通过id查询")
-    @ApiOperation(value = "手机设备表-通过id查询", notes = "手机设备表-通过id查询")
+    @AutoLog(value = "人群包列表-通过id查询")
+    @ApiOperation(value = "人群包列表-通过id查询", notes = "人群包列表-通过id查询")
     @GetMapping(value = "/queryById")
-    public Result<AppiumDevice> queryById(@RequestParam(name = "id", required = true) String id) {
-        Result<AppiumDevice> result = new Result<AppiumDevice>();
-        AppiumDevice appiumDevice = appiumDeviceService.getById(id);
-        if (appiumDevice == null) {
+    public Result<KuaishouPopulation> queryById(@RequestParam(name = "id", required = true) String id) {
+        Result<KuaishouPopulation> result = new Result<KuaishouPopulation>();
+        KuaishouPopulation kuaishouPopulation = kuaishouPopulationService.getById(id);
+        if (kuaishouPopulation == null) {
             result.error500("未找到对应实体");
         } else {
-            result.setResult(appiumDevice);
+            result.setResult(kuaishouPopulation);
             result.setSuccess(true);
         }
         return result;
@@ -188,13 +187,13 @@ public class AppiumDeviceController {
     @RequestMapping(value = "/exportXls")
     public ModelAndView exportXls(HttpServletRequest request, HttpServletResponse response) {
         // Step.1 组装查询条件
-        QueryWrapper<AppiumDevice> queryWrapper = null;
+        QueryWrapper<KuaishouPopulation> queryWrapper = null;
         try {
             String paramsStr = request.getParameter("paramsStr");
             if (oConvertUtils.isNotEmpty(paramsStr)) {
                 String deString = URLDecoder.decode(paramsStr, "UTF-8");
-                AppiumDevice appiumDevice = JSON.parseObject(deString, AppiumDevice.class);
-                queryWrapper = QueryGenerator.initQueryWrapper(appiumDevice, request.getParameterMap());
+                KuaishouPopulation kuaishouPopulation = JSON.parseObject(deString, KuaishouPopulation.class);
+                queryWrapper = QueryGenerator.initQueryWrapper(kuaishouPopulation, request.getParameterMap());
             }
         } catch (UnsupportedEncodingException e) {
             e.printStackTrace();
@@ -202,11 +201,11 @@ public class AppiumDeviceController {
 
         //Step.2 AutoPoi 导出Excel
         ModelAndView mv = new ModelAndView(new JeecgEntityExcelView());
-        List<AppiumDevice> pageList = appiumDeviceService.list(queryWrapper);
+        List<KuaishouPopulation> pageList = kuaishouPopulationService.list(queryWrapper);
         //导出文件名称
-        mv.addObject(NormalExcelConstants.FILE_NAME, "手机设备表列表");
-        mv.addObject(NormalExcelConstants.CLASS, AppiumDevice.class);
-        mv.addObject(NormalExcelConstants.PARAMS, new ExportParams("手机设备表列表数据", "导出人:Jeecg", "导出信息"));
+        mv.addObject(NormalExcelConstants.FILE_NAME, "人群包列表列表");
+        mv.addObject(NormalExcelConstants.CLASS, KuaishouPopulation.class);
+        mv.addObject(NormalExcelConstants.PARAMS, new ExportParams("人群包列表列表数据", "导出人:Jeecg", "导出信息"));
         mv.addObject(NormalExcelConstants.DATA_LIST, pageList);
         return mv;
     }
@@ -229,9 +228,9 @@ public class AppiumDeviceController {
             params.setHeadRows(1);
             params.setNeedSave(true);
             try {
-                List<AppiumDevice> listAppiumDevices = ExcelImportUtil.importExcel(file.getInputStream(), AppiumDevice.class, params);
-                appiumDeviceService.saveBatch(listAppiumDevices);
-                return Result.ok("文件导入成功!数据行数:" + listAppiumDevices.size());
+                List<KuaishouPopulation> listKuaishouPopulations = ExcelImportUtil.importExcel(file.getInputStream(), KuaishouPopulation.class, params);
+                kuaishouPopulationService.saveBatch(listKuaishouPopulations);
+                return Result.ok("文件导入成功!数据行数:" + listKuaishouPopulations.size());
             } catch (Exception e) {
                 log.error(e.getMessage(), e);
                 return Result.error("文件导入失败:" + e.getMessage());

+ 113 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/entity/KuaishouPopulation.java

@@ -0,0 +1,113 @@
+package cn.com.ctop.kuaishou.modules.batch.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import com.fasterxml.jackson.annotation.JsonFormat;
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.experimental.Accessors;
+import org.jeecgframework.poi.excel.annotation.Excel;
+import org.springframework.format.annotation.DateTimeFormat;
+
+import java.util.Date;
+
+/**
+ * 人群包列表
+ *
+ * @author jeecg-boot
+ * @version V1.0
+ * @date 2020-05-06
+ */
+@Data
+@TableName("ctop_kuaishou_population")
+@EqualsAndHashCode(callSuper = false)
+@Accessors(chain = true)
+@ApiModel(value = "ctop_kuaishou_population对象", description = "人群包列表")
+public class KuaishouPopulation {
+
+    /**
+     * id
+     */
+    @TableId(type = IdType.AUTO)
+    @ApiModelProperty(value = "id")
+    private Long id;
+    /**
+     * 账户ID
+     */
+    @Excel(name = "账户ID", width = 15)
+    @ApiModelProperty(value = "账户ID")
+    private Long accountId;
+    /**
+     * 人群包id
+     */
+    @Excel(name = "人群包id", width = 15)
+    @ApiModelProperty(value = "人群包id")
+    private Long orientationId;
+    /**
+     * 人群包名称
+     */
+    @Excel(name = "人群包名称", width = 15)
+    @ApiModelProperty(value = "人群包名称")
+    private String orientationName;
+    /**
+     * 人群数据类型0:非上传人群包1:IMEI2:IDFA3:IMEI_MD54:IDFA_MD55:手机号-MD5
+     */
+    @Excel(name = "人群数据类型0:非上传人群包1:IMEI2:IDFA3:IMEI_MD54:IDFA_MD55:手机号-MD5", width = 15)
+    @ApiModelProperty(value = "人群数据类型0:非上传人群包1:IMEI2:IDFA3:IMEI_MD54:IDFA_MD55:手机号-MD5")
+
+    private Integer populationType;
+    /**
+     * 上传数量
+     */
+    @Excel(name = "上传数量", width = 15)
+    @ApiModelProperty(value = "上传数量")
+    private Long recordSize;
+    /**
+     * 匹配数量
+     */
+    @Excel(name = "匹配数量", width = 15)
+    @ApiModelProperty(value = "匹配数量")
+    private Long matchSize;
+    /**
+     * 覆盖人群
+     */
+    @Excel(name = "覆盖人群", width = 15)
+    @ApiModelProperty(value = "覆盖人群")
+    private Long coverNum;
+
+    private java.lang.Integer status;
+    private java.lang.Integer type;
+    /**
+     * 推送时间
+     */
+    @Excel(name = "推送时间", width = 15, format = "yyyy-MM-dd")
+    @JsonFormat(timezone = "GMT+8", pattern = "yyyy-MM-dd")
+    @DateTimeFormat(pattern = "yyyy-MM-dd")
+    @ApiModelProperty(value = "推送时间")
+    private Date putTime;
+
+
+    private Long thirdPlatformCode;
+    /**
+     * 付费人群包-第3方平台code
+     * <p>
+     * private java.lang.Integer thirdPlatformCode;
+     * /**创建时间
+     */
+    @Excel(name = "创建时间", width = 20, format = "yyyy-MM-dd HH:mm:ss")
+    @JsonFormat(timezone = "GMT+8", pattern = "yyyy-MM-dd HH:mm:ss")
+    @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss")
+    @ApiModelProperty(value = "创建时间")
+    private java.util.Date createTime;
+    /**
+     * 修改时间
+     */
+    @Excel(name = "修改时间", width = 20, format = "yyyy-MM-dd HH:mm:ss")
+    @JsonFormat(timezone = "GMT+8", pattern = "yyyy-MM-dd HH:mm:ss")
+    @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss")
+    @ApiModelProperty(value = "修改时间")
+    private Date updateTime;
+}

+ 15 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/mapper/KuaishouPopulationMapper.java

@@ -0,0 +1,15 @@
+package cn.com.ctop.kuaishou.modules.batch.mapper;
+
+import cn.com.ctop.kuaishou.modules.batch.entity.KuaishouPopulation;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+/**
+ * 人群包列表
+ *
+ * @author: jeecg-boot
+ * @date: 2020-05-06
+ * @cersion: V1.0
+ */
+public interface KuaishouPopulationMapper extends BaseMapper<KuaishouPopulation> {
+
+}

+ 1 - 1
module-crawler/src/main/java/cn/com/ctop/crawler/modules/appium/mapper/xml/AppiumTaskMapper.xml

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
-<mapper namespace="cn.com.ctop.crawler.modules.appium.mapper.AppiumTaskMapper">
+<mapper namespace="cn.com.ctop.kuaishou.modules.batch.mapper.KuaishouPopulationMapper">
 
 </mapper>

+ 1 - 1
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/IKuaishouInterfaceService.java

@@ -439,7 +439,7 @@ public interface IKuaishouInterfaceService {
      * @param accessToken
      * @return
      */
-    JSONArray getPopulationList(Long accountId, String accessToken);
+    void getPopulationList(Long accountId, String accessToken);
 
 
     /**

+ 15 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/IKuaishouPopulationService.java

@@ -0,0 +1,15 @@
+package cn.com.ctop.kuaishou.modules.batch.service;
+
+import cn.com.ctop.kuaishou.modules.batch.entity.KuaishouPopulation;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+/**
+ * 人群包列表
+ *
+ * @author jeecg-boot
+ * @version V1.0
+ * @date 2020-05-06
+ */
+public interface IKuaishouPopulationService extends IService<KuaishouPopulation> {
+
+}

+ 19 - 26
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/impl/BatchServiceImpl.java

@@ -470,7 +470,6 @@ public class BatchServiceImpl implements IBatchService {
             }
 
             JSONObject unitJson = new JSONObject();
-
             unitJson.put("campaign_id", copyToCampaignId);
 
             // 资源位置
@@ -478,7 +477,6 @@ public class BatchServiceImpl implements IBatchService {
             if (!Check.isNull(scene_id)) {
                 unitJson.put("scene_id", scene_id);
             }
-
             // 资源创作方式
             if (!Check.isNull(group.getUnitType())) {
                 if (!Check.isNull(group.getUnitType())) {
@@ -826,7 +824,6 @@ public class BatchServiceImpl implements IBatchService {
             throw new Exception("入参为空");
         }
 
-
         Long accountId = requestJson.getLong("accountId");
         if (Check.isNull(accountId)) {
             throw new Exception("请输入需创建账号");
@@ -842,7 +839,6 @@ public class BatchServiceImpl implements IBatchService {
             throw new Exception("请选择广告组");
         }
 
-
         JSONObject creativeJson = new JSONObject();
         creativeJson.put("advertiser_id", accountId);
         creativeJson.put("unit_id", unitId);
@@ -851,10 +847,7 @@ public class BatchServiceImpl implements IBatchService {
         if (!Check.isNull(requestJson.get("creativeMaterialType"))) {
             creativeJson.put("creative_material_type", requestJson.get("creativeMaterialType"));
         }
-
         String action_bar_text = requestJson.getString("actionBarText");
-        // 视频id
-
         String click_track_url = requestJson.getString("clickTrackUrl");
         JSONObject returnJson = new JSONObject();
         JSONArray successArr = new JSONArray();
@@ -871,34 +864,34 @@ public class BatchServiceImpl implements IBatchService {
                         for (int j = 0; j < imageArr.size(); j++) {
                             JSONObject imageJson = imageArr.getJSONObject(j);
                             if (!Check.isNull(imageJson)) {
-
                                 String name = imageJson.getString("name");
                                 creativeJson.put("action_bar_text", action_bar_text);
                                 creativeJson.put("description", description);
-
                                 creativeJson.put("creative_name", name);
                                 creativeJson.put("photo_id", photo_id);
                                 creativeJson.put("click_track_url", click_track_url);
 
                                 String imageToken = null;
                                 String signature = imageJson.getString("signature");
-                                QueryWrapper<KuaiShouImageGet> queryWrapper = new QueryWrapper<>();
-                                queryWrapper.eq("account_id", accountId);
-                                queryWrapper.eq("signature", signature);
-                                queryWrapper.last("limit 1");
-                                KuaiShouImageGet imageGet = imageGetService.getOne(queryWrapper);
-                                if (!Check.isNull(imageGet)) {
-                                    imageToken = imageGet.getImageToken();
-                                } else {
-                                    String url = imageGetService.getUrlByCode(signature);
-                                    imageToken = this.kuauiShouImageUpload(url, signature, accountId, oauthToken.getAccessToken());
-                                }
-                                if (Check.isNull(imageToken)) {
-                                    JSONObject failJson = new JSONObject();
-                                    failJson.put("creativeName", name);
-                                    failJson.put("failMessage", "获取图片文件失败");
-                                    failArr.add(failJson);
-                                    continue;
+                                if (!Check.isNull(signature)) {
+                                    QueryWrapper<KuaiShouImageGet> queryWrapper = new QueryWrapper<>();
+                                    queryWrapper.eq("account_id", accountId);
+                                    queryWrapper.eq("signature", signature);
+                                    queryWrapper.last("limit 1");
+                                    KuaiShouImageGet imageGet = imageGetService.getOne(queryWrapper);
+                                    if (!Check.isNull(imageGet)) {
+                                        imageToken = imageGet.getImageToken();
+                                    } else {
+                                        String url = imageGetService.getUrlByCode(signature);
+                                        imageToken = this.kuauiShouImageUpload(url, signature, accountId, oauthToken.getAccessToken());
+                                    }
+                                    if (Check.isNull(imageToken)) {
+                                        JSONObject failJson = new JSONObject();
+                                        failJson.put("creativeName", name);
+                                        failJson.put("failMessage", "获取图片文件失败");
+                                        failArr.add(failJson);
+                                        continue;
+                                    }
                                 }
                                 creativeJson.put("image_token", imageToken);
                                 Map<String, Object> returnUnitMap = kuaishouInterfaceService.creativeCreate(oauthToken.getAccessToken(), accountId, creativeJson);

+ 32 - 17
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/impl/KuaishouInterfaceServiceImpl.java

@@ -132,6 +132,8 @@ public class KuaishouInterfaceServiceImpl implements IKuaishouInterfaceService {
     IKuaiShouImageGetService kuaiShouImageGetService;
     @Autowired
     private IKuaiShouDailyFlowsService kuaiShouDailyFlowsService;
+    @Autowired
+    private IKuaishouPopulationService populationService;
 
 
     @Override
@@ -1872,7 +1874,13 @@ public class KuaishouInterfaceServiceImpl implements IKuaishouInterfaceService {
                         Thread thread = new Thread() {
                             @Override
                             public void run() {
-                                getCreativeByCreativeId(accessToken, advertiserId, creativeId);
+                                try {
+                                    Thread.sleep(100);
+                                    getCreativeByCreativeId(accessToken, advertiserId, creativeId);
+                                } catch (InterruptedException e) {
+                                    e.printStackTrace();
+                                }
+
                             }
 
                         };
@@ -2820,8 +2828,10 @@ public class KuaishouInterfaceServiceImpl implements IKuaishouInterfaceService {
      * @param accessToken
      * @return
      */
+
+
     @Override
-    public JSONArray getPopulationList(Long accountId, String accessToken) {
+    public void getPopulationList(Long accountId, String accessToken) {
         try {
             String url = PropertiesUtils.getConfig("kuaishou_api_url") + KuaishouInterfaceConstant.POPULATION_LIST;
             Map<String, String> headers = new HashMap<>();
@@ -2837,36 +2847,41 @@ public class KuaishouInterfaceServiceImpl implements IKuaishouInterfaceService {
                 if (code == 0) {
                     JSONArray dataArr = resultJson.getJSONArray("data");
                     if (!Check.isNull(dataArr)) {
-                        JSONArray returnArr = new JSONArray();
+                        Map<String, Object> deleteMap = new HashMap<>();
+                        deleteMap.put("account_id", accountId);
+                        populationService.removeByMap(deleteMap);
                         for (int i = 0; i < dataArr.size(); i++) {
                             JSONObject dataJson = dataArr.getJSONObject(i);
                             if (!Check.isNull(dataJson)) {
-                                JSONObject returnJson = new JSONObject();
-                                returnJson.put("orientationId", dataJson.getLong("orientation_id"));
-                                returnJson.put("orientationName", dataJson.getString("orientation_name"));
-                                returnJson.put("type", dataJson.getInteger("type"));
-                                returnJson.put("populationType", dataJson.getLong("population_type"));
-                                returnJson.put("recordSize", dataJson.getLong("record_size"));
-                                returnJson.put("matchSize", dataJson.getLong("match_size"));
-                                returnJson.put("coverNum", dataJson.getLong("cover_num"));
-                                returnJson.put("status", dataJson.getInteger("status"));
-                                returnJson.put("createTime", dataJson.getInteger("create_time"));
-                                returnArr.add(returnJson);
+                                KuaishouPopulation population = new KuaishouPopulation();
+                                population.setOrientationId(dataJson.getLong("orientation_id"));
+                                population.setOrientationName(dataJson.getString("orientation_name"));
+                                population.setType(dataJson.getInteger("type"));
+                                population.setPopulationType(dataJson.getInteger("population_type"));
+                                population.setRecordSize(dataJson.getLong("record_size"));
+                                population.setMatchSize(dataJson.getLong("match_size"));
+                                population.setCoverNum(dataJson.getLong("cover_num"));
+                                population.setStatus(dataJson.getInteger("status"));
+                                population.setPutTime(dataJson.getDate("create_time"));
+                                population.setThirdPlatformCode(dataJson.getLong("third_platform_code"));
+                                population.setAccountId(accountId);
+                                populationService.save(population);
+
                             }
                         }
-                        return returnArr;
+
                     }
 
                 } else {
                     log.error("获取人群包管理返回数据为空,accountId:{}", accountId);
-                    return null;
+
                 }
             }
         } catch (Exception e) {
             e.printStackTrace();
         }
 
-        return null;
+
     }
 
 

+ 19 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/impl/KuaishouPopulationServiceImpl.java

@@ -0,0 +1,19 @@
+package cn.com.ctop.kuaishou.modules.batch.service.impl;
+
+import cn.com.ctop.kuaishou.modules.batch.entity.KuaishouPopulation;
+import cn.com.ctop.kuaishou.modules.batch.mapper.KuaishouPopulationMapper;
+import cn.com.ctop.kuaishou.modules.batch.service.IKuaishouPopulationService;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.springframework.stereotype.Service;
+
+/**
+ * 人群包列表
+ *
+ * @author jeecg-boot
+ * @version V1.0
+ * @date 2020-05-06
+ */
+@Service
+public class KuaishouPopulationServiceImpl extends ServiceImpl<KuaishouPopulationMapper, KuaishouPopulation> implements IKuaishouPopulationService {
+
+}