syh 5 лет назад
Родитель
Сommit
a01a9deea0
74 измененных файлов с 1124 добавлено и 1469 удалено
  1. 4 3
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/ByteDanceCleanMaterialJob.java
  2. 0 42
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/TestJob.java
  3. 0 21
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/TestXxlJob.java
  4. 0 4
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/mapper/CleanMaterialMapper.java
  5. 0 13
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/service/ICleanMaterialService.java
  6. 0 19
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/service/impl/ByteDanceCleanMaterialReportServiceImpl.java
  7. 1 28
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/service/impl/CleanMaterialServiceImpl.java
  8. 0 4
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/core/service/IHttpService.java
  9. 0 9
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/core/service/impl/HttpServiceImpl.java
  10. 0 3
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/core/service/impl/KuaishouCrawlerServiceImpl.java
  11. 0 228
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/log/vue/CrawlerLogList.vue
  12. 0 190
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/log/vue/modules/CrawlerLogModal.vue
  13. 0 197
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/log/vue/modules/CrawlerLogModal__Style#Drawer.vue
  14. 0 23
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/test/AppiumTask.java
  15. 0 461
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/test/AppiumTest.java
  16. 23 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/material/service/IBytedanceCleanMaterialService.java
  17. 45 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/material/service/impl/BytedanceCleanMaterialServiceImpl.java
  18. 1 1
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/entity/ByteDanceCleanMaterialReport.java
  19. 2 2
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/mapper/ByteDanceCleanMaterialReportMapper.java
  20. 5 1
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/BytedanceCreativeDailyReportMapper.java
  21. 2 2
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/mapper/xml/ByteDanceCleanMaterialReportMapper.xml
  22. 24 1
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/xml/BytedanceCreativeDailyReportMapper.xml
  23. 8 2
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/service/IByteDanceCleanMaterialReportService.java
  24. 54 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/ByteDanceCleanMaterialReportServiceImpl.java
  25. 1 1
      pom.xml
  26. 0 48
      xxl-job-bytedance-executor/src/main/java/com/xxl/job/executor/handler/bytedance/ByteDanceCleanMaterialJob.java
  27. 7 1
      xxl-job-bytedance-executor/pom.xml
  28. 0 0
      xxl-job-executor/src/main/java/com/xxl/job/executor/XxlJobExecutorApplication.java
  29. 0 0
      xxl-job-executor/src/main/java/com/xxl/job/executor/config/MybatisPlusConfig.java
  30. 0 0
      xxl-job-executor/src/main/java/com/xxl/job/executor/config/RedisConfig.java
  31. 0 0
      xxl-job-executor/src/main/java/com/xxl/job/executor/config/WebMvcConfiguration.java
  32. 0 0
      xxl-job-executor/src/main/java/com/xxl/job/executor/config/XxlJobConfig.java
  33. 0 0
      xxl-job-executor/src/main/java/com/xxl/job/executor/handler/SampleXxlJob.java
  34. 38 0
      xxl-job-executor/src/main/java/com/xxl/job/executor/handler/bytedance/ByteDanceCleanMaterialJob.java
  35. 4 14
      xxl-job-bytedance-executor/src/main/java/com/xxl/job/executor/handler/bytedance/ByteDanceHourlyAccountWarningLoadJob.java
  36. 2 10
      xxl-job-bytedance-executor/src/main/java/com/xxl/job/executor/handler/bytedance/BytedanceAdvertiserDailyReportLoadJob.java
  37. 7 12
      xxl-job-bytedance-executor/src/main/java/com/xxl/job/executor/handler/bytedance/BytedanceAdvertiserHourlyReportLoadJob.java
  38. 11 14
      xxl-job-bytedance-executor/src/main/java/com/xxl/job/executor/handler/bytedance/BytedanceAdvertisingDataLoadJob.java
  39. 1 9
      xxl-job-bytedance-executor/src/main/java/com/xxl/job/executor/handler/bytedance/BytedanceCreativeDailyReportLoadJob.java
  40. 6 10
      xxl-job-bytedance-executor/src/main/java/com/xxl/job/executor/handler/bytedance/BytedanceCreativeHourlyReportLoadJob.java
  41. 1 4
      xxl-job-bytedance-executor/src/main/java/com/xxl/job/executor/handler/bytedance/BytedancePlanDailyReportLoadJob.java
  42. 10 13
      xxl-job-bytedance-executor/src/main/java/com/xxl/job/executor/handler/bytedance/BytedancePlanHourlyReportLoadJob.java
  43. 1 14
      xxl-job-bytedance-executor/src/main/java/com/xxl/job/executor/handler/bytedance/BytedancePlanLoadJob.java
  44. 2 15
      xxl-job-bytedance-executor/src/main/java/com/xxl/job/executor/handler/bytedance/BytedancePlanRealTimeLoadJob.java
  45. 2 25
      xxl-job-bytedance-executor/src/main/java/com/xxl/job/executor/handler/bytedance/BytedanceSameDayCreativeLoadJob.java
  46. 1 25
      xxl-job-bytedance-executor/src/main/java/com/xxl/job/executor/handler/bytedance/BytedanceVideoInfoLoadJob.java
  47. 34 0
      xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouCampaignJob.java
  48. 46 0
      xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouDailyAccountReportLoadJob.java
  49. 45 0
      xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouDailyCampaignReportLoadJob.java
  50. 64 0
      xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouDailyCreativeReportLoadJob.java
  51. 45 0
      xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouDailyGroupReportLoadJob.java
  52. 47 0
      xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouDailyLoadFileJob.java
  53. 41 0
      xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouDailyReportTaskJob.java
  54. 34 0
      xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouGroupJob.java
  55. 26 0
      xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouHistoryLoadFileJob.java
  56. 52 0
      xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouHourlyAccountReportLoadJob.java
  57. 49 0
      xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouHourlyAccountWarningLoadJob.java
  58. 53 0
      xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouHourlyCampaignReportLoadJob.java
  59. 53 0
      xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouHourlyCreativeReportLoadJob.java
  60. 52 0
      xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouHourlyGroupReportLoadJob.java
  61. 43 0
      xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouMaterialsLoadJob.java
  62. 42 0
      xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouNowDateCampaignJob.java
  63. 40 0
      xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouNowDateCreativeJob.java
  64. 41 0
      xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouNowDateGroupJob.java
  65. 21 0
      xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouReportDailyGroupChannelCodeJob.java
  66. 33 0
      xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouSuZaoLoadJob.java
  67. 0 0
      xxl-job-executor/src/main/resources/application-dev.yml
  68. 0 0
      xxl-job-executor/src/main/resources/application-prod.yml
  69. 0 0
      xxl-job-executor/src/main/resources/application-test.yml
  70. 0 0
      xxl-job-executor/src/main/resources/application.yml
  71. 0 0
      xxl-job-executor/src/main/resources/logback-dev.xml
  72. 0 0
      xxl-job-executor/src/main/resources/logback-prod.xml
  73. 0 0
      xxl-job-executor/src/main/resources/logback-test.xml
  74. 0 0
      xxl-job-executor/src/main/resources/logback.xml

+ 4 - 3
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/ByteDanceCleanMaterialJob.java

@@ -1,6 +1,7 @@
 package org.jeecg.modules.ctop.job;
 
 import cn.com.ctop.common.module.utils.Check;
+import cn.com.ctop.toutiao.modules.report.service.IByteDanceCleanMaterialReportService;
 import lombok.extern.slf4j.Slf4j;
 import org.jeecg.modules.ctop.service.ICleanMaterialService;
 import org.quartz.Job;
@@ -17,7 +18,7 @@ import java.util.concurrent.Executors;
 @Slf4j
 public class ByteDanceCleanMaterialJob implements Job {
     @Autowired
-    private ICleanMaterialService cleanMaterialService;
+    private IByteDanceCleanMaterialReportService byteDanceCleanMaterialReportService;
     static ExecutorService executorService = null;
 
     @Override
@@ -25,7 +26,7 @@ public class ByteDanceCleanMaterialJob implements Job {
         Thread thread = new Thread() {
             @Override
             public void run() {
-                List<String> signatureList = cleanMaterialService.getByteDanceSignature();
+                List<String> signatureList = byteDanceCleanMaterialReportService.getSignature();
                 if (Check.isNull(signatureList)) {
                     log.error("头条素材数据获取为空");
                     return;
@@ -36,7 +37,7 @@ public class ByteDanceCleanMaterialJob implements Job {
                     executorService.submit(new Runnable() {
                         @Override
                         public void run() {
-                            cleanMaterialService.calculationByteDanceReport(signature);
+                            byteDanceCleanMaterialReportService.calculationMaterialReport(signature);
                         }
                     });
                 }

+ 0 - 42
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/TestJob.java

@@ -1,42 +0,0 @@
-package org.jeecg.modules.ctop.job;
-
-import org.openqa.selenium.WebDriver;
-import org.openqa.selenium.firefox.FirefoxDriver;
-import org.openqa.selenium.firefox.FirefoxOptions;
-import org.openqa.selenium.firefox.GeckoDriverService;
-import org.quartz.Job;
-import org.quartz.JobExecutionContext;
-import org.quartz.JobExecutionException;
-
-import java.io.File;
-import java.util.concurrent.TimeUnit;
-
-/**
- * @author jeecg-boot
- */
-public class TestJob implements Job {
-    @Override
-    public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
-        System.setProperty("webdriver.gecko.driver", "/usr/lib/geckodriver");
-        GeckoDriverService service = new GeckoDriverService.Builder().usingDriverExecutable(new File("/usr/lib/geckodriver")).usingAnyFreePort().build();
-        FirefoxOptions options = new FirefoxOptions();
-        options.addArguments("--headless");
-        options.addArguments("--incognito");
-        options.addArguments("--no-sandbox");
-        options.addArguments("--disable-gpu");
-        options.addArguments("--window-size=1290,1080");
-        WebDriver driver = new FirefoxDriver(options);
-        try {
-            driver.manage().timeouts().pageLoadTimeout(5, TimeUnit.SECONDS);
-            driver.manage().timeouts().setScriptTimeout(5, TimeUnit.SECONDS);
-            driver.get("http://b.e.kuaishou.com");
-        } catch (Exception e) {
-            e.printStackTrace();
-        } finally {
-            driver.quit();
-            service.stop();
-        }
-
-
-    }
-}

+ 0 - 21
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/TestXxlJob.java

@@ -1,21 +0,0 @@
-//package org.jeecg.modules.ctop.job;
-//
-//import com.xxl.job.core.biz.model.ReturnT;
-//import com.xxl.job.core.handler.annotation.XxlJob;
-//import lombok.extern.slf4j.Slf4j;
-//import org.springframework.stereotype.Component;
-//
-//import java.util.concurrent.TimeUnit;
-//
-//@Component
-//@Slf4j
-//public class TestXxlJob{
-//
-//    @XxlJob("testJob")
-//    public ReturnT<String> execute(String s) throws Exception {
-//        log.info("开始执行");
-//        TimeUnit.SECONDS.sleep(2);
-//        log.info("执行完成");
-//        return ReturnT.SUCCESS;
-//    }
-//}

+ 0 - 4
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/mapper/CleanMaterialMapper.java

@@ -1,7 +1,6 @@
 package org.jeecg.modules.ctop.mapper;
 
 import org.apache.ibatis.annotations.Param;
-import org.jeecg.modules.ctop.entity.ByteDanceCleanMaterialReport;
 import org.jeecg.modules.ctop.entity.KuaiShouCleanMaterialReport;
 
 import java.util.List;
@@ -11,7 +10,4 @@ public interface CleanMaterialMapper {
 
     KuaiShouCleanMaterialReport calculationMaterialReport(@Param("signature") String signature);
 
-    List<String> getByteDanceSignatureList();
-
-    ByteDanceCleanMaterialReport calculationByteDanceMaterialReport(String signature);
 }

+ 0 - 13
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/service/ICleanMaterialService.java

@@ -15,17 +15,4 @@ public interface ICleanMaterialService {
      * @param signature
      */
     void calculationReport(String signature);
-
-    /**
-     * 头条 唯一素材MD5标识
-     * @return
-     */
-    List<String> getByteDanceSignature();
-
-    /**
-     * 计算头条素材报表
-     *
-     * @param signature
-     */
-    void calculationByteDanceReport(String signature);
 }

+ 0 - 19
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/service/impl/ByteDanceCleanMaterialReportServiceImpl.java

@@ -1,19 +0,0 @@
-package org.jeecg.modules.ctop.service.impl;
-
-import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
-import org.jeecg.modules.ctop.entity.ByteDanceCleanMaterialReport;
-import org.jeecg.modules.ctop.mapper.ByteDanceCleanMaterialReportMapper;
-import org.jeecg.modules.ctop.service.IByteDanceCleanMaterialReportService;
-import org.springframework.stereotype.Service;
-
-/**
- * 头条-数据清洗
- *
- * @author jeecg-boot
- * @version V1.0
- * @date 2020-02-25
- */
-@Service
-public class ByteDanceCleanMaterialReportServiceImpl extends ServiceImpl<ByteDanceCleanMaterialReportMapper, ByteDanceCleanMaterialReport> implements IByteDanceCleanMaterialReportService {
-
-}

+ 1 - 28
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/service/impl/CleanMaterialServiceImpl.java

@@ -2,10 +2,9 @@ package org.jeecg.modules.ctop.service.impl;
 
 import cn.com.ctop.common.module.utils.Check;
 import cn.com.ctop.kuaishou.modules.report.service.IKuaishouReportDailyCreativeService;
-import org.jeecg.modules.ctop.entity.ByteDanceCleanMaterialReport;
+import cn.com.ctop.toutiao.modules.report.service.IByteDanceCleanMaterialReportService;
 import org.jeecg.modules.ctop.entity.KuaiShouCleanMaterialReport;
 import org.jeecg.modules.ctop.mapper.CleanMaterialMapper;
-import org.jeecg.modules.ctop.service.IByteDanceCleanMaterialReportService;
 import org.jeecg.modules.ctop.service.ICleanMaterialService;
 import org.jeecg.modules.ctop.service.IKuaiShouCleanMaterialReportService;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -50,30 +49,4 @@ public class CleanMaterialServiceImpl implements ICleanMaterialService {
             cleanMaterialReportService.saveOrUpdate(calculationMaterialReport);
         }
     }
-
-    /**
-     * 头条MD5 集合
-     *
-     * @return
-     */
-    @Override
-    public List<String> getByteDanceSignature() {
-        return cleanMaterialMapper.getByteDanceSignatureList();
-    }
-
-    /**
-     * 头条素材报表
-     *
-     * @param signature
-     */
-    @Override
-    public void calculationByteDanceReport(String signature) {
-        if (Check.isNull(signature)) {
-            return;
-        }
-        ByteDanceCleanMaterialReport calculationMaterialReport = cleanMaterialMapper.calculationByteDanceMaterialReport(signature);
-        if (!Check.isNull(calculationMaterialReport)) {
-            byteDanceCleanMaterialReportService.saveOrUpdate(calculationMaterialReport);
-        }
-    }
 }

+ 0 - 4
module-crawler/src/main/java/cn/com/ctop/crawler/modules/core/service/IHttpService.java

@@ -1,4 +0,0 @@
-package cn.com.ctop.crawler.modules.core.service;
-
-public interface IHttpService {
-}

+ 0 - 9
module-crawler/src/main/java/cn/com/ctop/crawler/modules/core/service/impl/HttpServiceImpl.java

@@ -1,9 +0,0 @@
-package cn.com.ctop.crawler.modules.core.service.impl;
-
-import cn.com.ctop.crawler.modules.core.service.IHttpService;
-import org.springframework.stereotype.Service;
-
-@Service
-public class HttpServiceImpl implements IHttpService {
-
-}

+ 0 - 3
module-crawler/src/main/java/cn/com/ctop/crawler/modules/core/service/impl/KuaishouCrawlerServiceImpl.java

@@ -9,7 +9,6 @@ import cn.com.ctop.crawler.modules.account.service.IKuaishouAppAccountService;
 import cn.com.ctop.crawler.modules.core.service.IKuaishouCrawlerService;
 import cn.com.ctop.crawler.modules.core.util.KuaishouUtil;
 import cn.com.ctop.crawler.modules.log.entity.CrawlerLog;
-import cn.com.ctop.crawler.modules.log.service.ICrawlerLogService;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
@@ -18,8 +17,6 @@ import java.util.Random;
 @Service
 public class KuaishouCrawlerServiceImpl implements IKuaishouCrawlerService {
     @Autowired
-    private ICrawlerLogService crawlerLogService;
-    @Autowired
     private IIpPoolService iIpPoolService;
     @Autowired
     private IKuaishouAppAccountService kuaishouAppAccountService;

+ 0 - 228
module-crawler/src/main/java/cn/com/ctop/crawler/modules/log/vue/CrawlerLogList.vue

@@ -1,228 +0,0 @@
-<template>
-    <a-card :bordered="false">
-
-        <!-- 查询区域 -->
-        <div class="table-page-search-wrapper">
-            <a-form layout="inline">
-                <a-row :gutter="24">
-
-                    <a-col :md="6" :sm="8">
-                        <a-form-item label="主键ID">
-                            <a-input placeholder="请输入主键ID" v-model="queryParam.requestId"></a-input>
-                        </a-form-item>
-                    </a-col>
-                    <a-col :md="6" :sm="8">
-                        <a-form-item label="请求URL">
-                            <a-input placeholder="请输入请求URL" v-model="queryParam.url"></a-input>
-                        </a-form-item>
-                    </a-col>
-                    <template v-if="toggleSearchStatus">
-                        <a-col :md="6" :sm="8">
-                            <a-form-item label="请求路径">
-                                <a-input placeholder="请输入请求路径" v-model="queryParam.path"></a-input>
-                            </a-form-item>
-                        </a-col>
-                        <a-col :md="6" :sm="8">
-                            <a-form-item label="请求方式">
-                                <a-input placeholder="请输入请求方式" v-model="queryParam.method"></a-input>
-                            </a-form-item>
-                        </a-col>
-                        <a-col :md="6" :sm="8">
-                            <a-form-item label="请求cookie">
-                                <a-input placeholder="请输入请求cookie" v-model="queryParam.requestCookie"></a-input>
-                            </a-form-item>
-                        </a-col>
-                    </template>
-                    <a-col :md="6" :sm="8">
-            <span style="float: left;overflow: hidden;" class="table-page-search-submitButtons">
-              <a-button type="primary" @click="searchQuery" icon="search">查询</a-button>
-              <a-button type="primary" @click="searchReset" icon="reload" style="margin-left: 8px">重置</a-button>
-              <a @click="handleToggleSearch" style="margin-left: 8px">
-                {{ toggleSearchStatus ? '收起' : '展开' }}
-                <a-icon :type="toggleSearchStatus ? 'up' : 'down'"/>
-              </a>
-            </span>
-                    </a-col>
-
-                </a-row>
-            </a-form>
-        </div>
-
-        <!-- 操作按钮区域 -->
-        <div class="table-operator">
-            <a-button @click="handleAdd" type="primary" icon="plus">新增</a-button>
-            <a-button type="primary" icon="download" @click="handleExportXls('爬虫日志表')">导出</a-button>
-            <a-upload name="file" :showUploadList="false" :multiple="false" :headers="tokenHeader"
-                      :action="importExcelUrl" @change="handleImportExcel">
-                <a-button type="primary" icon="import">导入</a-button>
-            </a-upload>
-            <a-dropdown v-if="selectedRowKeys.length > 0">
-                <a-menu slot="overlay">
-                    <a-menu-item key="1" @click="batchDel">
-                        <a-icon type="delete"/>
-                        删除
-                    </a-menu-item>
-                </a-menu>
-                <a-button style="margin-left: 8px"> 批量操作
-                    <a-icon type="down"/>
-                </a-button>
-            </a-dropdown>
-        </div>
-
-        <!-- table区域-begin -->
-        <div>
-            <div class="ant-alert ant-alert-info" style="margin-bottom: 16px;">
-                <i class="anticon anticon-info-circle ant-alert-icon"></i> 已选择 <a style="font-weight: 600">{{
-                selectedRowKeys.length }}</a>项
-                <a style="margin-left: 24px" @click="onClearSelected">清空</a>
-            </div>
-
-            <a-table
-                    ref="table"
-                    size="middle"
-                    bordered
-                    rowKey="id"
-                    :columns="columns"
-                    :dataSource="dataSource"
-                    :pagination="ipagination"
-                    :loading="loading"
-                    :rowSelection="{selectedRowKeys: selectedRowKeys, onChange: onSelectChange}"
-                    @change="handleTableChange">
-
-        <span slot="action" slot-scope="text, record">
-          <a @click="handleEdit(record)">编辑</a>
-
-          <a-divider type="vertical"/>
-          <a-dropdown>
-            <a class="ant-dropdown-link">更多 <a-icon type="down"/></a>
-            <a-menu slot="overlay">
-              <a-menu-item>
-                <a-popconfirm title="确定删除吗?" @confirm="() => handleDelete(record.id)">
-                  <a>删除</a>
-                </a-popconfirm>
-              </a-menu-item>
-            </a-menu>
-          </a-dropdown>
-        </span>
-
-            </a-table>
-        </div>
-        <!-- table区域-end -->
-
-        <!-- 表单区域 -->
-        <crawlerLog-modal ref="modalForm" @ok="modalFormOk"></crawlerLog-modal>
-    </a-card>
-</template>
-
-<script>
-    import CrawlerLogModal from './modules/CrawlerLogModal'
-    import {JeecgListMixin} from '@/mixins/JeecgListMixin'
-
-    export default {
-        name: "CrawlerLogList",
-        mixins: [JeecgListMixin],
-        components: {
-            CrawlerLogModal
-        },
-        data() {
-            return {
-                description: '爬虫日志表管理页面',
-                // 表头
-                columns: [
-                    {
-                        title: '#',
-                        dataIndex: '',
-                        key: 'rowIndex',
-                        width: 60,
-                        align: "center",
-                        customRender: function (t, r, index) {
-                            return parseInt(index) + 1;
-                        }
-                    },
-                    {
-                        title: '主键ID',
-                        align: "center",
-                        dataIndex: 'requestId'
-                    },
-                    {
-                        title: '请求URL',
-                        align: "center",
-                        dataIndex: 'url'
-                    },
-                    {
-                        title: '请求路径',
-                        align: "center",
-                        dataIndex: 'path'
-                    },
-                    {
-                        title: '请求方式',
-                        align: "center",
-                        dataIndex: 'method'
-                    },
-                    {
-                        title: '请求cookie',
-                        align: "center",
-                        dataIndex: 'requestCookie'
-                    },
-                    {
-                        title: '请求header',
-                        align: "center",
-                        dataIndex: 'requestHeader'
-                    },
-                    {
-                        title: '请求body',
-                        align: "center",
-                        dataIndex: 'requestBody'
-                    },
-                    {
-                        title: '响应cookie',
-                        align: "center",
-                        dataIndex: 'responseCookie'
-                    },
-                    {
-                        title: '响应header',
-                        align: "center",
-                        dataIndex: 'responseHeader'
-                    },
-                    {
-                        title: '响应body',
-                        align: "center",
-                        dataIndex: 'responseBody'
-                    },
-                    {
-                        title: '请求状态',
-                        align: "center",
-                        dataIndex: 'requestStatus'
-                    },
-                    {
-                        title: '请求耗时',
-                        align: "center",
-                        dataIndex: 'requestTime'
-                    },
-                    {
-                        title: '操作',
-                        dataIndex: 'action',
-                        align: "center",
-                        scopedSlots: {customRender: 'action'},
-                    }
-                ],
-                url: {
-                    list: "/cn.com.ctop.crawler.modules.log/crawlerLog/list",
-                    delete: "/cn.com.ctop.crawler.modules.log/crawlerLog/delete",
-                    deleteBatch: "/cn.com.ctop.crawler.modules.log/crawlerLog/deleteBatch",
-                    exportXlsUrl: "cn.com.ctop.crawler.modules.log/crawlerLog/exportXls",
-                    importExcelUrl: "cn.com.ctop.crawler.modules.log/crawlerLog/importExcel",
-                },
-            }
-        },
-        computed: {
-            importExcelUrl: function () {
-                return `${window._CONFIG['domianURL']}/${this.url.importExcelUrl}`;
-            }
-        },
-        methods: {}
-    }
-</script>
-<style scoped>
-    @import '~@assets/less/common.less'
-</style>

+ 0 - 190
module-crawler/src/main/java/cn/com/ctop/crawler/modules/log/vue/modules/CrawlerLogModal.vue

@@ -1,190 +0,0 @@
-<template>
-    <a-modal
-            :title="title"
-            :width="800"
-            :visible="visible"
-            :confirmLoading="confirmLoading"
-            @ok="handleOk"
-            @cancel="handleCancel"
-            cancelText="关闭">
-
-        <a-spin :spinning="confirmLoading">
-            <a-form :form="form">
-
-                <a-form-item
-                        :labelCol="labelCol"
-                        :wrapperCol="wrapperCol"
-                        label="主键ID">
-                    <a-input placeholder="请输入主键ID" v-decorator="['requestId', validatorRules.requestId ]"/>
-                </a-form-item>
-                <a-form-item
-                        :labelCol="labelCol"
-                        :wrapperCol="wrapperCol"
-                        label="请求URL">
-                    <a-input placeholder="请输入请求URL" v-decorator="['url', {}]"/>
-                </a-form-item>
-                <a-form-item
-                        :labelCol="labelCol"
-                        :wrapperCol="wrapperCol"
-                        label="请求路径">
-                    <a-input placeholder="请输入请求路径" v-decorator="['path', {}]"/>
-                </a-form-item>
-                <a-form-item
-                        :labelCol="labelCol"
-                        :wrapperCol="wrapperCol"
-                        label="请求方式">
-                    <a-input placeholder="请输入请求方式" v-decorator="['method', {}]"/>
-                </a-form-item>
-                <a-form-item
-                        :labelCol="labelCol"
-                        :wrapperCol="wrapperCol"
-                        label="请求cookie">
-                    <a-input placeholder="请输入请求cookie" v-decorator="['requestCookie', {}]"/>
-                </a-form-item>
-                <a-form-item
-                        :labelCol="labelCol"
-                        :wrapperCol="wrapperCol"
-                        label="请求header">
-                    <a-input placeholder="请输入请求header" v-decorator="['requestHeader', {}]"/>
-                </a-form-item>
-                <a-form-item
-                        :labelCol="labelCol"
-                        :wrapperCol="wrapperCol"
-                        label="请求body">
-                    <a-input placeholder="请输入请求body" v-decorator="['requestBody', {}]"/>
-                </a-form-item>
-                <a-form-item
-                        :labelCol="labelCol"
-                        :wrapperCol="wrapperCol"
-                        label="响应cookie">
-                    <a-input placeholder="请输入响应cookie" v-decorator="['responseCookie', {}]"/>
-                </a-form-item>
-                <a-form-item
-                        :labelCol="labelCol"
-                        :wrapperCol="wrapperCol"
-                        label="响应header">
-                    <a-input placeholder="请输入响应header" v-decorator="['responseHeader', {}]"/>
-                </a-form-item>
-                <a-form-item
-                        :labelCol="labelCol"
-                        :wrapperCol="wrapperCol"
-                        label="响应body">
-                    <a-input placeholder="请输入响应body" v-decorator="['responseBody', {}]"/>
-                </a-form-item>
-                <a-form-item
-                        :labelCol="labelCol"
-                        :wrapperCol="wrapperCol"
-                        label="请求状态">
-                    <a-input-number v-decorator="[ 'requestStatus', {}]"/>
-                </a-form-item>
-                <a-form-item
-                        :labelCol="labelCol"
-                        :wrapperCol="wrapperCol"
-                        label="请求耗时">
-                    <a-input placeholder="请输入请求耗时" v-decorator="['requestTime', {}]"/>
-                </a-form-item>
-
-            </a-form>
-        </a-spin>
-    </a-modal>
-</template>
-
-<script>
-    import {httpAction} from '@/api/manage'
-    import pick from 'lodash.pick'
-    import moment from "moment"
-
-    export default {
-        name: "CrawlerLogModal",
-        data() {
-            return {
-                title: "操作",
-                visible: false,
-                model: {},
-                labelCol: {
-                    xs: {span: 24},
-                    sm: {span: 5},
-                },
-                wrapperCol: {
-                    xs: {span: 24},
-                    sm: {span: 16},
-                },
-
-                confirmLoading: false,
-                form: this.$form.createForm(this),
-                validatorRules: {
-                    requestId: {rules: [{required: true, message: '请输入主键ID!'}]},
-                },
-                url: {
-                    add: "/cn.com.ctop.crawler.modules.log/crawlerLog/add",
-                    edit: "/cn.com.ctop.crawler.modules.log/crawlerLog/edit",
-                },
-            }
-        },
-        created() {
-        },
-        methods: {
-            add() {
-                this.edit({});
-            },
-            edit(record) {
-                this.form.resetFields();
-                this.model = Object.assign({}, record);
-                this.visible = true;
-                this.$nextTick(() => {
-                    this.form.setFieldsValue(pick(this.model, 'requestId', 'url', 'path', 'method', 'requestCookie', 'requestHeader', 'requestBody', 'responseCookie', 'responseHeader', 'responseBody', 'requestStatus', 'requestTime'))
-                    //时间格式化
-                });
-
-            },
-            close() {
-                this.$emit('close');
-                this.visible = false;
-            },
-            handleOk() {
-                const that = this;
-                // 触发表单验证
-                this.form.validateFields((err, values) => {
-                    if (!err) {
-                        that.confirmLoading = true;
-                        let httpurl = '';
-                        let method = '';
-                        if (!this.model.id) {
-                            httpurl += this.url.add;
-                            method = 'post';
-                        } else {
-                            httpurl += this.url.edit;
-                            method = 'put';
-                        }
-                        let formData = Object.assign(this.model, values);
-                        //时间格式化
-
-                        console.log(formData)
-                        httpAction(httpurl, formData, method).then((res) => {
-                            if (res.success) {
-                                that.$message.success(res.message);
-                                that.$emit('ok');
-                            } else {
-                                that.$message.warning(res.message);
-                            }
-                        }).finally(() => {
-                            that.confirmLoading = false;
-                            that.close();
-                        })
-
-
-                    }
-                })
-            },
-            handleCancel() {
-                this.close()
-            },
-
-
-        }
-    }
-</script>
-
-<style lang="less" scoped>
-
-</style>

+ 0 - 197
module-crawler/src/main/java/cn/com/ctop/crawler/modules/log/vue/modules/CrawlerLogModal__Style#Drawer.vue

@@ -1,197 +0,0 @@
-<template>
-    <a-drawer
-            :title="title"
-            :width="800"
-            placement="right"
-            :closable="false"
-            @close="close"
-            :visible="visible"
-    >
-
-        <a-spin :spinning="confirmLoading">
-            <a-form :form="form">
-
-                <a-form-item
-                        :labelCol="labelCol"
-                        :wrapperCol="wrapperCol"
-                        label="主键ID">
-                    <a-input placeholder="请输入主键ID" v-decorator="['requestId', validatorRules.requestId ]"/>
-                </a-form-item>
-                <a-form-item
-                        :labelCol="labelCol"
-                        :wrapperCol="wrapperCol"
-                        label="请求URL">
-                    <a-input placeholder="请输入请求URL" v-decorator="['url', {}]"/>
-                </a-form-item>
-                <a-form-item
-                        :labelCol="labelCol"
-                        :wrapperCol="wrapperCol"
-                        label="请求路径">
-                    <a-input placeholder="请输入请求路径" v-decorator="['path', {}]"/>
-                </a-form-item>
-                <a-form-item
-                        :labelCol="labelCol"
-                        :wrapperCol="wrapperCol"
-                        label="请求方式">
-                    <a-input placeholder="请输入请求方式" v-decorator="['method', {}]"/>
-                </a-form-item>
-                <a-form-item
-                        :labelCol="labelCol"
-                        :wrapperCol="wrapperCol"
-                        label="请求cookie">
-                    <a-input placeholder="请输入请求cookie" v-decorator="['requestCookie', {}]"/>
-                </a-form-item>
-                <a-form-item
-                        :labelCol="labelCol"
-                        :wrapperCol="wrapperCol"
-                        label="请求header">
-                    <a-input placeholder="请输入请求header" v-decorator="['requestHeader', {}]"/>
-                </a-form-item>
-                <a-form-item
-                        :labelCol="labelCol"
-                        :wrapperCol="wrapperCol"
-                        label="请求body">
-                    <a-input placeholder="请输入请求body" v-decorator="['requestBody', {}]"/>
-                </a-form-item>
-                <a-form-item
-                        :labelCol="labelCol"
-                        :wrapperCol="wrapperCol"
-                        label="响应cookie">
-                    <a-input placeholder="请输入响应cookie" v-decorator="['responseCookie', {}]"/>
-                </a-form-item>
-                <a-form-item
-                        :labelCol="labelCol"
-                        :wrapperCol="wrapperCol"
-                        label="响应header">
-                    <a-input placeholder="请输入响应header" v-decorator="['responseHeader', {}]"/>
-                </a-form-item>
-                <a-form-item
-                        :labelCol="labelCol"
-                        :wrapperCol="wrapperCol"
-                        label="响应body">
-                    <a-input placeholder="请输入响应body" v-decorator="['responseBody', {}]"/>
-                </a-form-item>
-                <a-form-item
-                        :labelCol="labelCol"
-                        :wrapperCol="wrapperCol"
-                        label="请求状态">
-                    <a-input-number v-decorator="[ 'requestStatus', {}]"/>
-                </a-form-item>
-                <a-form-item
-                        :labelCol="labelCol"
-                        :wrapperCol="wrapperCol"
-                        label="请求耗时">
-                    <a-input placeholder="请输入请求耗时" v-decorator="['requestTime', {}]"/>
-                </a-form-item>
-
-            </a-form>
-        </a-spin>
-        <a-button type="primary" @click="handleOk">确定</a-button>
-        <a-button type="primary" @click="handleCancel">取消</a-button>
-    </a-drawer>
-</template>
-
-<script>
-    import {httpAction} from '@/api/manage'
-    import pick from 'lodash.pick'
-    import moment from "moment"
-
-    export default {
-        name: "CrawlerLogModal",
-        data() {
-            return {
-                title: "操作",
-                visible: false,
-                model: {},
-                labelCol: {
-                    xs: {span: 24},
-                    sm: {span: 5},
-                },
-                wrapperCol: {
-                    xs: {span: 24},
-                    sm: {span: 16},
-                },
-
-                confirmLoading: false,
-                form: this.$form.createForm(this),
-                validatorRules: {
-                    requestId: {rules: [{required: true, message: '请输入主键ID!'}]},
-                },
-                url: {
-                    add: "/cn.com.ctop.crawler.modules.log/crawlerLog/add",
-                    edit: "/cn.com.ctop.crawler.modules.log/crawlerLog/edit",
-                },
-            }
-        },
-        created() {
-        },
-        methods: {
-            add() {
-                this.edit({});
-            },
-            edit(record) {
-                this.form.resetFields();
-                this.model = Object.assign({}, record);
-                this.visible = true;
-                this.$nextTick(() => {
-                    this.form.setFieldsValue(pick(this.model, 'requestId', 'url', 'path', 'method', 'requestCookie', 'requestHeader', 'requestBody', 'responseCookie', 'responseHeader', 'responseBody', 'requestStatus', 'requestTime'))
-                    //时间格式化
-                });
-
-            },
-            close() {
-                this.$emit('close');
-                this.visible = false;
-            },
-            handleOk() {
-                const that = this;
-                // 触发表单验证
-                this.form.validateFields((err, values) => {
-                    if (!err) {
-                        that.confirmLoading = true;
-                        let httpurl = '';
-                        let method = '';
-                        if (!this.model.id) {
-                            httpurl += this.url.add;
-                            method = 'post';
-                        } else {
-                            httpurl += this.url.edit;
-                            method = 'put';
-                        }
-                        let formData = Object.assign(this.model, values);
-                        //时间格式化
-
-                        console.log(formData)
-                        httpAction(httpurl, formData, method).then((res) => {
-                            if (res.success) {
-                                that.$message.success(res.message);
-                                that.$emit('ok');
-                            } else {
-                                that.$message.warning(res.message);
-                            }
-                        }).finally(() => {
-                            that.confirmLoading = false;
-                            that.close();
-                        })
-
-
-                    }
-                })
-            },
-            handleCancel() {
-                this.close()
-            },
-
-
-        }
-    }
-</script>
-
-<style lang="less" scoped>
-    /** Button按钮间距 */
-    .ant-btn {
-        margin-left: 30px;
-        margin-bottom: 30px;
-        float: right;
-    }
-</style>

+ 0 - 23
module-crawler/src/main/java/cn/com/ctop/crawler/modules/test/AppiumTask.java

@@ -1,23 +0,0 @@
-package cn.com.ctop.crawler.modules.test;
-
-import lombok.Data;
-
-import java.util.List;
-
-@Data
-public class AppiumTask {
-    private String findType;
-    private String findKey;
-    private String clickType;
-    private Integer offsiteX;
-    private Integer offsiteY;
-    private String loopType;
-    private Long loopRate;
-    private Boolean isMust;
-    private String textEqualType;
-    private String textEqualKey;
-    private String swapType;
-    private Long waitTime;
-    private List<AppiumTask> subTask;
-    private Boolean isClose;
-}

+ 0 - 461
module-crawler/src/main/java/cn/com/ctop/crawler/modules/test/AppiumTest.java

@@ -1,461 +0,0 @@
-//package cn.com.ctop.crawler.modules.test;
-//
-//import cn.com.ctop.crawler.modules.core.util.AppiumUtil;
-//import com.google.gson.Gson;
-//import io.appium.java_client.TouchAction;
-//import io.appium.java_client.android.AndroidDriver;
-//import io.appium.java_client.touch.WaitOptions;
-//import io.appium.java_client.touch.offset.PointOption;
-//import lombok.Data;
-//import org.openqa.selenium.Point;
-//import org.openqa.selenium.WebElement;
-//import org.openqa.selenium.remote.DesiredCapabilities;
-//
-//import java.net.MalformedURLException;
-//import java.net.URL;
-//import java.time.Duration;
-//import java.util.ArrayList;
-//import java.util.List;
-//
-//@Data
-//public class AppiumTest {
-//    private Boolean isLoop = true;
-//    private Duration duration = Duration.ofSeconds(1);
-//
-//
-//
-//    //    "http://39.106.184.70:4723/wd/hub"
-//    public AndroidDriver initAndroidDriver(String serverUrl, String deviceName, String appPackage, String appActivity) throws Exception {
-//        return new AndroidDriver(new URL(serverUrl), initCapabilities(deviceName, appPackage, appActivity));
-//    }
-//
-//    public DesiredCapabilities initCapabilities(String deviceName, String appPackage, String appActivity) {
-//        DesiredCapabilities desiredCapabilities = new DesiredCapabilities();
-//        //指定测试设备的名称
-////        desiredCapabilities.setCapability("deviceName", "127.0.0.1:62001");
-////        desiredCapabilities.setCapability("deviceName", "183.131.157.66:10005");
-//        desiredCapabilities.setCapability("deviceName", deviceName);
-//        //添加操作系统配置
-//        desiredCapabilities.setCapability("platformName", "Android");
-//        //添加操作系统版本设置
-////        desiredCapabilities.setCapability("platformVersion", "4.4.2");
-//        //指定想要测试应用的包名
-////        desiredCapabilities.setCapability("appPackage", "com.smile.gifmaker");
-//        desiredCapabilities.setCapability("appPackage", appPackage);
-//        desiredCapabilities.setCapability("noReset", true);
-//        desiredCapabilities.setCapability("unicodeKeyboard", true);
-//        desiredCapabilities.setCapability("resetKeyboard", true);
-//        //指定想要测试应用的入口activity
-////        desiredCapabilities.setCapability("appActivity", "com.yxcorp.gifshow.HomeActivity");
-//        desiredCapabilities.setCapability("appActivity", appActivity);
-//        return desiredCapabilities;
-//    }
-//
-//    public List<AppiumTask> initTask() {
-//        List<AppiumTask> list = new ArrayList<>();
-//        AppiumTask appiumTask1 = new AppiumTask();
-//        appiumTask1.setFindType("class");
-//        appiumTask1.setFindKey("android.widget.CheckedTextView");
-//        appiumTask1.setClickType("element");
-//        appiumTask1.setLoopType("noloop");
-//        appiumTask1.setIsMust(true);
-//        appiumTask1.setWaitTime(3000L);
-//
-//        AppiumTask appiumTask2 = new AppiumTask();
-//        appiumTask2.setFindType("xpath");
-//        appiumTask2.setFindKey("//android.view.View[@content-desc=\"发现\"]");
-//        appiumTask2.setClickType("element");
-//        appiumTask2.setLoopType("noloop");
-//        appiumTask2.setIsMust(true);
-//        appiumTask2.setWaitTime(3000L);
-//
-//        AppiumTask appiumTask3 = new AppiumTask();
-//        appiumTask3.setFindType("id");
-//        appiumTask3.setFindKey("com.smile.gifmaker:id/subject");
-//        appiumTask3.setClickType("no");
-//        appiumTask3.setLoopType("loop");
-//        appiumTask3.setIsMust(true);
-//        appiumTask3.setSwapType("up");
-//
-//        AppiumTask appiumTask31 = new AppiumTask();
-//        appiumTask31.setFindType("loop");
-//        appiumTask31.setClickType("point");
-//        appiumTask31.setLoopType("noloop");
-//        appiumTask31.setIsMust(false);
-//        appiumTask31.setTextEqualType("eq");
-//        appiumTask31.setTextEqualKey("广告");
-//        appiumTask31.setOffsiteX(-100);
-//        appiumTask31.setOffsiteY(-100);
-//        appiumTask31.setWaitTime(2000L);
-//
-////        AppiumTask appiumTask32 = new AppiumTask();
-////        appiumTask32.setFindType("loop");
-////        appiumTask32.setClickType("no");
-////        appiumTask32.setLoopType("noloop");
-////        appiumTask32.setIsMust(false);
-////        appiumTask32.setTextEqualType("neq");
-////        appiumTask32.setTextEqualKey("广告");
-////
-////        AppiumTask appiumTask321 = new AppiumTask();
-////        appiumTask321.setFindType("id");
-////        appiumTask321.setFindKey("com.smile.gifmaker:id/back_btn");
-////        appiumTask321.setClickType("element");
-////        appiumTask321.setLoopType("noloop");
-////        appiumTask321.setIsMust(false);
-//
-//        AppiumTask appiumTask311 = new AppiumTask();
-//        appiumTask311.setFindType("loop");
-//        appiumTask311.setClickType("no");
-//        appiumTask311.setLoopType("noloop");
-//        appiumTask311.setIsMust(true);
-//        appiumTask311.setSwapType("up");
-//
-//        AppiumTask appiumTask312 = new AppiumTask();
-//        appiumTask312.setFindType("id");
-//        appiumTask312.setWaitTime(5000L);
-//        appiumTask312.setFindKey("com.smile.gifmaker:id/ad_simplified_style_enhanced_ad_install_text");
-//        appiumTask312.setClickType("element");
-//        appiumTask312.setLoopType("noloop");
-//        appiumTask312.setIsMust(false);
-//
-//
-//        AppiumTask appiumTask3121 = new AppiumTask();
-//        appiumTask3121.setFindType("id");
-//        appiumTask3121.setFindKey("com.android.packageinstaller:id/ok_button");
-//        appiumTask3121.setClickType("element");
-//        appiumTask3121.setLoopType("noloop");
-//        appiumTask3121.setIsMust(true);
-//        appiumTask3121.setLoopRate(3000L);
-//
-//        AppiumTask appiumTask3122 = new AppiumTask();
-//        appiumTask3122.setFindType("id");
-//        appiumTask3122.setFindKey("com.android.packageinstaller:id/launch_button");
-//        appiumTask3122.setClickType("element");
-//        appiumTask3122.setLoopType("noloop");
-//        appiumTask3122.setIsMust(true);
-//        appiumTask3121.setLoopRate(3000L);
-//
-//        AppiumTask appiumTask3124 = new AppiumTask();
-//        appiumTask3124.setFindType("id");
-//        appiumTask3124.setFindKey("com.android.packageinstaller:id/permission_allow_button");
-//        appiumTask3124.setClickType("element");
-//        appiumTask3124.setLoopType("noloop");
-//        appiumTask3124.setIsMust(false);
-//        appiumTask3124.setWaitTime(3000L);
-//
-//        AppiumTask appiumTask3125 = new AppiumTask();
-//        appiumTask3125.setFindType("id");
-//        appiumTask3125.setFindKey("com.android.packageinstaller:id/permission_allow_button");
-//        appiumTask3125.setClickType("element");
-//        appiumTask3125.setLoopType("noloop");
-//        appiumTask3125.setIsMust(false);
-//        appiumTask3125.setWaitTime(3000L);
-//
-//        AppiumTask appiumTask3126 = new AppiumTask();
-//        appiumTask3126.setFindType("id");
-//        appiumTask3126.setFindKey("com.android.packageinstaller:id/permission_allow_button");
-//        appiumTask3126.setClickType("element");
-//        appiumTask3126.setLoopType("noloop");
-//        appiumTask3126.setIsMust(false);
-//        appiumTask3126.setWaitTime(3000L);
-//
-//        AppiumTask appiumTask3127 = new AppiumTask();
-//        appiumTask3127.setFindType("id");
-//        appiumTask3127.setFindKey("com.android.packageinstaller:id/permission_allow_button");
-//        appiumTask3127.setClickType("element");
-//        appiumTask3127.setLoopType("noloop");
-//        appiumTask3127.setIsMust(false);
-//        appiumTask3127.setWaitTime(3000L);
-//        appiumTask3127.setIsClose(true);
-//
-//        List<AppiumTask> subList4 = new ArrayList<>();
-//        subList4.add(appiumTask3121);
-//        subList4.add(appiumTask3122);
-////        subList4.add(appiumTask3123);
-//        subList4.add(appiumTask3124);
-//        subList4.add(appiumTask3125);
-//        subList4.add(appiumTask3126);
-//        subList4.add(appiumTask3127);
-//        appiumTask312.setSubTask(subList4);
-//
-//        List<AppiumTask> subList3 = new ArrayList<>();
-//        subList3.add(appiumTask311);
-//        subList3.add(appiumTask312);
-//        appiumTask31.setSubTask(subList3);
-//
-////        List<AppiumTask> subList2 = new ArrayList<>();
-////        subList2.add(appiumTask321);
-////        appiumTask32.setSubTask(subList2);
-//
-//        List<AppiumTask> subList1 = new ArrayList<>();
-//        subList1.add(appiumTask31);
-////        subList1.add(appiumTask32);
-//        appiumTask3.setSubTask(subList1);
-//
-//        list.add(appiumTask1);
-//        list.add(appiumTask2);
-//        list.add(appiumTask3);
-//        return list;
-//    }
-//
-//    public void t() {
-//        try {
-////            AndroidDriver androidDriver = initAndroidDriver("http://39.106.184.70:4723/wd/hub","183.131.157.66:10005","com.smile.gifmaker","com.yxcorp.gifshow.HomeActivity");
-//            AndroidDriver androidDriver = initAndroidDriver("http://39.106.184.70:4723/wd/hub", "183.131.157.66:10005", "com.smile.gifmaker", "com.yxcorp.gifshow.HomeActivity");
-//
-//            List<AppiumTask> list = initTask();
-//            for (AppiumTask task : list) {
-//                runTask(androidDriver, task, null);
-//            }
-//            androidDriver.quit();
-//        } catch (Exception e) {
-//            e.printStackTrace();
-//        }
-//
-//
-//    }
-//
-//    public void runTask(AndroidDriver androidDriver, AppiumTask appiumTask, WebElement loopElement) throws Exception {
-//        if (appiumTask != null) {
-//            if (appiumTask.getWaitTime() != null && appiumTask.getWaitTime() > 0) {
-//                Thread.sleep(appiumTask.getWaitTime());
-//            }
-//            if (appiumTask.getLoopType() != null && "noloop".equals(appiumTask.getLoopType())) {
-//                if (appiumTask.getFindType() != null) {
-//                    WebElement element = null;
-//                    if ("id".equals(appiumTask.getFindType())) {
-//                        element = AppiumUtil.getElementById(androidDriver, appiumTask.getFindKey());
-//                    } else if ("class".equals(appiumTask.getFindType())) {
-//                        element = getElementByClass(androidDriver, appiumTask.getFindKey());
-//                    } else if ("xpath".equals(appiumTask.getFindType())) {
-//                        element = getElementByXpath(androidDriver, appiumTask.getFindKey());
-//                    } else if ("loop".equals(appiumTask.getFindType())) {
-//                        element = loopElement;
-//                    }
-//                    if (element != null) {
-//                        if (appiumTask.getTextEqualType() != null) {
-//                            if ("eq".equals(appiumTask.getTextEqualType())) {
-//                                if (!element.getText().equals(appiumTask.getTextEqualKey())) {
-//                                    return;
-//                                }
-//                            } else if ("neq".equals(appiumTask.getTextEqualType())) {
-//                                if (element.getText() == null || element.getText().equals(appiumTask.getTextEqualKey())) {
-//                                    return;
-//                                }
-//                            }
-//                        }
-//                        if (appiumTask.getClickType() != null) {
-//                            if ("element".equals(appiumTask.getClickType())) {
-//                                element.click();
-//                            } else if ("point".equals(appiumTask.getClickType())) {
-//                                Point point = element.getLocation();
-//                                tapPoint(androidDriver, point.getX() + appiumTask.getOffsiteX(), point.getY() + appiumTask.getOffsiteY());
-//                            }
-//                        }
-//                        List<AppiumTask> subTask = appiumTask.getSubTask();
-//                        if (subTask != null && subTask.size() > 0) {
-//                            for (AppiumTask task : subTask) {
-//                                runTask(androidDriver, task, element);
-//                            }
-//                        }
-//                    } else {
-//                        if (appiumTask.getIsMust() != null && appiumTask.getIsMust() == true) {
-//                            if (appiumTask.getLoopRate() != null && appiumTask.getLoopRate() > 0) {
-//                                Thread.sleep(appiumTask.getLoopRate());
-//                            }
-//                            runTask(androidDriver, appiumTask, element);
-//                            return;
-//                        }
-//                        if (appiumTask.getClickType() != null && "point".equals(appiumTask.getClickType())) {
-//                            tapPoint(androidDriver, appiumTask.getOffsiteX(), appiumTask.getOffsiteY());
-//                        }
-//                    }
-//                }
-//                if (appiumTask.getSwapType() != null) {
-//                    if ("up".equals(appiumTask.getSwapType())) {
-//                        this.swipeToUp(androidDriver);
-//                    } else if ("down".equals(appiumTask.getSwapType())) {
-//                        this.swipeToDown(androidDriver);
-//                    } else if ("left".equals(appiumTask.getSwapType())) {
-//                        this.swipeToLeft(androidDriver);
-//                    } else if ("right".equals(appiumTask.getSwapType())) {
-//                        this.swipeToRight(androidDriver);
-//                    }
-//                }
-//            } else if (appiumTask.getLoopType() != null && "loop".equals(appiumTask.getLoopType())) {
-//                while (isLoop) {
-//                    if (appiumTask.getFindType() != null) {
-//                        List<WebElement> list = null;
-//                        if ("id".equals(appiumTask.getFindType())) {
-//                            list = androidDriver.findElementsById(appiumTask.getFindKey());
-//                        } else if ("class".equals(appiumTask.getFindType())) {
-//                            list = androidDriver.findElementsByClassName(appiumTask.getFindKey());
-//                        } else if ("xpath".equals(appiumTask.getFindType())) {
-//                            list = androidDriver.findElementsByXPath(appiumTask.getFindKey());
-//                        }
-//                        if (list != null && list.size() > 0) {
-//                            for (WebElement element : list) {
-//                                if (!isLoop) {
-//                                    break;
-//                                }
-//                                if (appiumTask.getLoopRate() != null && appiumTask.getLoopRate() > 0) {
-//                                    Thread.sleep(appiumTask.getLoopRate());
-//                                }
-//                                List<AppiumTask> subTask = appiumTask.getSubTask();
-//                                if (subTask != null && subTask.size() > 0) {
-//                                    for (AppiumTask task : subTask) {
-//                                        runTask(androidDriver, task, element);
-//                                    }
-//                                }
-//                            }
-//                        }
-//                    }
-//                    if (appiumTask.getSwapType() != null) {
-//                        if ("up".equals(appiumTask.getSwapType())) {
-//                            this.swipeToUp(androidDriver);
-//                        } else if ("down".equals(appiumTask.getSwapType())) {
-//                            this.swipeToDown(androidDriver);
-//                        } else if ("left".equals(appiumTask.getSwapType())) {
-//                            this.swipeToLeft(androidDriver);
-//                        } else if ("right".equals(appiumTask.getSwapType())) {
-//                            this.swipeToRight(androidDriver);
-//                        }
-//                    }
-//                }
-//
-//            }
-//            if (appiumTask.getIsClose() != null && appiumTask.getIsClose()) {
-//                this.isLoop = false;
-//            }
-//        }
-//    }
-//
-//    public static void main(String[] args) throws MalformedURLException, InterruptedException {
-//        // TODO Auto-generated method stub
-//        //1.添加配置,创建DesiredCapabilities对象
-//        AppiumTest appiumTest = new AppiumTest();
-//        appiumTest.t();
-//        //2.创建驱动...URL是appium的固定地址;指定appium通讯的地址,将相对应的配置传入到驱动里边
-//
-////        com.longene.lglocation/.activity.MainActivity
-////        com.longene.setcardproperty/.MainActivity
-//        //打开地图后点击同意继续
-////        androidDriver.findElementById("com.smile.gifmaker:id/button").click();
-//        //点击进入地图
-//        //加线程等待
-////        Thread.sleep(5000);
-////        while (true){
-////            try {
-////                AndroidDriver androidDriver = appiumTest.initAndroidDriver("http://39.106.184.70:4723/wd/hub","","","");
-////
-////                List<WebElement> list = androidDriver.findElementsByClassName("android.widget.CheckedTextView");
-////                if(list != null && list.size() > 0){
-////                    list.get(0).click();
-////                }
-////
-////                List<WebElement> list1 = androidDriver.findElementsByXPath("//android.view.View[@content-desc=\"发现\"]");
-////                if(list1 != null && list1.size() > 0){
-////                    list1.get(0).click();
-////                }
-////                appiumTest.swipeToUp(androidDriver);
-////
-////                Thread.sleep(2000);
-////                List<WebElement> webElementList = androidDriver.findElementsById("com.smile.gifmaker:id/subject");
-////                if(webElementList != null && webElementList.size() > 0){
-////                    for(WebElement element : webElementList){
-////                        if(element.getText().equals("广告")){
-////                            Point point = element.getLocation();
-////                            appiumTest.tapPoint(androidDriver,point.getX()-100,point.getY()-100);
-//////                            Thread.sleep(1000);
-//////                            androidDriver.findElementById("com.smile.gifmaker:id/like_button").click();
-//////                            Thread.sleep(1000);
-////                            appiumTest.swipeToUp(androidDriver);
-////                            Thread.sleep(5000);
-//////                            String text = androidDriver.findElementById("com.smile.gifmaker:id/label").getText();
-//////                            String title = text.split(":")[1].trim();
-////
-//////                            if(title.startsWith("超级有趣的小游戏")){
-//////                                Thread.sleep(1000);
-//////                                androidDriver.findElementById("com.smile.gifmaker:id/editor_holder_text").click();
-//////                                Thread.sleep(1000);
-//////                                androidDriver.findElementById("com.smile.gifmaker:id/editor").sendKeys("这个广告看着很吸引人!");
-//////                                Thread.sleep(1000);
-//////                                androidDriver.findElementById("com.smile.gifmaker:id/finish_button").click();
-////                                Thread.sleep(5000);
-////                                List<WebElement> webElement = androidDriver.findElementsById("com.smile.gifmaker:id/ad_simplified_style_enhanced_ad_install_text");
-////                                if (webElement != null && webElement.size() > 0){
-////                                    webElement.get(0).click();
-////                                    while (true){
-////                                    Thread.sleep(5000);
-////                                        List<WebElement> element1 = androidDriver.findElementsById("com.android.packageinstaller:id/ok_button");
-////                                        if(element1 != null && element1.size() > 0){
-////                                            element1.get(0).click();
-////                                            break;
-////                                        }
-////                                    }
-////                                    while (true){
-////                                        Thread.sleep(5000);
-////                                        List<WebElement> element1 = androidDriver.findElementsById("com.android.packageinstaller:id/launch_button");
-////                                        if(element1 != null && element1.size() > 0){
-////                                            element1.get(0).click();
-////                                            break;
-////                                        }
-////                                    }
-////                                    while (true){
-////                                        Thread.sleep(5000);
-////                                        List<WebElement> element1 = androidDriver.findElementsById("com.android.packageinstaller:id/permission_allow_button");
-////                                        if (element1 != null && element1.size() > 0){
-////                                            element1.get(0).click();
-////                                        }else {
-////                                            break;
-////                                        }
-////                                    }
-////                                    Thread.sleep(5000);
-////                                    List<WebElement> element1 = androidDriver.findElementsById("com.game.matrix_pixelpaint:id/nr");
-////                                    if(element1 != null && element1.size() > 0){
-////                                        element1.get(0).click();
-////                                    }
-////                                    for(int i = 0; i < 10;i++){
-////                                        Thread.sleep(2000);
-////                                        appiumTest.swipeToUp(androidDriver);
-////                                    }
-////                                }else {
-////                                    androidDriver.findElementById("com.smile.gifmaker:id/back_btn").click();
-////                                }
-////                                //按钮  com.smile.gifmaker:id/ad_simplified_style_enhanced_ad_install_text
-////                                //转化  com.smile.gifmaker:id/ad_simplified_style_enhanced_app_name_text
-////                                //喜欢  com.smile.gifmaker:id/like_button
-////                                //举报  com.smile.gifmaker:id/inform_button
-////                                //分享  com.smile.gifmaker:id/forward_button
-////                                //头像  com.smile.gifmaker:id/avatar
-////                                //返回  com.smile.gifmaker:id/back_btn
-////                                //昵称标题 com.smile.gifmaker:id/label
-////                                //评论数 com.smile.gifmaker:id/more_comments
-////                                //点赞数 com.smile.gifmaker:id/number_like
-////                                //播放数 com.smile.gifmaker:id/number_review
-////                                //新增评论 com.smile.gifmaker:id/text_top
-////                                //评论内容 com.smile.gifmaker:id/editor
-////                                //评论提交 com.smile.gifmaker:id/finish_button
-////
-//////                            }else {
-//////                                androidDriver.findElementById("com.smile.gifmaker:id/back_btn").click();
-//////                            }
-////
-////
-//////                            Thread.sleep(5000);
-//////                            androidDriver.findElementById("com.smile.gifmaker:id/back_btn").click();
-////                        }
-////                    }
-////                }
-////            }catch (Exception e){
-////                e.printStackTrace();
-////            }
-////
-////        }
-//
-////        androidDriver.findElementById("com.smile.gifmaker:id/player_cover").click();
-////        Thread.sleep(5000);
-////        //退出
-////        androidDriver.quit();
-//    }
-//}

+ 23 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/material/service/IBytedanceCleanMaterialService.java

@@ -0,0 +1,23 @@
+package cn.com.ctop.toutiao.modules.material.service;
+
+import java.util.List;
+
+/**
+ * @author 宋英豪
+ * @date 2020年5月9日09:52:48
+ */
+public interface IBytedanceCleanMaterialService {
+    /**
+     * 头条 唯一素材MD5标识
+     *
+     * @return
+     */
+    List<String> getSignature();
+
+    /**
+     * 计算头条素材报表
+     *
+     * @param signature
+     */
+    void calculationMaterialReport(String signature);
+}

+ 45 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/material/service/impl/BytedanceCleanMaterialServiceImpl.java

@@ -0,0 +1,45 @@
+package cn.com.ctop.toutiao.modules.material.service.impl;
+
+import cn.com.ctop.common.module.utils.Check;
+import cn.com.ctop.toutiao.modules.material.service.IBytedanceCleanMaterialService;
+import cn.com.ctop.toutiao.modules.report.entity.ByteDanceCleanMaterialReport;
+import cn.com.ctop.toutiao.modules.report.mapper.BytedanceCreativeDailyReportMapper;
+import cn.com.ctop.toutiao.modules.report.service.IByteDanceCleanMaterialReportService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+
+@Service
+public class BytedanceCleanMaterialServiceImpl implements IBytedanceCleanMaterialService {
+    @Autowired
+    private BytedanceCreativeDailyReportMapper bytedanceCreativeDailyReportMapper;
+    @Autowired
+    private IByteDanceCleanMaterialReportService byteDanceCleanMaterialReportService;
+
+    /**
+     * MD5 集合
+     *
+     * @return
+     */
+    @Override
+    public List<String> getSignature() {
+        return bytedanceCreativeDailyReportMapper.getSignatureList();
+    }
+
+    /**
+     * 头条素材报表
+     *
+     * @param signature
+     */
+    @Override
+    public void calculationMaterialReport(String signature) {
+        if (Check.isNull(signature)) {
+            return;
+        }
+        ByteDanceCleanMaterialReport calculationMaterialReport = bytedanceCreativeDailyReportMapper.calculationMaterialReport(signature);
+        if (!Check.isNull(calculationMaterialReport)) {
+            byteDanceCleanMaterialReportService.saveOrUpdate(calculationMaterialReport);
+        }
+    }
+}

+ 1 - 1
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/entity/ByteDanceCleanMaterialReport.java

@@ -1,4 +1,4 @@
-package org.jeecg.modules.ctop.entity;
+package cn.com.ctop.toutiao.modules.report.entity;
 
 import com.baomidou.mybatisplus.annotation.IdType;
 import com.baomidou.mybatisplus.annotation.TableId;

+ 2 - 2
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/mapper/ByteDanceCleanMaterialReportMapper.java

@@ -1,7 +1,7 @@
-package org.jeecg.modules.ctop.mapper;
+package cn.com.ctop.toutiao.modules.report.mapper;
 
+import cn.com.ctop.toutiao.modules.report.entity.ByteDanceCleanMaterialReport;
 import com.baomidou.mybatisplus.core.mapper.BaseMapper;
-import org.jeecg.modules.ctop.entity.ByteDanceCleanMaterialReport;
 
 /**
  * 头条-数据清洗

+ 5 - 1
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/BytedanceCreativeDailyReportMapper.java

@@ -1,5 +1,6 @@
 package cn.com.ctop.toutiao.modules.report.mapper;
 
+import cn.com.ctop.toutiao.modules.report.entity.ByteDanceCleanMaterialReport;
 import cn.com.ctop.toutiao.modules.report.entity.BytedanceCreativeDailyReport;
 import com.baomidou.mybatisplus.core.mapper.BaseMapper;
 import org.apache.ibatis.annotations.Param;
@@ -18,6 +19,9 @@ public interface BytedanceCreativeDailyReportMapper extends BaseMapper<Bytedance
 
     List<BytedanceCreativeDailyReport> bytedanceCreativeAccountReport(Map<String, Object> paramsMap);
 
-
     void replaceBatch(@Param(value = "reports") List<BytedanceCreativeDailyReport> reports);
+
+    List<String> getSignatureList();
+
+    ByteDanceCleanMaterialReport calculationMaterialReport(@Param(value = "signature") String signature);
 }

+ 2 - 2
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/mapper/xml/ByteDanceCleanMaterialReportMapper.xml

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
-<mapper namespace="org.jeecg.modules.ctop.mapper.ByteDanceCleanMaterialReportMapper">
+<mapper namespace="cn.com.ctop.toutiao.modules.report.mapper.ByteDanceCleanMaterialReportMapper">
 
-</mapper>
+</mapper>

+ 24 - 1
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/xml/BytedanceCreativeDailyReportMapper.xml

@@ -3,7 +3,7 @@
 <mapper namespace="cn.com.ctop.toutiao.modules.report.mapper.BytedanceCreativeDailyReportMapper">
     <select id="bytedanceCreativeDailyReport" parameterType="Map"
             resultType="cn.com.ctop.toutiao.modules.report.entity.BytedanceCreativeDailyReport">
-        select
+    select
         t2.creative_id as creativeId,  <!-- 素材id -->
         t4.video_url as videoUrl,  <!-- 视频url -->
         sum(t2.show_num) as showNum,  <!-- 展示量 -->
@@ -226,5 +226,28 @@
         </foreach>
     </insert>
 
+    <select id="getSignatureList" resultType="java.lang.String">
+    SELECT
+      DISTINCT(signature)
+      from ctop_bytedance_report_creative_daily
+      WHERE signature !=''
+    </select>
+    <select id="calculationMaterialReport"
+            resultType="cn.com.ctop.toutiao.modules.report.entity.ByteDanceCleanMaterialReport">
+        select
+        signature id,
+        sum(cost) cost,
+        sum(show_num) showNum,
+        sum(click) click,
+        sum(convert_num) convertNum,
+        sum(active) active,
+        sum(register) register,
+        sum(form) form,
+        sum(total_play) totalPlay,
+        sum(valid_play) validPlay
+        from ctop_bytedance_report_creative_daily
+        where
+        signature = #{signature}
+    </select>
 
 </mapper>

+ 8 - 2
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/service/IByteDanceCleanMaterialReportService.java

@@ -1,7 +1,9 @@
-package org.jeecg.modules.ctop.service;
+package cn.com.ctop.toutiao.modules.report.service;
 
+import cn.com.ctop.toutiao.modules.report.entity.ByteDanceCleanMaterialReport;
 import com.baomidou.mybatisplus.extension.service.IService;
-import org.jeecg.modules.ctop.entity.ByteDanceCleanMaterialReport;
+
+import java.util.List;
 
 /**
  * 头条-数据清洗
@@ -12,4 +14,8 @@ import org.jeecg.modules.ctop.entity.ByteDanceCleanMaterialReport;
  */
 public interface IByteDanceCleanMaterialReportService extends IService<ByteDanceCleanMaterialReport> {
 
+    List<String> getSignature();
+
+
+    void calculationMaterialReport(String signature);
 }

+ 54 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/ByteDanceCleanMaterialReportServiceImpl.java

@@ -0,0 +1,54 @@
+package cn.com.ctop.toutiao.modules.report.service.impl;
+
+import cn.com.ctop.common.module.utils.Check;
+import cn.com.ctop.toutiao.modules.report.entity.ByteDanceCleanMaterialReport;
+import cn.com.ctop.toutiao.modules.report.mapper.ByteDanceCleanMaterialReportMapper;
+import cn.com.ctop.toutiao.modules.report.mapper.BytedanceCreativeDailyReportMapper;
+import cn.com.ctop.toutiao.modules.report.service.IByteDanceCleanMaterialReportService;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+
+/**
+ * 头条-数据清洗
+ *
+ * @author jeecg-boot
+ * @version V1.0
+ * @date 2020-02-25
+ */
+@Service
+public class ByteDanceCleanMaterialReportServiceImpl extends ServiceImpl<ByteDanceCleanMaterialReportMapper, ByteDanceCleanMaterialReport> implements IByteDanceCleanMaterialReportService {
+    @Autowired
+    private BytedanceCreativeDailyReportMapper bytedanceCreativeDailyReportMapper;
+    @Autowired
+    private
+    IByteDanceCleanMaterialReportService byteDanceCleanMaterialReportService;
+
+    /**
+     * 头条MD5 集合
+     *
+     * @return
+     */
+    @Override
+    public List<String> getSignature() {
+        return bytedanceCreativeDailyReportMapper.getSignatureList();
+    }
+
+    /**
+     * 头条素材报表
+     *
+     * @param signature
+     */
+    @Override
+    public void calculationMaterialReport(String signature) {
+        if (Check.isNull(signature)) {
+            return;
+        }
+        ByteDanceCleanMaterialReport calculationMaterialReport = bytedanceCreativeDailyReportMapper.calculationMaterialReport(signature);
+        if (!Check.isNull(calculationMaterialReport)) {
+            byteDanceCleanMaterialReportService.saveOrUpdate(calculationMaterialReport);
+        }
+    }
+}

+ 1 - 1
pom.xml

@@ -25,7 +25,7 @@
         <module>performance-appraisal</module>
         <module>jeecg-boot-module-system</module>
         <module>xxl-job-core</module>
-        <module>xxl-job-bytedance-executor</module>
+        <module>xxl-job-executor</module>
 
     </modules>
 

+ 0 - 48
xxl-job-bytedance-executor/src/main/java/com/xxl/job/executor/handler/bytedance/ByteDanceCleanMaterialJob.java

@@ -1,48 +0,0 @@
-//package com.xxl.job.executor.handler.bytedance;
-//
-//import cn.com.ctop.common.module.utils.Check;
-//import lombok.extern.slf4j.Slf4j;
-//import org.jeecg.modules.ctop.service.ICleanMaterialService;
-//import org.quartz.Job;
-//import org.quartz.JobExecutionContext;
-//import org.springframework.beans.factory.annotation.Autowired;
-//
-//import java.util.List;
-//import java.util.concurrent.ExecutorService;
-//import java.util.concurrent.Executors;
-//
-///**
-// * 清洗素材报表
-// */
-//@Slf4j
-//public class ByteDanceCleanMaterialJob implements Job {
-//    @Autowired
-//    private ICleanMaterialService cleanMaterialService;
-//    static ExecutorService executorService = null;
-//
-//    @Override
-//    public void execute(JobExecutionContext jobExecutionContext) {
-//        Thread thread = new Thread() {
-//            @Override
-//            public void run() {
-//                List<String> signatureList = cleanMaterialService.getByteDanceSignature();
-//                if (Check.isNull(signatureList)) {
-//                    log.error("头条素材数据获取为空");
-//                    return;
-//                }
-//                executorService = Executors.newFixedThreadPool(3);
-//                for (int i = 0; i < signatureList.size(); i++) {
-//                    String signature = signatureList.get(i);
-//                    executorService.submit(new Runnable() {
-//                        @Override
-//                        public void run() {
-//                            cleanMaterialService.calculationByteDanceReport(signature);
-//                        }
-//                    });
-//                }
-//            }
-//        };
-//        thread.start();
-//
-//    }
-//}

+ 7 - 1
xxl-job-bytedance-executor/pom.xml

@@ -9,7 +9,8 @@
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
-    <artifactId>xxl-job-bytedance-executor</artifactId>
+    <artifactId>xxl-job-executor</artifactId>
+    <name>xxl-job-executor</name>
 
     <dependencies>
         <!-- spring-boot-starter-web (spring-webmvc + tomcat) -->
@@ -45,6 +46,11 @@
             <artifactId>module-toutiao</artifactId>
             <version>2.0.2</version>
         </dependency>
+        <dependency>
+            <groupId>cn.com.ctop</groupId>
+            <artifactId>module-kuaishou</artifactId>
+            <version>2.0.2</version>
+        </dependency>
 
     </dependencies>
 

xxl-job-bytedance-executor/src/main/java/com/xxl/job/executor/XxlJobExecutorApplication.java → xxl-job-executor/src/main/java/com/xxl/job/executor/XxlJobExecutorApplication.java


xxl-job-bytedance-executor/src/main/java/com/xxl/job/executor/config/MybatisPlusConfig.java → xxl-job-executor/src/main/java/com/xxl/job/executor/config/MybatisPlusConfig.java


xxl-job-bytedance-executor/src/main/java/com/xxl/job/executor/config/RedisConfig.java → xxl-job-executor/src/main/java/com/xxl/job/executor/config/RedisConfig.java


xxl-job-bytedance-executor/src/main/java/com/xxl/job/executor/config/WebMvcConfiguration.java → xxl-job-executor/src/main/java/com/xxl/job/executor/config/WebMvcConfiguration.java


xxl-job-bytedance-executor/src/main/java/com/xxl/job/executor/config/XxlJobConfig.java → xxl-job-executor/src/main/java/com/xxl/job/executor/config/XxlJobConfig.java


xxl-job-bytedance-executor/src/main/java/com/xxl/job/executor/handler/SampleXxlJob.java → xxl-job-executor/src/main/java/com/xxl/job/executor/handler/SampleXxlJob.java


+ 38 - 0
xxl-job-executor/src/main/java/com/xxl/job/executor/handler/bytedance/ByteDanceCleanMaterialJob.java

@@ -0,0 +1,38 @@
+package com.xxl.job.executor.handler.bytedance;
+
+import cn.com.ctop.common.module.utils.Check;
+import cn.com.ctop.toutiao.modules.report.service.IByteDanceCleanMaterialReportService;
+import com.xxl.job.core.biz.model.ReturnT;
+import com.xxl.job.core.handler.annotation.XxlJob;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+/**
+ * 清洗素材报表
+ *
+ * @author 宋英豪
+ */
+@Slf4j
+@Component
+public class ByteDanceCleanMaterialJob {
+    @Autowired
+    private IByteDanceCleanMaterialReportService cleanMaterialService;
+    static ExecutorService executorService = null;
+
+    @XxlJob("byteDanceCleanMaterial")
+    public ReturnT<String> execute(String param) throws Exception {
+        List<String> signatureList = cleanMaterialService.getSignature();
+        if (Check.isNull(signatureList)) {
+            log.error("头条素材数据获取为空");
+            return ReturnT.FAIL;
+        }
+        executorService = Executors.newFixedThreadPool(3);
+        signatureList.forEach(signature -> executorService.submit(() -> cleanMaterialService.calculationMaterialReport(signature)));
+        return ReturnT.SUCCESS;
+    }
+}

+ 4 - 14
xxl-job-bytedance-executor/src/main/java/com/xxl/job/executor/handler/bytedance/ByteDanceHourlyAccountWarningLoadJob.java

@@ -15,8 +15,6 @@ import java.util.List;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 
-import static org.jeecg.common.util.DateUtils.getAnotherDay;
-
 /**
  * 头条预警
  *
@@ -34,7 +32,7 @@ public class ByteDanceHourlyAccountWarningLoadJob {
     @XxlJob("byteDanceHourlyAccountWarning")
     public ReturnT<String> execute(String param) throws Exception {
         log.info("开始时段消耗预警定时任务");
-        long l = System.currentTimeMillis();
+        long startTime = System.currentTimeMillis();
         List<CtopOauthToken> tokens = tokenService.getTokenListByType(CtopAdConstant.PLATFORM_TYPE_BYTEDANCE);
         if (null == tokens || tokens.size() <= 0) {
             log.info("定时获取头条数据异常:未获取到可用的token");
@@ -42,17 +40,9 @@ public class ByteDanceHourlyAccountWarningLoadJob {
         }
         executorService = Executors.newFixedThreadPool(3);
         String statDate = DateUtils.getDate("yyyy-MM-dd");
-        String anotherDay = getAnotherDay("yyyy-MM-dd", statDate, -1);
-        for (CtopOauthToken token : tokens) {
-            executorService.submit(new Runnable() {
-                @Override
-                public void run() {
-                    byteDanceHourlyAccountWarningService.byteDanceAccountWarning(token.getAccountId(), statDate, anotherDay);
-                }
-
-            });
-        }
-        log.info("时段消耗预警定时任务执行完毕,耗时:{} ms", System.currentTimeMillis() - l);
+        String anotherDay = DateUtils.getAnotherDay("yyyy-MM-dd", statDate, 1);
+        tokens.forEach(token -> executorService.submit(() -> byteDanceHourlyAccountWarningService.byteDanceAccountWarning(token.getAccountId(), statDate, anotherDay)));
+        log.info("时段消耗预警定时任务执行完毕,耗时:{} ms", System.currentTimeMillis() - startTime);
         return ReturnT.SUCCESS;
     }
 }

+ 2 - 10
xxl-job-bytedance-executor/src/main/java/com/xxl/job/executor/handler/bytedance/BytedanceAdvertiserDailyReportLoadJob.java

@@ -17,7 +17,7 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 
 /**
- * @author syh
+ * @author 宋英豪
  * 头条广告主日报数据获取任务
  * 第二天凌晨跑前一天的数据
  */
@@ -41,15 +41,7 @@ public class BytedanceAdvertiserDailyReportLoadJob {
             return ReturnT.FAIL;
         }
         executorService = Executors.newFixedThreadPool(5);
-        tokens.forEach(token -> {
-            executorService.submit(new Runnable() {
-                @Override
-                public void run() {
-                    //获取广告主信息数据
-                    reportService.getAdvertiserReport(token, getDate, getDate, CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
-                }
-            });
-        });
+        tokens.forEach(token -> executorService.submit(() -> reportService.getAdvertiserReport(token, getDate, getDate, CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY)));
         return ReturnT.SUCCESS;
     }
 

+ 7 - 12
xxl-job-bytedance-executor/src/main/java/com/xxl/job/executor/handler/bytedance/BytedanceAdvertiserHourlyReportLoadJob.java

@@ -18,7 +18,7 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 
 /**
- * @author syh
+ * @author 宋英豪
  * 头条广告主时报获取任务
  */
 @Slf4j
@@ -44,19 +44,14 @@ public class BytedanceAdvertiserHourlyReportLoadJob {
         }
         executorService = Executors.newFixedThreadPool(4);
         tokens.forEach(token -> {
-            executorService.submit(new Runnable() {
-                @Override
-                public void run() {
-                    if (null != hour && "00".equals(hour)) {
-                        Date finalGetDate = DateUtils.addDay(getDate, -1);
-                        //获取广告主信息数据
-                        reportService.getAdvertiserReport(token, finalGetDate, finalGetDate, CtopAdConstant.BYTEDANCE_REPORT_TYPE_HOURLY);
-                    }
-                    reportService.getAdvertiserReport(token, getDate, getDate, CtopAdConstant.BYTEDANCE_REPORT_TYPE_HOURLY);
+            executorService.submit(() -> {
+                if (null != hour && "00".equals(hour)) {
+                    Date finalGetDate = DateUtils.addDay(getDate, -1);
+                    //获取广告主信息数据
+                    reportService.getAdvertiserReport(token, finalGetDate, finalGetDate, CtopAdConstant.BYTEDANCE_REPORT_TYPE_HOURLY);
                 }
-
+                reportService.getAdvertiserReport(token, getDate, getDate, CtopAdConstant.BYTEDANCE_REPORT_TYPE_HOURLY);
             });
-
         });
         log.info("头条时报数据获取结束");
         return ReturnT.SUCCESS;

+ 11 - 14
xxl-job-bytedance-executor/src/main/java/com/xxl/job/executor/handler/bytedance/BytedanceAdvertisingDataLoadJob.java

@@ -18,6 +18,7 @@ import java.util.concurrent.Executors;
 
 /**
  * 头条广告物料数据获取任务
+ * @author 宋英豪
  */
 @Slf4j
 public class BytedanceAdvertisingDataLoadJob {
@@ -38,20 +39,16 @@ public class BytedanceAdvertisingDataLoadJob {
         List<CtopOauthToken> tokens = tokenService.getTokenListByType(CtopAdConstant.PLATFORM_TYPE_BYTEDANCE);
         executorService = Executors.newFixedThreadPool(5);
         tokens.forEach(token -> {
-            executorService.submit(new Runnable() {
-                @Override
-                public void run() {
-                    try {
-                        //1:获取当日广告组数据
-                        advertiserDataService.getAdvertiserCampaign(token.getAccountId() + "", "", dateString);
-                        //2:获取当日广告计划数据
-                        advertiserDataService.getAdvertiserPlan(token, "", dateString, null);
-                        //3:获取当日创意数据
-                        creativeService.getAdvertiserCreative(token.getAccountId() + "", "", dateString);
-                    } catch (Exception e) {
-                        e.printStackTrace();
-                    } finally {
-                    }
+            executorService.submit(() -> {
+                try {
+                    //1:获取当日广告组数据
+                    advertiserDataService.getAdvertiserCampaign(token.getAccountId() + "", "", dateString);
+                    //2:获取当日广告计划数据
+                    advertiserDataService.getAdvertiserPlan(token, "", dateString, null);
+                    //3:获取当日创意数据
+                    creativeService.getAdvertiserCreative(token.getAccountId() + "", "", dateString);
+                } catch (Exception e) {
+                    e.printStackTrace();
                 }
             });
         });

+ 1 - 9
xxl-job-bytedance-executor/src/main/java/com/xxl/job/executor/handler/bytedance/BytedanceCreativeDailyReportLoadJob.java

@@ -39,15 +39,7 @@ public class BytedanceCreativeDailyReportLoadJob {
             return ReturnT.FAIL;
         }
         executorService = Executors.newFixedThreadPool(5);
-        tokens.forEach(token -> {
-            executorService.submit(new Runnable() {
-                @Override
-                public void run() {
-                    //获取广告创意信息数据
-                    reportService.getAdvertiserCreativeReport(token, getDate, getDate, CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
-                }
-            });
-        });
+        tokens.forEach(token -> executorService.submit(() -> reportService.getAdvertiserCreativeReport(token, getDate, getDate, CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY)));
         return ReturnT.SUCCESS;
     }
 

+ 6 - 10
xxl-job-bytedance-executor/src/main/java/com/xxl/job/executor/handler/bytedance/BytedanceCreativeHourlyReportLoadJob.java

@@ -44,17 +44,13 @@ public class BytedanceCreativeHourlyReportLoadJob {
         }
         executorService = Executors.newFixedThreadPool(4);
         tokens.forEach(token -> {
-            executorService.submit(new Runnable() {
-                @Override
-                public void run() {
-                    if (null != hour && "00".equals(hour)) {
-                        Date finalGetDate = DateUtils.addDay(getDate, -1);
-                        //获取广告创意信息数据
-                        reportService.getAdvertiserCreativeReport(token, finalGetDate, finalGetDate, CtopAdConstant.BYTEDANCE_REPORT_TYPE_HOURLY);
-                    }
-                    reportService.getAdvertiserCreativeReport(token, getDate, getDate, CtopAdConstant.BYTEDANCE_REPORT_TYPE_HOURLY);
+            executorService.submit(() -> {
+                if (null != hour && "00".equals(hour)) {
+                    Date finalGetDate = DateUtils.addDay(getDate, -1);
+                    //获取广告创意信息数据
+                    reportService.getAdvertiserCreativeReport(token, finalGetDate, finalGetDate, CtopAdConstant.BYTEDANCE_REPORT_TYPE_HOURLY);
                 }
-
+                reportService.getAdvertiserCreativeReport(token, getDate, getDate, CtopAdConstant.BYTEDANCE_REPORT_TYPE_HOURLY);
             });
 
         });

+ 1 - 4
xxl-job-bytedance-executor/src/main/java/com/xxl/job/executor/handler/bytedance/BytedancePlanDailyReportLoadJob.java

@@ -42,10 +42,7 @@ public class BytedancePlanDailyReportLoadJob {
             log.info("定时获取头条数据异常:为获取到可用的token");
             return ReturnT.FAIL;
         }
-        tokens.forEach(token -> {
-            //获取广告计划信息数据
-            reportService.getAdvertiserPlanReport(token, getDate, getDate, CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
-        });
+        tokens.forEach(token -> reportService.getAdvertiserPlanReport(token, getDate, getDate, CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY));
         formatZybData(215, getDate);
         planDailyReportService.cleanChannelCodeData(112, getDate);
         planDailyReportService.cleanChannelCodeData(113, getDate);

+ 10 - 13
xxl-job-bytedance-executor/src/main/java/com/xxl/job/executor/handler/bytedance/BytedancePlanHourlyReportLoadJob.java

@@ -47,20 +47,17 @@ public class BytedancePlanHourlyReportLoadJob implements Job {
         executorService = Executors.newFixedThreadPool(5);
         countDownLatch = new CountDownLatch(tokens.size());
         tokens.forEach(token -> {
-            executorService.submit(new Runnable() {
-                @Override
-                public void run() {
-                    try {
-                        if (null != hour && "00".equals(hour)) {
-                            Date finalGetDate = DateUtils.addDay(getDate, -1);
-                            reportService.getAdvertiserPlanReport(token, finalGetDate, finalGetDate, CtopAdConstant.BYTEDANCE_REPORT_TYPE_HOURLY);
-                        }
-                        reportService.getAdvertiserPlanReport(token, getDate, getDate, CtopAdConstant.BYTEDANCE_REPORT_TYPE_HOURLY);
-                    } catch (Exception e) {
-                        e.printStackTrace();
-                    } finally {
-                        countDownLatch.countDown();
+            executorService.submit(() -> {
+                try {
+                    if (null != hour && "00".equals(hour)) {
+                        Date finalGetDate = DateUtils.addDay(getDate, -1);
+                        reportService.getAdvertiserPlanReport(token, finalGetDate, finalGetDate, CtopAdConstant.BYTEDANCE_REPORT_TYPE_HOURLY);
                     }
+                    reportService.getAdvertiserPlanReport(token, getDate, getDate, CtopAdConstant.BYTEDANCE_REPORT_TYPE_HOURLY);
+                } catch (Exception e) {
+                    e.printStackTrace();
+                } finally {
+                    countDownLatch.countDown();
                 }
             });
         });

+ 1 - 14
xxl-job-bytedance-executor/src/main/java/com/xxl/job/executor/handler/bytedance/BytedancePlanLoadJob.java

@@ -28,20 +28,7 @@ public class BytedancePlanLoadJob {
         //1:查询当日数据
         List<CtopOauthToken> tokens = tokenService.getTokenListByType(CtopAdConstant.PLATFORM_TYPE_BYTEDANCE);
         executorService = Executors.newFixedThreadPool(4);
-        tokens.forEach(token -> {
-            executorService.submit(new Runnable() {
-                @Override
-                public void run() {
-                    try {
-                        //1:获取当日广告计划数据
-                        advertiserDataService.getAdvertiserPlan(token, "", null, null);
-                    } catch (Exception e) {
-                        e.printStackTrace();
-                    } finally {
-                    }
-                }
-            });
-        });
+        tokens.forEach(token -> executorService.submit(() -> advertiserDataService.getAdvertiserPlan(token, "", null, null)));
         log.info("头条全量计划数据获取完成");
         return ReturnT.SUCCESS;
     }

+ 2 - 15
xxl-job-bytedance-executor/src/main/java/com/xxl/job/executor/handler/bytedance/BytedancePlanRealTimeLoadJob.java

@@ -23,7 +23,7 @@ import java.util.concurrent.Executors;
 public class BytedancePlanRealTimeLoadJob {
     @Autowired
     private ICtopOauthTokenService tokenService;
-    static ExecutorService executorService = null;
+    private ExecutorService executorService = null;
     @Autowired
     private IByteDanceAdvertiserDataService advertiserDataService;
 
@@ -33,20 +33,7 @@ public class BytedancePlanRealTimeLoadJob {
         List<CtopOauthToken> tokens = tokenService.getTokenListByType(CtopAdConstant.PLATFORM_TYPE_BYTEDANCE);
         executorService = Executors.newFixedThreadPool(4);
         String nowDate = DateUtils.date2Str();
-        tokens.forEach(token -> {
-            executorService.submit(new Runnable() {
-                @Override
-                public void run() {
-                    try {
-                        //1:获取当日广告计划数据
-                        advertiserDataService.getAdvertiserPlan(token, "", nowDate, nowDate);
-                    } catch (Exception e) {
-                        e.printStackTrace();
-                    } finally {
-                    }
-                }
-            });
-        });
+        tokens.forEach(token -> executorService.submit(() -> advertiserDataService.getAdvertiserPlan(token, "", nowDate, nowDate)));
         log.info("头条全量计划数据获取完成");
         return ReturnT.SUCCESS;
     }

+ 2 - 25
xxl-job-bytedance-executor/src/main/java/com/xxl/job/executor/handler/bytedance/BytedanceSameDayCreativeLoadJob.java

@@ -13,7 +13,6 @@ import org.springframework.stereotype.Component;
 
 import java.util.Date;
 import java.util.List;
-import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 
@@ -23,37 +22,15 @@ public class BytedanceSameDayCreativeLoadJob {
     @Autowired
     private ICtopOauthTokenService tokenService;
     static ExecutorService executorService = null;
-    //线程计数器
-    static CountDownLatch countDownLatch = null;
     @Autowired
     private IByteDanceAdvertiserDataService advertiserDataService;
 
     @XxlJob("bytedanceSameDayCreative")
     public ReturnT<String> execute(String param) throws Exception {
         List<CtopOauthToken> tokens = tokenService.getTokenListByType(CtopAdConstant.PLATFORM_TYPE_BYTEDANCE);
+        String getDateStr = DateUtils.formatDate(new Date());
         executorService = Executors.newFixedThreadPool(6);
-        countDownLatch = new CountDownLatch(tokens.size());
-        tokens.forEach(token -> {
-            executorService.submit(new Runnable() {
-                @Override
-                public void run() {
-                    try {
-                        String getDateString = DateUtils.formatDate(new Date());
-                        advertiserDataService.getAdvertiserCreative(token, getDateString);
-                    } catch (Exception e) {
-                        e.printStackTrace();
-                    } finally {
-                        countDownLatch.countDown();
-                    }
-                }
-            });
-        });
-        try {
-            countDownLatch.await();
-        } catch (InterruptedException e) {
-            e.printStackTrace();
-        }
-        log.info("创意物料数据同步完成");
+        tokens.forEach(token -> executorService.submit(() -> advertiserDataService.getAdvertiserCreative(token, getDateStr)));
         return ReturnT.SUCCESS;
     }
 }

+ 1 - 25
xxl-job-bytedance-executor/src/main/java/com/xxl/job/executor/handler/bytedance/BytedanceVideoInfoLoadJob.java

@@ -11,7 +11,6 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
 import java.util.List;
-import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 
@@ -21,8 +20,6 @@ public class BytedanceVideoInfoLoadJob {
     @Autowired
     private ICtopOauthTokenService tokenService;
     static ExecutorService executorService = null;
-    //线程计数器
-    static CountDownLatch countDownLatch = null;
     @Autowired
     private IByteDanceAdvertiserDataService advertiserDataService;
 
@@ -30,28 +27,7 @@ public class BytedanceVideoInfoLoadJob {
     public ReturnT<String> execute(String param) throws Exception {
         List<CtopOauthToken> tokens = tokenService.getTokenListByType(CtopAdConstant.PLATFORM_TYPE_BYTEDANCE);
         executorService = Executors.newFixedThreadPool(4);
-        countDownLatch = new CountDownLatch(tokens.size());
-        tokens.forEach(token -> {
-            executorService.submit(new Runnable() {
-                @Override
-                public void run() {
-                    try {
-                        //获取全量视频素材数据
-                        advertiserDataService.getMaterialList(token);
-                    } catch (Exception e) {
-                        e.printStackTrace();
-                    } finally {
-                        countDownLatch.countDown();
-                    }
-                }
-            });
-        });
-        try {
-            countDownLatch.await();
-        } catch (InterruptedException e) {
-            e.printStackTrace();
-        }
-        log.info("物料数据同步完成");
+        tokens.forEach(token -> executorService.submit(() -> advertiserDataService.getMaterialList(token)));
         return ReturnT.SUCCESS;
     }
 }

+ 34 - 0
xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouCampaignJob.java

@@ -0,0 +1,34 @@
+package com.xxl.job.executor.handler.kuaishou;
+
+import cn.com.ctop.common.module.entity.CtopOauthToken;
+import cn.com.ctop.common.module.service.ICtopOauthTokenService;
+import cn.com.ctop.kuaishou.modules.batch.service.IKuaishouInterfaceService;
+import com.xxl.job.core.biz.model.ReturnT;
+import com.xxl.job.core.handler.annotation.XxlJob;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+@Slf4j
+@Component
+public class KuaishouCampaignJob {
+    @Autowired
+    private ICtopOauthTokenService tokenService;
+    @Autowired
+    private IKuaishouInterfaceService kuaishouInterfaceService;
+
+    static ExecutorService executorService = null;
+
+    @XxlJob("kuaishouCampaignJob")
+    public ReturnT<String> execute(String param) throws Exception {
+        List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
+        executorService = Executors.newFixedThreadPool(5);
+        tokens.forEach(token -> executorService.submit(() -> kuaishouInterfaceService.getCampaignList(token, null, null)));
+        log.info("快手物料数据同步完成");
+        return ReturnT.SUCCESS;
+    }
+}

+ 46 - 0
xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouDailyAccountReportLoadJob.java

@@ -0,0 +1,46 @@
+package com.xxl.job.executor.handler.kuaishou;
+
+import cn.com.ctop.common.module.entity.CtopOauthToken;
+import cn.com.ctop.common.module.service.ICtopOauthTokenService;
+import cn.com.ctop.kuaishou.modules.batch.service.IKuaishouInterfaceService;
+import com.xxl.job.core.biz.model.ReturnT;
+import com.xxl.job.core.handler.annotation.XxlJob;
+import lombok.extern.slf4j.Slf4j;
+import org.jeecg.common.util.DateUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+/**
+ * 快手广告主分天数据
+ *
+ * @author syh
+ */
+@Slf4j
+@Component
+public class KuaishouDailyAccountReportLoadJob {
+    @Autowired
+    private ICtopOauthTokenService tokenService;
+    @Autowired
+    private IKuaishouInterfaceService kuaishouInterfaceService;
+    static ExecutorService executorService = null;
+
+    @XxlJob("kuaishouDailyAccountReport")
+    public ReturnT<String> execute(String param) throws Exception {
+        Date getDate = DateUtils.addDay(new Date(), -1);
+        //1:查询当日数据
+        List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
+        if (null == tokens || tokens.size() <= 0) {
+            log.info("定时获取快手数据异常:未获取到可用的token");
+            return ReturnT.FAIL;
+        }
+
+        executorService = Executors.newFixedThreadPool(3);
+        tokens.forEach(token -> executorService.submit(() -> kuaishouInterfaceService.getAdvertiserReportDaily(token, getDate, getDate)));
+        return ReturnT.SUCCESS;
+    }
+}

+ 45 - 0
xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouDailyCampaignReportLoadJob.java

@@ -0,0 +1,45 @@
+package com.xxl.job.executor.handler.kuaishou;
+
+import cn.com.ctop.common.module.entity.CtopOauthToken;
+import cn.com.ctop.common.module.service.ICtopOauthTokenService;
+import cn.com.ctop.kuaishou.modules.batch.service.IKuaishouInterfaceService;
+import com.xxl.job.core.biz.model.ReturnT;
+import com.xxl.job.core.handler.annotation.XxlJob;
+import lombok.extern.slf4j.Slf4j;
+import org.jeecg.common.util.DateUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+/**
+ * 广告计划分天报表
+ *
+ * @author syh
+ */
+@Slf4j
+@Component
+public class KuaishouDailyCampaignReportLoadJob {
+    @Autowired
+    private ICtopOauthTokenService tokenService;
+    @Autowired
+    private IKuaishouInterfaceService kuaishouInterfaceService;
+    static ExecutorService executorService = null;
+
+    @XxlJob("kuaishouDailyCampaignReport")
+    public ReturnT<String> execute(String param) throws Exception {
+        Date getDate = DateUtils.addDay(new Date(), -1);
+        //1:查询当日数据
+        List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
+        if (null == tokens || tokens.size() <= 0) {
+            log.info("定时获取快手数据异常:未获取到可用的token");
+            return ReturnT.FAIL;
+        }
+        executorService = Executors.newFixedThreadPool(3);
+        tokens.forEach(token -> executorService.submit(() -> kuaishouInterfaceService.getAdvertiserCampaignReportDaily(token, getDate, getDate)));
+        return ReturnT.SUCCESS;
+    }
+}

+ 64 - 0
xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouDailyCreativeReportLoadJob.java

@@ -0,0 +1,64 @@
+package com.xxl.job.executor.handler.kuaishou;
+
+import cn.com.ctop.common.module.entity.CtopOauthToken;
+import cn.com.ctop.common.module.service.ICtopOauthTokenService;
+import cn.com.ctop.common.module.utils.Check;
+import cn.com.ctop.kuaishou.modules.batch.service.IKuaiShouCampaignService;
+import cn.com.ctop.kuaishou.modules.batch.service.IKuaishouInterfaceService;
+import com.alibaba.fastjson.JSONArray;
+import com.xxl.job.core.biz.model.ReturnT;
+import com.xxl.job.core.handler.annotation.XxlJob;
+import lombok.extern.slf4j.Slf4j;
+import org.jeecg.common.util.DateUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+/**
+ * 获取创意分天报表
+ *
+ * @author jeecg-boot
+ */
+@Slf4j
+@Component
+public class KuaishouDailyCreativeReportLoadJob {
+    @Autowired
+    private ICtopOauthTokenService tokenService;
+    @Autowired
+    private IKuaishouInterfaceService kuaishouInterfaceService;
+
+    static ExecutorService executorService = null;
+
+    @Autowired
+    private IKuaiShouCampaignService campaignService;
+
+    @XxlJob("kuaishouDailyCreativeReport")
+    public ReturnT<String> execute(String param) throws Exception {
+        Date getDate = DateUtils.addDay(new Date(), -1);
+        executorService = Executors.newFixedThreadPool(4);
+        List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
+        if (null == tokens || tokens.size() <= 0) {
+            log.info("定时获取快手数据异常:未获取到可用的token");
+            return ReturnT.FAIL;
+        }
+        tokens.forEach(token -> {
+            log.info("获取创意分天报表开始,accountId:{}", token.getAccountId());
+            kuaishouInterfaceService.getCampaignList(token, null, null);
+            try {
+                Thread.sleep(5 * 100);
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+            List<JSONArray> campaigns = campaignService.getJsonArrByAccount(token.getAccountId(), 50);
+            if (!Check.isNull(campaigns) && campaigns.size() > 0) {
+                campaigns.forEach(campaign -> executorService.submit(() -> kuaishouInterfaceService.getAdvertiserCreativeReportDaily(token, getDate, getDate, campaign)));
+            }
+        });
+        return ReturnT.SUCCESS;
+    }
+
+}

+ 45 - 0
xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouDailyGroupReportLoadJob.java

@@ -0,0 +1,45 @@
+package com.xxl.job.executor.handler.kuaishou;
+
+import cn.com.ctop.common.module.entity.CtopOauthToken;
+import cn.com.ctop.common.module.service.ICtopOauthTokenService;
+import cn.com.ctop.kuaishou.modules.batch.service.IKuaishouInterfaceService;
+import com.xxl.job.core.biz.model.ReturnT;
+import com.xxl.job.core.handler.annotation.XxlJob;
+import lombok.extern.slf4j.Slf4j;
+import org.jeecg.common.util.DateUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+/**
+ * 获取广告组分天报表
+ *
+ * @author syh
+ */
+@Slf4j
+@Component
+public class KuaishouDailyGroupReportLoadJob {
+    @Autowired
+    private ICtopOauthTokenService tokenService;
+    @Autowired
+    private IKuaishouInterfaceService kuaishouInterfaceService;
+    static ExecutorService executorService = null;
+
+    @XxlJob("kuaishouDailyGroupReport")
+    public ReturnT<String> execute(String param) throws Exception {
+        Date getDate = DateUtils.addDay(new Date(), -1);
+        //1:查询当日数据
+        List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
+        if (null == tokens || tokens.size() <= 0) {
+            log.info("定时获取快手数据异常:未获取到可用的token");
+            return ReturnT.FAIL;
+        }
+        executorService = Executors.newFixedThreadPool(3);
+        tokens.forEach(token -> executorService.submit(() -> kuaishouInterfaceService.getAdvertiserGroupReportDaily(token, getDate, getDate)));
+        return ReturnT.SUCCESS;
+    }
+}

+ 47 - 0
xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouDailyLoadFileJob.java

@@ -0,0 +1,47 @@
+package com.xxl.job.executor.handler.kuaishou;
+
+import cn.com.ctop.common.module.entity.CtopOauthToken;
+import cn.com.ctop.common.module.service.ICtopOauthTokenService;
+import cn.com.ctop.kuaishou.modules.batch.service.IKuaiShouDailyReportTaskService;
+import com.xxl.job.core.biz.model.ReturnT;
+import com.xxl.job.core.handler.annotation.XxlJob;
+import lombok.extern.slf4j.Slf4j;
+import org.jeecg.common.util.DateUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import static org.jeecg.common.util.DateUtils.getAnotherDay;
+
+/**
+ * 获取文件状态并下载入库
+ *
+ * @author yumeng
+ */
+@Slf4j
+@Component
+public class KuaishouDailyLoadFileJob {
+    @Autowired
+    private ICtopOauthTokenService tokenService;
+    private ExecutorService executorService = null;
+    @Autowired
+    private IKuaiShouDailyReportTaskService dailyReportTaskService;
+
+    @XxlJob("kuaishouDailyLoadFile")
+    public ReturnT<String> execute(String param) throws Exception {
+        String nowDate = DateUtils.getDate("yyyy-MM-dd");
+        String statDate = getAnotherDay("yyyy-MM-dd", nowDate, -1);
+        //  查询快手token
+        List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
+        if (null == tokens || tokens.size() <= 0) {
+            log.info("定时获取快手数据异常:未获取到可用的token");
+            return ReturnT.FAIL;
+        }
+        executorService = Executors.newFixedThreadPool(3);
+        tokens.forEach(token -> executorService.submit(() -> dailyReportTaskService.getTaskList(token.getAccountId(), token.getAccessToken(), statDate)));
+        return ReturnT.SUCCESS;
+    }
+}

+ 41 - 0
xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouDailyReportTaskJob.java

@@ -0,0 +1,41 @@
+package com.xxl.job.executor.handler.kuaishou;
+
+import cn.com.ctop.common.module.entity.CtopOauthToken;
+import cn.com.ctop.common.module.service.ICtopOauthTokenService;
+import cn.com.ctop.common.module.utils.CtopAdConstant;
+import cn.com.ctop.kuaishou.modules.batch.service.IKuaiShouHistoryReportTaskService;
+import com.xxl.job.core.biz.model.ReturnT;
+import com.xxl.job.core.handler.annotation.XxlJob;
+import lombok.extern.slf4j.Slf4j;
+import org.jeecg.common.util.DateUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.Date;
+import java.util.List;
+
+@Slf4j
+@Component
+public class KuaishouDailyReportTaskJob {
+    @Autowired
+    private ICtopOauthTokenService tokenService;
+    @Autowired
+    private IKuaiShouHistoryReportTaskService reportTaskService;
+
+    @XxlJob("kuaishouDailyReportTask")
+    public ReturnT<String> execute(String param) throws Exception {
+        log.info("快手获取日报数据任务执行开始");
+        Long start = System.currentTimeMillis();
+        Date getDate = DateUtils.addDay(new Date(), -1);
+        List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
+        if (null == tokens || tokens.size() <= 0) {
+            log.info("快手获取日报数据任务执行失败:未获取到可用的token");
+            return ReturnT.FAIL;
+        }
+        String getDateStr = DateUtils.formatDate(getDate);
+        tokens.forEach(token -> reportTaskService.createTask(token.getAccountId(), token.getAccessToken(), getDateStr, getDateStr, CtopAdConstant.KUAISHOU_LOAD_JOB_TYPE_DAILY));
+        Long end = System.currentTimeMillis();
+        log.info("快手获取日报数据任务执行结束,执行耗时:{}秒", (end - start) / 1000);
+        return ReturnT.SUCCESS;
+    }
+}

+ 34 - 0
xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouGroupJob.java

@@ -0,0 +1,34 @@
+package com.xxl.job.executor.handler.kuaishou;
+
+import cn.com.ctop.common.module.entity.CtopOauthToken;
+import cn.com.ctop.common.module.service.ICtopOauthTokenService;
+import cn.com.ctop.kuaishou.modules.batch.service.IKuaishouInterfaceService;
+import com.xxl.job.core.biz.model.ReturnT;
+import com.xxl.job.core.handler.annotation.XxlJob;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+@Slf4j
+@Component
+public class KuaishouGroupJob {
+    @Autowired
+    private ICtopOauthTokenService tokenService;
+    @Autowired
+    private IKuaishouInterfaceService kuaishouInterfaceService;
+
+    static ExecutorService executorService = null;
+
+    @XxlJob("KuaishouGroup")
+    public ReturnT<String> execute(String param) throws Exception {
+        List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
+        executorService = Executors.newFixedThreadPool(3);
+        tokens.forEach(token -> executorService.submit(() -> kuaishouInterfaceService.getGroupList(token, null, null)));
+        log.info("快手物料数据同步完成");
+        return ReturnT.SUCCESS;
+    }
+}

+ 26 - 0
xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouHistoryLoadFileJob.java

@@ -0,0 +1,26 @@
+package com.xxl.job.executor.handler.kuaishou;
+
+import cn.com.ctop.kuaishou.modules.batch.service.IKuaiShouHistoryReportTaskService;
+import com.xxl.job.core.biz.model.ReturnT;
+import com.xxl.job.core.handler.annotation.XxlJob;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+/**
+ * 获取文件状态并下载入库
+ *
+ * @author yumeng
+ */
+@Slf4j
+@Component
+public class KuaishouHistoryLoadFileJob {
+    @Autowired
+    private IKuaiShouHistoryReportTaskService historyReportTaskService;
+
+    @XxlJob("kuaishouHistoryLoadFile")
+    public ReturnT<String> execute(String param) throws Exception {
+        historyReportTaskService.getTaskList();
+        return ReturnT.SUCCESS;
+    }
+}

+ 52 - 0
xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouHourlyAccountReportLoadJob.java

@@ -0,0 +1,52 @@
+package com.xxl.job.executor.handler.kuaishou;
+
+import cn.com.ctop.common.module.entity.CtopOauthToken;
+import cn.com.ctop.common.module.service.ICtopOauthTokenService;
+import cn.com.ctop.kuaishou.modules.batch.service.IKuaishouInterfaceService;
+import com.xxl.job.core.biz.model.ReturnT;
+import com.xxl.job.core.handler.annotation.XxlJob;
+import lombok.extern.slf4j.Slf4j;
+import org.jeecg.common.util.DateUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+/**
+ * 获取广告主分时报表
+ *
+ * @author syh
+ */
+@Slf4j
+@Component
+public class KuaishouHourlyAccountReportLoadJob {
+    @Autowired
+    private ICtopOauthTokenService tokenService;
+    @Autowired
+    private IKuaishouInterfaceService kuaishouInterfaceService;
+    static ExecutorService executorService = null;
+
+    @XxlJob("kuaishouHourlyAccountReport")
+    public ReturnT<String> execute(String param) throws Exception {
+        Date getDate = new Date();
+        SimpleDateFormat simpleDateFormat = new SimpleDateFormat("HH");
+        String hour = simpleDateFormat.format(getDate);
+        if (null != hour && "00".equals(hour)) {
+            getDate = DateUtils.addDay(getDate, -1);
+        }
+        //1:查询当日数据
+        List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
+        if (null == tokens || tokens.size() <= 0) {
+            log.info("定时获取快手数据异常:未获取到可用的token");
+            return ReturnT.FAIL;
+        }
+        executorService = Executors.newFixedThreadPool(5);
+        Date finalGetDate = getDate;
+        tokens.forEach(token -> executorService.submit(() -> kuaishouInterfaceService.getAdvertiserReportHourly(token, finalGetDate, finalGetDate)));
+        return ReturnT.SUCCESS;
+    }
+}

+ 49 - 0
xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouHourlyAccountWarningLoadJob.java

@@ -0,0 +1,49 @@
+package com.xxl.job.executor.handler.kuaishou;
+
+import cn.com.ctop.common.module.entity.CtopOauthToken;
+import cn.com.ctop.common.module.service.ICtopOauthTokenService;
+import cn.com.ctop.kuaishou.modules.batch.service.IAccountWarningService;
+import com.xxl.job.core.biz.model.ReturnT;
+import com.xxl.job.core.handler.annotation.XxlJob;
+import lombok.extern.slf4j.Slf4j;
+import org.jeecg.common.util.DateUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import static org.jeecg.common.util.DateUtils.getAnotherDay;
+
+/**
+ * 获取广告主分时报表
+ *
+ * @author syh
+ */
+@Slf4j
+@Component
+public class KuaishouHourlyAccountWarningLoadJob {
+    @Autowired
+    private ICtopOauthTokenService tokenService;
+    @Autowired
+    private IAccountWarningService accountWarningService;
+    static ExecutorService executorService = null;
+
+    @XxlJob("kuaishouHourlyAccountWarning")
+    public ReturnT<String> execute(String param) throws Exception {
+        log.info("开始时段消耗预警定时任务");
+        long l = System.currentTimeMillis();
+        List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
+        if (null == tokens || tokens.size() <= 0) {
+            log.info("定时获取快手数据异常:未获取到可用的token");
+            return ReturnT.FAIL;
+        }
+        String statDate = DateUtils.getDate("yyyy-MM-dd");
+        String anotherDay = getAnotherDay("yyyy-MM-dd", statDate, -1);
+        executorService = Executors.newFixedThreadPool(3);
+        tokens.forEach(token -> executorService.submit(() -> accountWarningService.kuaiShouAccountWarning(token.getAccountId(), statDate, anotherDay)));
+        log.info("异步时段消耗预警定时任务执行完毕,耗时:{} ms", System.currentTimeMillis() - l);
+        return ReturnT.SUCCESS;
+    }
+}

+ 53 - 0
xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouHourlyCampaignReportLoadJob.java

@@ -0,0 +1,53 @@
+package com.xxl.job.executor.handler.kuaishou;
+
+import cn.com.ctop.common.module.entity.CtopOauthToken;
+import cn.com.ctop.common.module.service.ICtopOauthTokenService;
+import cn.com.ctop.kuaishou.modules.batch.service.IKuaishouInterfaceService;
+import com.xxl.job.core.biz.model.ReturnT;
+import com.xxl.job.core.handler.annotation.XxlJob;
+import lombok.extern.slf4j.Slf4j;
+import org.jeecg.common.util.DateUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+/**
+ * 获取广告计划分时报表
+ *
+ * @author syh
+ */
+@Slf4j
+@Component
+public class KuaishouHourlyCampaignReportLoadJob {
+    @Autowired
+    private ICtopOauthTokenService tokenService;
+    @Autowired
+    private IKuaishouInterfaceService kuaishouInterfaceService;
+
+    static ExecutorService executorService = null;
+
+    @XxlJob("kuaishouHourlyCampaignReport")
+    public ReturnT<String> execute(String param) throws Exception {
+        Date getDate = new Date();
+        SimpleDateFormat simpleDateFormat = new SimpleDateFormat("HH");
+        String hour = simpleDateFormat.format(getDate);
+        if (null != hour && "00".equals(hour)) {
+            getDate = DateUtils.addDay(getDate, -1);
+        }
+        //1:查询当日数据
+        List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
+        if (null == tokens || tokens.size() <= 0) {
+            log.info("定时获取快手数据异常:未获取到可用的token");
+            return ReturnT.FAIL;
+        }
+        executorService = Executors.newFixedThreadPool(5);
+        Date finalGetDate = getDate;
+        tokens.forEach(token -> executorService.submit(() -> kuaishouInterfaceService.getAdvertiserCampaignReportHourly(token, finalGetDate, finalGetDate)));
+        return ReturnT.SUCCESS;
+    }
+}

+ 53 - 0
xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouHourlyCreativeReportLoadJob.java

@@ -0,0 +1,53 @@
+package com.xxl.job.executor.handler.kuaishou;
+
+import cn.com.ctop.common.module.entity.CtopOauthToken;
+import cn.com.ctop.common.module.service.ICtopOauthTokenService;
+import cn.com.ctop.kuaishou.modules.batch.service.IKuaishouInterfaceService;
+import com.xxl.job.core.biz.model.ReturnT;
+import com.xxl.job.core.handler.annotation.XxlJob;
+import lombok.extern.slf4j.Slf4j;
+import org.jeecg.common.util.DateUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+/**
+ * 获取 广告创意分时数据
+ *
+ * @author syh
+ */
+@Slf4j
+@Component
+public class KuaishouHourlyCreativeReportLoadJob {
+    @Autowired
+    private ICtopOauthTokenService tokenService;
+    @Autowired
+    private IKuaishouInterfaceService kuaishouInterfaceService;
+
+    static ExecutorService executorService = null;
+
+    @XxlJob("KuaishouHourlyCreativeReport")
+    public ReturnT<String> execute(String param) throws Exception {
+        Date getDate = new Date();
+        SimpleDateFormat simpleDateFormat = new SimpleDateFormat("HH");
+        String hour = simpleDateFormat.format(getDate);
+        if (null != hour && "00".equals(hour)) {
+            getDate = DateUtils.addDay(getDate, -1);
+        }
+        //1:查询当日数据
+        List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
+        if (null == tokens || tokens.size() <= 0) {
+            log.info("定时获取快手数据异常:未获取到可用的token");
+            return ReturnT.FAIL;
+        }
+        executorService = Executors.newFixedThreadPool(3);
+        Date finalGetDate = getDate;
+        tokens.forEach(token -> executorService.submit(() -> kuaishouInterfaceService.getAdvertiserCreativeReportHourly(token, finalGetDate, finalGetDate)));
+        return ReturnT.SUCCESS;
+    }
+}

+ 52 - 0
xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouHourlyGroupReportLoadJob.java

@@ -0,0 +1,52 @@
+package com.xxl.job.executor.handler.kuaishou;
+
+import cn.com.ctop.common.module.entity.CtopOauthToken;
+import cn.com.ctop.common.module.service.ICtopOauthTokenService;
+import cn.com.ctop.kuaishou.modules.batch.service.IKuaishouInterfaceService;
+import com.xxl.job.core.biz.model.ReturnT;
+import com.xxl.job.core.handler.annotation.XxlJob;
+import lombok.extern.slf4j.Slf4j;
+import org.jeecg.common.util.DateUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+/**
+ * 获取 广告组分时数据
+ *
+ * @author syh
+ */
+@Slf4j
+@Component
+public class KuaishouHourlyGroupReportLoadJob {
+    @Autowired
+    private ICtopOauthTokenService tokenService;
+    @Autowired
+    private IKuaishouInterfaceService kuaishouInterfaceService;
+    static ExecutorService executorService = null;
+
+    @XxlJob("kuaishouHourlyGroupReport")
+    public ReturnT<String> execute(String param) throws Exception {
+        Date getDate = new Date();
+        SimpleDateFormat simpleDateFormat = new SimpleDateFormat("HH");
+        String hour = simpleDateFormat.format(getDate);
+        if (null != hour && "00".equals(hour)) {
+            getDate = DateUtils.addDay(getDate, -1);
+        }
+        //1:查询当日数据
+        List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
+        if (null == tokens || tokens.size() <= 0) {
+            log.info("定时获取快手数据异常:未获取到可用的token");
+            return ReturnT.FAIL;
+        }
+        executorService = Executors.newFixedThreadPool(5);
+        Date finalGetDate = getDate;
+        tokens.forEach(token -> executorService.submit(() -> kuaishouInterfaceService.getAdvertiserGroupReportHourly(token, finalGetDate, finalGetDate)));
+        return ReturnT.SUCCESS;
+    }
+}

+ 43 - 0
xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouMaterialsLoadJob.java

@@ -0,0 +1,43 @@
+package com.xxl.job.executor.handler.kuaishou;
+
+import cn.com.ctop.common.module.entity.CtopOauthToken;
+import cn.com.ctop.common.module.service.ICtopOauthTokenService;
+import cn.com.ctop.kuaishou.modules.batch.service.IKuaishouInterfaceService;
+import com.xxl.job.core.biz.model.ReturnT;
+import com.xxl.job.core.handler.annotation.XxlJob;
+import lombok.extern.slf4j.Slf4j;
+import org.jeecg.common.util.DateUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+@Slf4j
+public class KuaishouMaterialsLoadJob {
+    @Autowired
+    private ICtopOauthTokenService tokenService;
+    @Autowired
+    private IKuaishouInterfaceService kuaishouInterfaceService;
+
+    static ExecutorService executorService = null;
+
+    @XxlJob("kuaishouMaterials")
+    public ReturnT<String> execute(String param) throws Exception {
+        //增加接口的开始和结束日期
+        Date startDate = DateUtils.addDay(new Date(), -1);
+        Date endDate = new Date();
+        //1:查询当日数据
+        List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
+        executorService = Executors.newFixedThreadPool(8);
+        tokens.forEach(token -> executorService.submit(() -> {
+            //获取全量视频素材数据
+            kuaishouInterfaceService.getVideoList(token, startDate, endDate);
+            // 获取图片信息数据
+            kuaishouInterfaceService.getImageList(token, startDate, endDate);
+        }));
+        log.info("快手物料数据同步完成");
+        return ReturnT.SUCCESS;
+    }
+}

+ 42 - 0
xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouNowDateCampaignJob.java

@@ -0,0 +1,42 @@
+package com.xxl.job.executor.handler.kuaishou;
+
+import cn.com.ctop.common.module.entity.CtopOauthToken;
+import cn.com.ctop.common.module.service.ICtopOauthTokenService;
+import cn.com.ctop.kuaishou.modules.batch.service.IKuaishouInterfaceService;
+import com.xxl.job.core.biz.model.ReturnT;
+import com.xxl.job.core.handler.annotation.XxlJob;
+import lombok.extern.slf4j.Slf4j;
+import org.jeecg.common.util.DateUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+@Slf4j
+@Component
+public class KuaishouNowDateCampaignJob {
+    @Autowired
+    private ICtopOauthTokenService tokenService;
+    @Autowired
+    private IKuaishouInterfaceService kuaishouInterfaceService;
+
+    static ExecutorService executorService = null;
+
+    @XxlJob("kuaishouNowDateCampaign")
+    public ReturnT<String> execute(String param) throws Exception {
+        //1:查询当日数据
+        Date endDate = DateUtils.getDate();
+        String anotherDay = DateUtils.getAnotherDay("yyyy-MM-dd", DateUtils.formatDate(endDate), -1);
+        SimpleDateFormat sim = new SimpleDateFormat("yyyy-MM-dd");
+        Date startDate = sim.parse(anotherDay);
+        List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
+        executorService = Executors.newFixedThreadPool(5);
+        tokens.forEach(token -> executorService.submit(() -> kuaishouInterfaceService.getCampaignList(token, startDate, endDate)));
+        log.info("快手物料数据同步完成");
+        return ReturnT.SUCCESS;
+    }
+}

+ 40 - 0
xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouNowDateCreativeJob.java

@@ -0,0 +1,40 @@
+package com.xxl.job.executor.handler.kuaishou;
+
+import cn.com.ctop.common.module.entity.CtopOauthToken;
+import cn.com.ctop.common.module.service.ICtopOauthTokenService;
+import cn.com.ctop.kuaishou.modules.batch.service.IKuaishouInterfaceService;
+import com.xxl.job.core.biz.model.ReturnT;
+import com.xxl.job.core.handler.annotation.XxlJob;
+import lombok.extern.slf4j.Slf4j;
+import org.jeecg.common.util.DateUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+@Slf4j
+public class KuaishouNowDateCreativeJob {
+    @Autowired
+    private ICtopOauthTokenService tokenService;
+    @Autowired
+    private IKuaishouInterfaceService kuaishouInterfaceService;
+
+    static ExecutorService executorService = null;
+
+    @XxlJob("kuaishouNowDateCreative")
+    public ReturnT<String> execute(String param) throws Exception {
+        Date endDate = DateUtils.getDate();
+        String anotherDay = DateUtils.getAnotherDay("yyyy-MM-dd", DateUtils.formatDate(endDate), -1);
+        SimpleDateFormat sim = new SimpleDateFormat("yyyy-MM-dd");
+        Date startDate = sim.parse(anotherDay);
+        List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
+        executorService = Executors.newFixedThreadPool(5);
+        tokens.forEach(token -> executorService.submit(() -> kuaishouInterfaceService.getCreativeList(token, startDate, endDate)));
+        log.info("快手物料数据同步完成");
+        return ReturnT.SUCCESS;
+    }
+
+}

+ 41 - 0
xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouNowDateGroupJob.java

@@ -0,0 +1,41 @@
+package com.xxl.job.executor.handler.kuaishou;
+
+import cn.com.ctop.common.module.entity.CtopOauthToken;
+import cn.com.ctop.common.module.service.ICtopOauthTokenService;
+import cn.com.ctop.kuaishou.modules.batch.service.IKuaishouInterfaceService;
+import com.xxl.job.core.biz.model.ReturnT;
+import com.xxl.job.core.handler.annotation.XxlJob;
+import lombok.extern.slf4j.Slf4j;
+import org.jeecg.common.util.DateUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+@Slf4j
+public class KuaishouNowDateGroupJob {
+    @Autowired
+    private ICtopOauthTokenService tokenService;
+    @Autowired
+    private IKuaishouInterfaceService kuaishouInterfaceService;
+
+    static ExecutorService executorService = null;
+
+    @XxlJob("kuaishouNowDateGroup")
+    public ReturnT<String> execute(String param) throws Exception {
+        //1:查询当日数据
+        Date endDate = DateUtils.getDate();
+        String anotherDay = DateUtils.getAnotherDay("yyyy-MM-dd", DateUtils.formatDate(endDate), -1);
+        SimpleDateFormat sim = new SimpleDateFormat("yyyy-MM-dd");
+        Date startDate = sim.parse(anotherDay);
+        List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
+        executorService = Executors.newFixedThreadPool(5);
+        tokens.forEach(token -> executorService.submit(() -> kuaishouInterfaceService.getGroupList(token, startDate, endDate)));
+        log.info("快手物料数据同步完成");
+        return ReturnT.SUCCESS;
+    }
+
+}

+ 21 - 0
xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouReportDailyGroupChannelCodeJob.java

@@ -0,0 +1,21 @@
+package com.xxl.job.executor.handler.kuaishou;
+
+import cn.com.ctop.kuaishou.modules.report.service.IKuaishouReportDailyGroupService;
+import com.xxl.job.core.biz.model.ReturnT;
+import com.xxl.job.core.handler.annotation.XxlJob;
+import org.jeecg.common.util.DateUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.util.Date;
+
+public class KuaishouReportDailyGroupChannelCodeJob {
+    @Autowired
+    private IKuaishouReportDailyGroupService groupService;
+
+    @XxlJob("kuaishouReportDailyGroupChannelCode")
+    public ReturnT<String> execute(String param) throws Exception {
+        String date = DateUtils.formatDate(DateUtils.addDay(new Date(), -1));
+        groupService.formatChannelCode(date);
+        return ReturnT.SUCCESS;
+    }
+}

+ 33 - 0
xxl-job-executor/src/main/java/com/xxl/job/executor/handler/kuaishou/KuaishouSuZaoLoadJob.java

@@ -0,0 +1,33 @@
+package com.xxl.job.executor.handler.kuaishou;
+
+import cn.com.ctop.common.module.entity.CtopOauthToken;
+import cn.com.ctop.common.module.service.ICtopOauthTokenService;
+import cn.com.ctop.kuaishou.modules.batch.service.IKuaishouInterfaceService;
+import com.xxl.job.core.biz.model.ReturnT;
+import com.xxl.job.core.handler.annotation.XxlJob;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+@Slf4j
+@Component
+public class KuaishouSuZaoLoadJob {
+    @Autowired
+    private ICtopOauthTokenService tokenService;
+    @Autowired
+    private IKuaishouInterfaceService kuaishouInterfaceService;
+    static ExecutorService executorService = null;
+
+    @XxlJob("kuaishouSuZao")
+    public ReturnT<String> execute(String param) throws Exception {
+        List<CtopOauthToken> tokens = tokenService.selectKuaiShouToken();
+        executorService = Executors.newFixedThreadPool(8);
+        tokens.forEach(token -> executorService.submit(() -> kuaishouInterfaceService.getSuZaoList(token.getAccessToken(), token.getAccountId(), 1)));
+        log.info("快手素造数据同步完成");
+        return ReturnT.SUCCESS;
+    }
+}

xxl-job-bytedance-executor/src/main/resources/application-dev.yml → xxl-job-executor/src/main/resources/application-dev.yml


xxl-job-bytedance-executor/src/main/resources/application-prod.yml → xxl-job-executor/src/main/resources/application-prod.yml


xxl-job-bytedance-executor/src/main/resources/application-test.yml → xxl-job-executor/src/main/resources/application-test.yml


xxl-job-bytedance-executor/src/main/resources/application.yml → xxl-job-executor/src/main/resources/application.yml


xxl-job-bytedance-executor/src/main/resources/logback-dev.xml → xxl-job-executor/src/main/resources/logback-dev.xml


xxl-job-bytedance-executor/src/main/resources/logback-prod.xml → xxl-job-executor/src/main/resources/logback-prod.xml


xxl-job-bytedance-executor/src/main/resources/logback-test.xml → xxl-job-executor/src/main/resources/logback-test.xml


xxl-job-bytedance-executor/src/main/resources/logback.xml → xxl-job-executor/src/main/resources/logback.xml