Selaa lähdekoodia

修改穿山甲数据接口

syh 5 vuotta sitten
vanhempi
commit
183a26dc22
24 muutettua tiedostoa jossa 403 lisäystä ja 49 poistoa
  1. 1 0
      jeecg-boot-module-system/src/main/java/org/jeecg/config/ShiroConfig.java
  2. 1 1
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/BindAccountLoginController.java
  3. 40 11
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/PangolinController.java
  4. 3 2
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/job/PangolinDataJob.java
  5. 7 4
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/job/PangolinLoginJob.java
  6. 5 4
      jeecg-boot-module-system/src/main/resources/application.yml
  7. 87 8
      jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java
  8. 1 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/entity/PangolinReportBillMonth.java
  9. 1 1
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/entity/PangolinReportChannelDaily.java
  10. 6 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/PangolinReportBillMonthMapper.java
  11. 2 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/PangolinReportChannelDailyMapper.java
  12. 1 1
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/xml/PangolinReportAppHourMapper.xml
  13. 31 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/xml/PangolinReportBillMonthMapper.xml
  14. 36 4
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/xml/PangolinReportChannelDailyMapper.xml
  15. 2 1
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/PangolinReportAppHourService.java
  16. 3 3
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/PangolinReportBillMonthService.java
  17. 11 2
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinCrawlerServiceImpl.java
  18. 4 1
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinReportAppHourServiceImpl.java
  19. 23 3
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinReportBillMonthServiceImpl.java
  20. 34 3
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinReportChannelDailyServiceImpl.java
  21. 16 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/vo/PangolinReportAppMonthVO.java
  22. 45 0
      module-job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedanceFlowFundJob.java
  23. 36 0
      module-job-kuaishou/src/main/java/cn/com/ctop/job/kuaishou/handler/KuaiShouCleanMaterialJob.java
  24. 7 0
      pom.xml

+ 1 - 0
jeecg-boot-module-system/src/main/java/org/jeecg/config/ShiroConfig.java

@@ -148,6 +148,7 @@ public class ShiroConfig {
 		filterChainDefinitionMap.put("/ks/web/test", "anon");
         filterChainDefinitionMap.put("/ureport/**", "anon");
         filterChainDefinitionMap.put("/ctop/advertiser/**", "anon");
+		filterChainDefinitionMap.put("/pangolin/**", "anon");
 
 		// 添加自己的过滤器并且取名为jwt
 		Map<String, Filter> filterMap = new HashMap<String, Filter>(1);

+ 1 - 1
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/BindAccountLoginController.java

@@ -61,12 +61,12 @@ public class BindAccountLoginController {
         }
         //创建人
         String userId = json.getString("userId");
-        String password = json.getString("password");
         String loginType = json.getString("loginType");
         // 广告主名称
         String advertiserName = json.getString("advertiser_name");
         //账号
         String accountName = json.getString("accountName");
+        String password = json.getString("password");
         Map<String, Object> stringObjectMap = createInternalService.bindLogin(advertiserId, accountName, password, loginType, userId, advertiserName);
         Boolean trueOrFalse = (Boolean) stringObjectMap.get("success");
         if (Boolean.TRUE.equals(trueOrFalse)) {

+ 40 - 11
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/PangolinController.java

@@ -1,18 +1,24 @@
 package org.jeecg.modules.ctop.controller;
 
+import cn.com.ctop.common.module.service.ISysRoleService;
+import cn.com.ctop.common.module.utils.ResultMapUtils;
+import cn.com.ctop.common.module.utils.StatusCode;
 import cn.com.ctop.crawler.modules.pangolin.service.PangolinReportAppHourService;
 import cn.com.ctop.crawler.modules.pangolin.service.PangolinReportBillMonthService;
 import cn.com.ctop.crawler.modules.pangolin.service.PangolinReportChannelDailyService;
+import cn.com.ctop.crawler.modules.pangolin.vo.PangolinReportAppMonthVO;
 import com.alibaba.fastjson.JSONObject;
-import org.apache.shiro.SecurityUtils;
-import org.jeecg.common.system.vo.LoginUser;
+import com.github.pagehelper.PageInfo;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
 
+import java.util.HashMap;
 import java.util.Map;
 
+@RestController
 @RequestMapping("pangolin")
 public class PangolinController {
     @Autowired
@@ -21,14 +27,21 @@ public class PangolinController {
     private PangolinReportChannelDailyService pangolinReportChannelDailyService;
     @Autowired
     private PangolinReportBillMonthService pangolinReportBillMonthService;
-
+    @Autowired
+    private ISysRoleService sysRoleService;
     /**
      * 实时数据
      */
     @PostMapping("api/realtime")
-    public Map<String, Object> realTime() {
-        LoginUser user = (LoginUser) SecurityUtils.getSubject().getPrincipal();
-        return pangolinReportAppHourService.getRealTimeData(user.getId());
+    public Map<String, Object> realTime(@RequestBody JSONObject data) {
+//        LoginUser user = (LoginUser) SecurityUtils.getSubject().getPrincipal();
+//        String userId = user.getId();
+        String userId = "6b4c821adc414dc8b7718ab63ccbfcaf";
+        String roleCode = sysRoleService.getRoleCodeByUserId(userId);
+        if(null!=roleCode&&(roleCode.equals("admin"))){
+            userId = null;
+        }
+        return pangolinReportAppHourService.getRealTimeData(userId,data);
     }
 
     /**
@@ -36,17 +49,33 @@ public class PangolinController {
      */
     @PostMapping("api/activation/list")
     public Map<String, Object> activationList(@RequestBody JSONObject data) {
-        LoginUser user = (LoginUser) SecurityUtils.getSubject().getPrincipal();
-        return pangolinReportChannelDailyService.getActivationList(user.getId(), data);
+//        LoginUser user = (LoginUser) SecurityUtils.getSubject().getPrincipal();
+//        String userId = user.getId();
+        String userId = "6b4c821adc414dc8b7718ab63ccbfcaf";
+        String roleCode = sysRoleService.getRoleCodeByUserId(userId);
+        if(null!=roleCode&&(roleCode.equals("admin"))){
+            userId = null;
+        }
+        return pangolinReportChannelDailyService.getActivationList(userId, data);
     }
 
     /**
      * 对账单
      */
     @PostMapping("/api/checkApps/list")
-    public Map<String, Object> checkAppList(@RequestBody JSONObject data) {
-        LoginUser user = (LoginUser) SecurityUtils.getSubject().getPrincipal();
-        return pangolinReportBillMonthService.checkAppList(user.getId(), data);
+    public Map<String,Object> checkAppList(@RequestBody JSONObject data) {
+        Map<String,Object> result = new HashMap<>();
+        //        LoginUser user = (LoginUser) SecurityUtils.getSubject().getPrincipal();
+//        String userId = user.getId();
+        String userId = "6b4c821adc414dc8b7718ab63ccbfcaf";
+        String roleCode = sysRoleService.getRoleCodeByUserId(userId);
+        if(null!=roleCode&&(roleCode.equals("admin"))){
+            userId = null;
+        }
+        PageInfo<PangolinReportAppMonthVO> pageInfo = pangolinReportBillMonthService.checkAppList(userId, data);
+        result.put("data",pageInfo);
+        ResultMapUtils.setResultMap(result, StatusCode.COMMON_SUCCESS);
+        return result;
     }
 
 }

+ 3 - 2
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/job/PangolinDataJob.java

@@ -1,9 +1,10 @@
-package cn.com.ctop.crawler.modules.pangolin.job;
+package org.jeecg.modules.ctop.job;
 
 import cn.com.ctop.common.module.entity.BindAccountLogin;
 import cn.com.ctop.common.module.service.IBindAccountLoginService;
 import cn.com.ctop.crawler.modules.pangolin.entity.PangolinApp;
-import cn.com.ctop.crawler.modules.pangolin.service.*;
+import cn.com.ctop.crawler.modules.pangolin.service.PangolinAppService;
+import cn.com.ctop.crawler.modules.pangolin.service.PangolinCrawlerService;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import org.jeecg.common.util.DateUtils;
 import org.quartz.Job;

+ 7 - 4
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/job/PangolinLoginJob.java

@@ -1,4 +1,4 @@
-package cn.com.ctop.crawler.modules.pangolin.job;
+package org.jeecg.modules.ctop.job;
 
 import cn.com.ctop.common.module.entity.BindAccountLogin;
 import cn.com.ctop.common.module.service.IBindAccountLoginService;
@@ -21,10 +21,13 @@ public class PangolinLoginJob implements Job {
         QueryWrapper<BindAccountLogin> queryWrapper = new QueryWrapper<>();
         queryWrapper.eq("login_type","pangolin");
         queryWrapper.eq("status",1);
-        queryWrapper.isNull("cookie");
         List<BindAccountLogin> list = bindAccountLoginService.list(queryWrapper);
-        if (list != null && !list.isEmpty()) {
-            list.forEach(bindAccountLogin -> pangolinLoginService.pangolinLogin(bindAccountLogin.getAccountName(), bindAccountLogin.getPassword()));
+        if(list!=null&&!list.isEmpty()){
+            for (BindAccountLogin login:list) {
+                if(null==login.getCookie()||"".equals(login.getCookie().trim())){
+                    pangolinLoginService.pangolinLogin(login.getAccountName(), login.getPassword());
+                }
+            }
         }
     }
 }

+ 5 - 4
jeecg-boot-module-system/src/main/resources/application.yml

@@ -3,10 +3,11 @@ spring:
     active: @activatedProperties@
   application:
     name: system-web
-eureka:
-  client:
-    serviceUrl:
-      defaultZone: http://localhost:8761/eureka/
+#分页pageHelper
+pagehelper:
+  helper-dialect: mysql
+  reasonable: true
+  support-methods-arguments: true
 swagger:
   production: false
   basic:

+ 87 - 8
jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java

@@ -1,33 +1,112 @@
 package org.jeecg;
 
+import cn.com.ctop.common.module.entity.BindAccountLogin;
 import cn.com.ctop.common.module.entity.CtopOauthToken;
+import cn.com.ctop.common.module.service.IBindAccountLoginService;
 import cn.com.ctop.common.module.service.ICtopOauthTokenService;
 import cn.com.ctop.common.module.utils.CtopAdConstant;
-import cn.com.ctop.toutiao.modules.report.service.IReportService;
+import cn.com.ctop.crawler.modules.pangolin.entity.PangolinApp;
+import cn.com.ctop.crawler.modules.pangolin.service.PangolinAppService;
+import cn.com.ctop.crawler.modules.pangolin.service.PangolinCrawlerService;
+import cn.com.ctop.crawler.modules.pangolin.service.PangolinLoginService;
+import cn.com.ctop.toutiao.modules.report.service.IBytedanceFundDailyService;
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import lombok.extern.slf4j.Slf4j;
+import org.jeecg.common.util.DateUtils;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.test.context.junit4.SpringRunner;
 
-import java.util.Date;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 
 @RunWith(SpringRunner.class)
 @SpringBootTest
 @Slf4j
 public class SampleTest {
     @Autowired
-    private ICtopOauthTokenService tokenService;
+    private IBindAccountLoginService bindAccountLoginService;
+    @Autowired
+    private PangolinLoginService pangolinLoginService;
+    @Autowired
+    private PangolinCrawlerService pangolinCrawlerService;
+    @Autowired
+    private PangolinAppService pangolinAppService;
+    @Test
+    public void testPangolinLogin(){
+        QueryWrapper<BindAccountLogin> queryWrapper = new QueryWrapper<>();
+        queryWrapper.eq("login_type","pangolin");
+        queryWrapper.eq("status",1);
+        List<BindAccountLogin> list = bindAccountLoginService.list(queryWrapper);
+        if(list!=null&&!list.isEmpty()){
+            for (BindAccountLogin login:list) {
+                if(null==login.getCookie()||"".equals(login.getCookie().trim())){
+                    pangolinLoginService.pangolinLogin(login.getAccountName(), login.getPassword());
+                }
+            }
+        }
+    }
+
+    @Test
+    public void testPangolinData(){
+        QueryWrapper<BindAccountLogin> queryWrapper = new QueryWrapper<>();
+        queryWrapper.eq("login_type", "pangolin");
+        queryWrapper.eq("status", 1);
+        queryWrapper.isNotNull("cookie");
+        List<BindAccountLogin> list = bindAccountLoginService.list(queryWrapper);
+        if (list != null && !list.isEmpty()) {
+            for (BindAccountLogin bindAccountLogin : list) {
+                pangolinCrawlerService.getChannelList(bindAccountLogin);
+                pangolinCrawlerService.getActivationList(bindAccountLogin, DateUtils.getNowDate("yyyy-MM-dd"), 1);
+                pangolinCrawlerService.getCheckList(bindAccountLogin);
+                QueryWrapper<PangolinApp> queryWrapper1 = new QueryWrapper<>();
+                queryWrapper1.eq("account_name", bindAccountLogin.getAccountName());
+                List<PangolinApp> appList = pangolinAppService.list(queryWrapper1);
+                if (appList != null && !appList.isEmpty()) {
+                    appList.forEach(pangolinApp -> pangolinCrawlerService.getRealTimeList(bindAccountLogin, pangolinApp.getId()));
+                }
+            }
+        }
+    }
 
     @Autowired
-    private IReportService reportService;
+    private ICtopOauthTokenService tokenService;
+    @Autowired
+    private IBytedanceFundDailyService bytedanceFundDailyService;
+    static ExecutorService executorService = null;
+
 
     @Test
-    public void kuaisShouReport() {
-        CtopOauthToken token = tokenService.getTokenByAccountId(1666360425745416L);
-        Date getDate = new Date();
-        reportService.getAdvertiserReport(token, getDate, getDate, CtopAdConstant.BYTEDANCE_REPORT_TYPE_HOURLY);
+    public void testflowFund(){
+        List<CtopOauthToken> tokens = tokenService.getTokenListByType(CtopAdConstant.PLATFORM_TYPE_BYTEDANCE);
+        if (null == tokens || tokens.size() <= 0) {
+            log.error("头条账户流水数据异常:未获取到可用的token");
+            return ;
+        }
+        CountDownLatch countDownLatch   = new CountDownLatch(tokens.size());
+        executorService = Executors.newFixedThreadPool(5);
+        tokens.forEach(token -> {
+            executorService.submit(() -> {
+                try {
+                    for(int i=0;i<60;i++){
+                        String findDate = DateUtils.addDay(DateUtils.formatDate(),-i);
+                        bytedanceFundDailyService.loadFundDataByPage(token, findDate, findDate,1);
+                    }
+                }catch (Exception e){
+                }finally {
+                    countDownLatch.countDown();
+                }
+            });
+        });
+        try {
+            countDownLatch.await();
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+        }
     }
 }
 

+ 1 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/entity/PangolinReportBillMonth.java

@@ -19,6 +19,7 @@ public class PangolinReportBillMonth {
      * 月
      */
     private Integer month;
+    private String date;
     /**
      * appId
      */

+ 1 - 1
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/entity/PangolinReportChannelDaily.java

@@ -27,7 +27,7 @@ public class PangolinReportChannelDaily {
     /**
      * 操作系统
      */
-    private String os;
+    private String oprateSystem;
     /**
      * 激活数
      */

+ 6 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/PangolinReportBillMonthMapper.java

@@ -1,7 +1,13 @@
 package cn.com.ctop.crawler.modules.pangolin.mapper;
 
 import cn.com.ctop.crawler.modules.pangolin.entity.PangolinReportBillMonth;
+import cn.com.ctop.crawler.modules.pangolin.vo.PangolinReportAppMonthVO;
 import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import org.apache.ibatis.annotations.Param;
+
+import java.util.List;
 
 public interface PangolinReportBillMonthMapper extends BaseMapper<PangolinReportBillMonth> {
+    List<PangolinReportAppMonthVO> checkAppList(@Param(value = "userId") String userId, @Param(value = "startDate") String startDate,
+                                                @Param(value = "endDate") String endDate,@Param(value = "operateSystem") String operateSystem);
 }

+ 2 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/PangolinReportChannelDailyMapper.java

@@ -7,5 +7,7 @@ import org.apache.ibatis.annotations.Param;
 import java.util.List;
 
 public interface PangolinReportChannelDailyMapper extends BaseMapper<PangolinReportChannelDaily> {
+    Integer sumAppList(@Param(value = "startDate") String startDate, @Param(value = "endDate") String endDate, @Param(value = "userId") String userId, @Param(value = "type") String type);
+
     List<PangolinReportChannelDaily> getActivationList(@Param(value = "startDate") String startDate, @Param(value = "endDate") String endDate, @Param(value = "userId") String userId, @Param(value = "type") String type);
 }

+ 1 - 1
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/xml/PangolinReportAppHourMapper.xml

@@ -10,7 +10,7 @@
         from ctop_pangolin_report_app_hour appHourRate
         where appHourRate.date = appHourActive.date
         and appHourRate.hour = appHourActive.hour
-        and appHourRate.type = 'rate') as 'rateNum',
+        and appHourRate.type = 'rate') as 'activeRate',
         (select appHourClick.num
         from ctop_pangolin_report_app_hour appHourClick
         where appHourClick.date = appHourActive.date

+ 31 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/xml/PangolinReportBillMonthMapper.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="cn.com.ctop.crawler.modules.pangolin.mapper.PangolinReportBillMonthMapper">
+    <select id="checkAppList" resultType="cn.com.ctop.crawler.modules.pangolin.vo.PangolinReportAppMonthVO">
+        select
+       bill.date,
+       app.name as 'app_name',
+       bill.os as 'operate_system',
+       sum(bill.sum_internal_activations) as 'sum_internal_activations',
+       sum(bill.estimated_total_price) as 'estimated_total_price',
+       round(avg(bill.account_quality_score),2) as 'account_quality_score',
+       sum(bill.actual_total_price) as 'actual_total_price'
+from ctop_pangolin_report_bill_month bill
+         left join ctop_pangolin_app app on app.id = bill.app_id
+         left join ctop_bind_account_login login on login.account_name = app.account_name
+         where 1=1
+         <if test="userId!=null">
+             and login.user_id = #{userId}
+         </if>
+        <if test="startDate!=null">
+            and bill.date &gt;= #{startDate}
+        </if>
+        <if test="endDate!=null">
+           and  bill.date &lt;= #{endDate}
+        </if>
+        <if test="operateSystem!=null">
+           and bill.os = #{operateSystem}
+        </if>
+        group by bill.date,bill.os order by bill.date desc
+    </select>
+</mapper>

+ 36 - 4
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/xml/PangolinReportChannelDailyMapper.xml

@@ -6,7 +6,9 @@
             resultType="cn.com.ctop.crawler.modules.pangolin.entity.PangolinReportChannelDaily">
         select
             daily.date,
+            channel.name as 'channel_name',
             daily.app_name,
+            daily.oprate_system,
             sum(daily.external_activation) as 'external_activation',
             sum(daily.total_price)as 'total_price'
         from
@@ -14,11 +16,41 @@
         left join ctop_pangolin_channel channel on channel.name = daily.channel_name
         left join ctop_bind_account_login login on login.account_name = channel.account_name
         where 1=1
-        and login.user_id = #{userId}
-        and daily.date &gt;= #{startDate}
-        and daily.date &lt;= #{endDate}
-        and daily.os = #{type}
+        <if test="userId!=null">
+            and login.user_id = #{userId}
+        </if>
+        <if test="startDate!=null">
+            and daily.date &gt;= #{startDate}
+        </if>
+        <if test="endDate!=null">
+            and daily.date &lt;= #{endDate}
+        </if>
+        <if test="type!=null">
+            and daily.os = #{type}
+        </if>
         group by daily.date
     </select>
+    <select id="sumAppList" resultType="java.lang.Integer">
+        select
+            sum(daily.external_activation)
+        from
+        ctop_pangolin_report_channel_daily daily
+        left join ctop_pangolin_channel channel on channel.name = daily.channel_name
+        left join ctop_bind_account_login login on login.account_name = channel.account_name
+        where 1=1
+        <if test="userId!=null">
+            and login.user_id = #{userId}
+        </if>
+        <if test="startDate!=null">
+            and daily.date &gt;= #{startDate}
+        </if>
+        <if test="endDate!=null">
+            and daily.date &lt;= #{endDate}
+        </if>
+        <if test="type!=null">
+            and daily.os = #{type}
+        </if>
+
+    </select>
 
 </mapper>

+ 2 - 1
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/PangolinReportAppHourService.java

@@ -1,10 +1,11 @@
 package cn.com.ctop.crawler.modules.pangolin.service;
 
 import cn.com.ctop.crawler.modules.pangolin.entity.PangolinReportAppHour;
+import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.extension.service.IService;
 
 import java.util.Map;
 
 public interface PangolinReportAppHourService extends IService<PangolinReportAppHour> {
-    Map<String, Object> getRealTimeData(String userId);
+    Map<String, Object> getRealTimeData(String userId, JSONObject data);
 }

+ 3 - 3
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/PangolinReportBillMonthService.java

@@ -1,11 +1,11 @@
 package cn.com.ctop.crawler.modules.pangolin.service;
 
 import cn.com.ctop.crawler.modules.pangolin.entity.PangolinReportBillMonth;
+import cn.com.ctop.crawler.modules.pangolin.vo.PangolinReportAppMonthVO;
 import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.extension.service.IService;
-
-import java.util.Map;
+import com.github.pagehelper.PageInfo;
 
 public interface PangolinReportBillMonthService extends IService<PangolinReportBillMonth> {
-    Map<String, Object> checkAppList(String userId, JSONObject data);
+    PageInfo<PangolinReportAppMonthVO> checkAppList(String userId, JSONObject data);
 }

+ 11 - 2
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinCrawlerServiceImpl.java

@@ -79,6 +79,15 @@ public class PangolinCrawlerServiceImpl implements PangolinCrawlerService {
                     pangolinReportBillMonth.setSumInternalActivations(sumInternalActivations);
                     pangolinReportBillMonth.setYear(Integer.parseInt(date.split("-")[0]));
                     pangolinReportBillMonth.setMonth(Integer.parseInt(date.split("-")[1]));
+                    String getDate = date.split("-")[0];
+                    if(Integer.parseInt(date.split("-")[1])<10){
+                        getDate+= "-0";
+                        getDate+= date.split("-")[1];
+                    }else{
+                        getDate+= "-";
+                        getDate+= date.split("-")[1];
+                    }
+                    pangolinReportBillMonth.setDate(getDate);
                     QueryWrapper<PangolinReportBillMonth> queryWrapper = new QueryWrapper<>();
                     queryWrapper.eq("os", pangolinReportBillMonth.getOs());
                     queryWrapper.eq("app_id", pangolinReportBillMonth.getAppId());
@@ -204,7 +213,7 @@ public class PangolinCrawlerServiceImpl implements PangolinCrawlerService {
                     pangolinReportChannelDaily.setChannelName(channelName);
                     pangolinReportChannelDaily.setDate(DateUtils.parseDate(startDate, "yyyy-MM-dd"));
                     pangolinReportChannelDaily.setExternalActivation(externalActivation);
-                    pangolinReportChannelDaily.setOs(os);
+                    pangolinReportChannelDaily.setOprateSystem(os);
                     pangolinReportChannelDaily.setRetentionRatio(retentionRatio);
                     pangolinReportChannelDaily.setTotalPrice(totalPrice);
                     QueryWrapper<PangolinReportChannelDaily> queryWrapper = new QueryWrapper<>();
@@ -343,7 +352,7 @@ public class PangolinCrawlerServiceImpl implements PangolinCrawlerService {
 
     public boolean checkLogin(BindAccountLogin bindAccountLogin, String response) {
         if (response.contains("<html>")) {
-            bindAccountLogin.setCookie(null);
+            bindAccountLogin.setCookie("");
             bindAccountLoginService.updateById(bindAccountLogin);
             return false;
         }

+ 4 - 1
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinReportAppHourServiceImpl.java

@@ -6,6 +6,7 @@ import cn.com.ctop.crawler.modules.pangolin.entity.PangolinReportAppHour;
 import cn.com.ctop.crawler.modules.pangolin.mapper.PangolinReportAppHourMapper;
 import cn.com.ctop.crawler.modules.pangolin.service.PangolinReportAppHourService;
 import cn.com.ctop.crawler.modules.pangolin.vo.PangolinReportAppHourVO;
+import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import org.jeecg.common.util.DateUtils;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -21,9 +22,11 @@ public class PangolinReportAppHourServiceImpl extends ServiceImpl<PangolinReport
     private PangolinReportAppHourMapper pangolinReportAppHourMapper;
 
     @Override
-    public Map<String, Object> getRealTimeData(String userId) {
+    public Map<String, Object> getRealTimeData(String userId, JSONObject data) {
+
         Map<String, Object> resultMap = new HashMap<>();
         String today = DateUtils.formatDate();
+        String type = data.getString("type");
         //1:查询当天的数据
         List<PangolinReportAppHourVO> todayList = this.getRealTimeData(userId, today);
         resultMap.put("today", todayList);

+ 23 - 3
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinReportBillMonthServiceImpl.java

@@ -3,12 +3,15 @@ package cn.com.ctop.crawler.modules.pangolin.service.impl;
 import cn.com.ctop.crawler.modules.pangolin.entity.PangolinReportBillMonth;
 import cn.com.ctop.crawler.modules.pangolin.mapper.PangolinReportBillMonthMapper;
 import cn.com.ctop.crawler.modules.pangolin.service.PangolinReportBillMonthService;
+import cn.com.ctop.crawler.modules.pangolin.vo.PangolinReportAppMonthVO;
 import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.github.pagehelper.PageHelper;
+import com.github.pagehelper.PageInfo;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
-import java.util.Map;
+import java.util.List;
 
 @Service
 public class PangolinReportBillMonthServiceImpl extends ServiceImpl<PangolinReportBillMonthMapper, PangolinReportBillMonth> implements PangolinReportBillMonthService {
@@ -16,7 +19,24 @@ public class PangolinReportBillMonthServiceImpl extends ServiceImpl<PangolinRepo
     private PangolinReportBillMonthMapper pangolinReportBillMonthMapper;
 
     @Override
-    public Map<String, Object> checkAppList(String userId, JSONObject data) {
-        return null;
+    public PageInfo<PangolinReportAppMonthVO> checkAppList(String userId, JSONObject data) {
+        Integer pageNo = data.getInteger("pageNo");
+        if(null == pageNo||pageNo == 0){
+            pageNo = 1;
+        }
+        Integer pageSize = data.getInteger("pageSize");
+        if(null == pageSize||pageSize == 0){
+            pageSize = 10;
+        }
+        String startDate = data.getString("startDate");
+        String endDate = data.getString("endDate");
+        String operateSystem = data.getString("operateSystem");
+        if(null!=operateSystem&&operateSystem.trim().equals("all")){
+            operateSystem = null;
+        }
+        PageHelper.startPage(pageNo,pageSize);
+        List<PangolinReportAppMonthVO> list = pangolinReportBillMonthMapper.checkAppList(userId,startDate,endDate,operateSystem);
+        PageInfo<PangolinReportAppMonthVO> pageInfo = new PageInfo<>(list);
+        return pageInfo;
     }
 }

+ 34 - 3
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinReportChannelDailyServiceImpl.java

@@ -7,6 +7,9 @@ import cn.com.ctop.crawler.modules.pangolin.mapper.PangolinReportChannelDailyMap
 import cn.com.ctop.crawler.modules.pangolin.service.PangolinReportChannelDailyService;
 import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.github.pagehelper.PageHelper;
+import com.github.pagehelper.PageInfo;
+import org.jeecg.common.util.DateUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
@@ -22,11 +25,39 @@ public class PangolinReportChannelDailyServiceImpl extends ServiceImpl<PangolinR
     @Override
     public Map<String, Object> getActivationList(String userId, JSONObject data) {
         Map<String, Object> result = new HashMap<>();
+        Integer pageNo = data.getInteger("pageNo");
+        if(null == pageNo||pageNo == 0){
+            pageNo = 1;
+        }
+        Integer pageSize = data.getInteger("pageSize");
+        if(null == pageSize||pageSize == 0){
+            pageSize = 10;
+        }
+        String operateSystem = data.getString("operateSystem");
+        if(null!=operateSystem&&operateSystem.trim().equals("all")){
+            operateSystem = null;
+        }
         String startDate = data.getString("startDate");
         String endDate = data.getString("endDate");
-        String type = data.getString("type");
-        List<PangolinReportChannelDaily> dailyList = pangolinReportChannelDailyMapper.getActivationList(startDate, endDate, userId, type);
-        result.put("activationList", dailyList);
+        PageHelper.startPage(pageNo,pageSize);
+        List<PangolinReportChannelDaily> dailyList = pangolinReportChannelDailyMapper.getActivationList(startDate, endDate, userId, operateSystem);
+        PageInfo<PangolinReportChannelDaily> pageInfo = new PageInfo<>(dailyList);
+        result.put("data", pageInfo);
+        Integer totalActive = pangolinReportChannelDailyMapper.sumAppList(startDate,endDate,userId,operateSystem);
+        Long totalDays = 1L;
+        if(!startDate.equals(endDate)){
+            totalDays = DateUtils.getDiscrepantDays(startDate,endDate)+1;
+        }
+        if(null == totalActive){
+            result.put("totalActive","-");
+            result.put("avgActive","-");
+        }else if(0 == totalActive){
+            result.put("totalActive","0");
+            result.put("avgActive","0");
+        }else{
+            result.put("totalActive",totalActive+"");
+            result.put("avgActive",totalActive/totalDays+"");
+        }
         ResultMapUtils.setResultMap(result, StatusCode.COMMON_SUCCESS);
         return result;
     }

+ 16 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/vo/PangolinReportAppMonthVO.java

@@ -0,0 +1,16 @@
+package cn.com.ctop.crawler.modules.pangolin.vo;
+
+import lombok.Data;
+
+import java.math.BigDecimal;
+
+@Data
+public class PangolinReportAppMonthVO {
+    private String date;
+    private String appName;
+    private String operateSystem;
+    private Integer sumInternalActivations;
+    private BigDecimal estimatedTotalPrice;
+    private BigDecimal accountQualityScore;
+    private BigDecimal actualTotalPrice;
+}

+ 45 - 0
module-job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedanceFlowFundJob.java

@@ -0,0 +1,45 @@
+package cn.com.ctop.job.bytedance.handler;
+
+import cn.com.ctop.common.module.entity.CtopOauthToken;
+import cn.com.ctop.common.module.service.ICtopOauthTokenService;
+import cn.com.ctop.common.module.utils.CtopAdConstant;
+import cn.com.ctop.toutiao.modules.report.service.IBytedanceFundDailyService;
+import com.xxl.job.core.biz.model.ReturnT;
+import com.xxl.job.core.handler.annotation.XxlJob;
+import com.xxl.job.core.log.XxlJobLogger;
+import org.jeecg.common.util.DateUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+@Component
+public class BytedanceFlowFundJob {
+    @Autowired
+    private ICtopOauthTokenService tokenService;
+    @Autowired
+    private IBytedanceFundDailyService bytedanceFundDailyService;
+    static ExecutorService executorService = null;
+    @XxlJob("bytedanceFlowFund")
+    public ReturnT<String> execute(String param) throws Exception {
+        XxlJobLogger.log("头条账户流水数据获取开始");
+        Date getDate = new Date();
+        //1:查询当日数据
+        List<CtopOauthToken> tokens = tokenService.getTokenListByType(CtopAdConstant.PLATFORM_TYPE_BYTEDANCE);
+        if (null == tokens || tokens.size() <= 0) {
+            XxlJobLogger.log("头条账户流水数据异常:未获取到可用的token");
+            return ReturnT.FAIL;
+        }
+        executorService = Executors.newFixedThreadPool(4);
+        tokens.forEach(token -> {
+            executorService.submit(() -> {
+                bytedanceFundDailyService.loadFundDataByPage(token, DateUtils.formatDate(getDate), DateUtils.formatDate(getDate),1);
+            });
+        });
+        XxlJobLogger.log("头条账户流水数据获取结束");
+        return ReturnT.SUCCESS;
+    }
+}

+ 36 - 0
module-job-kuaishou/src/main/java/cn/com/ctop/job/kuaishou/handler/KuaiShouCleanMaterialJob.java

@@ -0,0 +1,36 @@
+package cn.com.ctop.job.kuaishou.handler;
+
+import cn.com.ctop.common.module.utils.Check;
+import cn.com.ctop.kuaishou.modules.report.service.IKuaiShouCleanMaterialReportService;
+import com.xxl.job.core.biz.model.ReturnT;
+import com.xxl.job.core.handler.annotation.XxlJob;
+import com.xxl.job.core.log.XxlJobLogger;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+@Component
+public class KuaiShouCleanMaterialJob {
+    @Autowired
+    private IKuaiShouCleanMaterialReportService kuaiShouCleanMaterialReportService;
+
+    static ExecutorService executorService = null;
+
+    @XxlJob("kuaishouCampaignJob")
+    public ReturnT<String> execute(String param) {
+        List<String> signatureList = kuaiShouCleanMaterialReportService.getSignature();
+        if (Check.isNull(signatureList)) {
+            XxlJobLogger.log("素材数据获取为空");
+            return ReturnT.FAIL;
+        }
+        executorService = Executors.newFixedThreadPool(5);
+        for (int i = 0; i < signatureList.size(); i++) {
+            String signature = signatureList.get(i);
+            executorService.submit(()-> kuaiShouCleanMaterialReportService.calculationReport(signature));
+        }
+        return ReturnT.SUCCESS;
+    }
+}

+ 7 - 0
pom.xml

@@ -180,6 +180,13 @@
             <version>1.2.56</version>
         </dependency>
 
+        <dependency>
+            <groupId>com.github.pagehelper</groupId>
+            <artifactId>pagehelper-spring-boot-starter</artifactId>
+            <!-- 特别注意版本问题 -->
+            <version>1.2.3</version>
+        </dependency>
+
         <!--mysql-->
         <dependency>
             <groupId>mysql</groupId>