Procházet zdrojové kódy

修改穿山甲代码逻辑

syh před 5 roky
rodič
revize
ca852aebdd
30 změnil soubory, kde provedl 541 přidání a 124 odebrání
  1. 27 47
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/PangolinController.java
  2. 0 3
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/ProjectMemberController.java
  3. 2 10
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/service/impl/CrawlerServiceImpl.java
  4. 5 2
      jeecg-boot-module-system/src/main/resources/application-prod.yml
  5. 35 1
      jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java
  6. 11 0
      module-common/src/main/java/cn/com/ctop/common/module/mapper/BindAccountLoginMapper.java
  7. 34 0
      module-common/src/main/java/cn/com/ctop/common/module/mapper/xml/BindAccountLoginMapper.xml
  8. 4 0
      module-common/src/main/java/cn/com/ctop/common/module/service/IBindAccountLoginService.java
  9. 20 1
      module-common/src/main/java/cn/com/ctop/common/module/service/impl/BindAccountLoginServiceImpl.java
  10. 22 0
      module-common/src/main/java/cn/com/ctop/common/module/vo/PangolinAccountVO.java
  11. 37 0
      module-common/src/main/java/cn/com/ctop/common/module/vo/PangolinProjectVO.java
  12. 1 1
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/vo/PangolinReportAppHourVO.java
  13. 1 1
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/vo/PangolinReportAppMonthVO.java
  14. 13 11
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/core/util/FfmpegUtils.java
  15. 11 3
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/service/impl/EffectCaseServiceImpl.java
  16. 35 18
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/service/impl/OceanEngineServiceImpl.java
  17. 3 1
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/PangolinAppMapper.java
  18. 13 1
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/PangolinReportAppHourMapper.java
  19. 8 3
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/PangolinReportBillMonthMapper.java
  20. 14 2
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/PangolinReportChannelDailyMapper.java
  21. 18 5
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/xml/PangolinAppMapper.xml
  22. 80 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/xml/PangolinReportAppHourMapper.xml
  23. 18 2
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/xml/PangolinReportBillMonthMapper.xml
  24. 32 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/xml/PangolinReportChannelDailyMapper.xml
  25. 2 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/PangolinAppService.java
  26. 1 1
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/PangolinReportBillMonthService.java
  27. 34 4
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinAppServiceImpl.java
  28. 20 3
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinReportAppHourServiceImpl.java
  29. 20 2
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinReportBillMonthServiceImpl.java
  30. 20 2
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinReportChannelDailyServiceImpl.java

+ 27 - 47
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/PangolinController.java

@@ -2,11 +2,10 @@ package org.jeecg.modules.ctop.controller;
 
 import cn.com.ctop.common.module.service.IBindAccountLoginService;
 import cn.com.ctop.common.module.service.ISysRoleService;
-import cn.com.ctop.common.module.service.IUserAllocationService;
 import cn.com.ctop.common.module.utils.ResultMapUtils;
 import cn.com.ctop.common.module.utils.StatusCode;
+import cn.com.ctop.common.module.vo.PangolinReportAppMonthVO;
 import cn.com.ctop.crawler.modules.pangolin.service.*;
-import cn.com.ctop.crawler.modules.pangolin.vo.PangolinReportAppMonthVO;
 import com.alibaba.fastjson.JSONObject;
 import com.github.pagehelper.PageInfo;
 import org.apache.shiro.SecurityUtils;
@@ -38,33 +37,6 @@ public class PangolinController {
     @Autowired
     private IBindAccountLoginService bindAccountLoginService;
 
-    @Autowired
-    private IUserAllocationService userAllocationService;
-    @PostMapping("api/user/list")
-    public Map<String, Object> userList(@RequestBody JSONObject data) {
-//        LoginUser user = (LoginUser) SecurityUtils.getSubject().getPrincipal();
-//        String userId = user.getId();
-        String userId = "6b4c821adc414dc8b7718ab63ccbfcaf";
-        String roleCode = sysRoleService.getRoleCodeByUserId(userId);
-        if(null!=roleCode&&(roleCode.equals("admin"))){
-            userId = null;
-        }
-
-        return userAllocationService.getPangolinAccountListByParams(userId,data);
-    }
-
-    @PostMapping("api/project/list")
-    public Map<String, Object> projectList(@RequestBody JSONObject data) {
-//        LoginUser user = (LoginUser) SecurityUtils.getSubject().getPrincipal();
-//        String userId = user.getId();
-        String userId = "6b4c821adc414dc8b7718ab63ccbfcaf";
-        String roleCode = sysRoleService.getRoleCodeByUserId(userId);
-        if(null!=roleCode&&(roleCode.equals("admin"))){
-            userId = null;
-        }
-        return userAllocationService.getPangolinProjectListByParams(userId);
-    }
-
     @PostMapping("api/channel/list")
     public Map<String, Object> channelList(@RequestBody JSONObject data) {
         LoginUser user = (LoginUser) SecurityUtils.getSubject().getPrincipal();
@@ -77,23 +49,11 @@ public class PangolinController {
         return pangolinChannelService.getListByUserId(userId);
     }
 
-//    @PostMapping("api/user/list")
-//    public Map<String, Object> userList() {
-//        LoginUser user = (LoginUser) SecurityUtils.getSubject().getPrincipal();
-//        String userId = user.getId();
-////        String userId = "6b4c821adc414dc8b7718ab63ccbfcaf";
-//        String roleCode = sysRoleService.getRoleCodeByUserId(userId);
-//        if(null!=roleCode&&(roleCode.equals("admin"))){
-//            userId = null;
-//        }
-//        return bindAccountLoginService.getListByUserId(userId);
-//    }
-
     @PostMapping("api/app/list")
     public Map<String, Object> appList(@RequestBody JSONObject data) {
         LoginUser user = (LoginUser) SecurityUtils.getSubject().getPrincipal();
-        String userId = user.getId();
-//        String userId = "6b4c821adc414dc8b7718ab63ccbfcaf";
+//        String userId = user.getId();
+        String userId = "6b4c821adc414dc8b7718ab63ccbfcaf";
         String roleCode = sysRoleService.getRoleCodeByUserId(userId);
         if(null!=roleCode&&(roleCode.equals("admin"))){
             userId = null;
@@ -102,14 +62,34 @@ public class PangolinController {
     }
 
 
+    @PostMapping("api/memberList")
+    public Map<String,Object> memberList(){
+        Map<String, Object> resultMap = new HashMap<>();
+        try {
+//            LoginUser user = (LoginUser) SecurityUtils.getSubject().getPrincipal();
+//            String userId = user.getId();
+            String userId = "6b4c821adc414dc8b7718ab63ccbfcaf";
+            String roleCode = sysRoleService.getRoleCodeByUserId(userId);
+            if (null!=roleCode&&"admin".equals(roleCode)) {
+                userId = null;
+            }
+            resultMap = pangolinAppService.getMemberList(userId);
+            return resultMap;
+        } catch (Exception e) {
+            ResultMapUtils.setResultMap(resultMap,StatusCode.COMMON_SERVER_ERROR);
+            return resultMap;
+        }
+    }
+
+
     /**
      * 实时数据
      */
     @PostMapping("api/realtime")
     public Map<String, Object> realTime(@RequestBody JSONObject data) {
         LoginUser user = (LoginUser) SecurityUtils.getSubject().getPrincipal();
-        String userId = user.getId();
-//        String userId = "6b4c821adc414dc8b7718ab63ccbfcaf";
+//        String userId = user.getId();
+        String userId = "6b4c821adc414dc8b7718ab63ccbfcaf";
         String roleCode = sysRoleService.getRoleCodeByUserId(userId);
         if(null!=roleCode&&(roleCode.equals("admin"))){
             userId = null;
@@ -139,8 +119,8 @@ public class PangolinController {
     public Map<String,Object> checkAppList(@RequestBody JSONObject data) {
         Map<String,Object> result = new HashMap<>();
         LoginUser user = (LoginUser) SecurityUtils.getSubject().getPrincipal();
-        String userId = user.getId();
-//        String userId = "6b4c821adc414dc8b7718ab63ccbfcaf";
+//        String userId = user.getId();
+        String userId = "6b4c821adc414dc8b7718ab63ccbfcaf";
         String roleCode = sysRoleService.getRoleCodeByUserId(userId);
         if(null!=roleCode&&(roleCode.equals("admin"))){
             userId = null;

+ 0 - 3
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/ProjectMemberController.java

@@ -10,11 +10,9 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiOperation;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.shiro.SecurityUtils;
 import org.jeecg.common.api.vo.Result;
 import org.jeecg.common.aspect.annotation.AutoLog;
 import org.jeecg.common.system.query.QueryGenerator;
-import org.jeecg.common.system.vo.LoginUser;
 import org.jeecg.modules.ctop.entity.Project;
 import org.jeecg.modules.ctop.entity.ProjectMember;
 import org.jeecg.modules.ctop.mapper.ProjectMemberMapper;
@@ -79,7 +77,6 @@ public class ProjectMemberController {
         return map;
     }
 
-
     /**
      * 我参与项目
      *

+ 2 - 10
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/service/impl/CrawlerServiceImpl.java

@@ -32,13 +32,10 @@ public class CrawlerServiceImpl implements ICrawlerService {
     public Map<String, Object> loadInspiration() {
         //(1):效果广告解析
         loadeffectAdByPage(100, 1);
-
         //(2):广告创意视频
         loadInspiredAdByPage(100, 1);
-
         //(3):快手热门作品
         loadHotPhotoByPage(100, 1);
-
         return null;
     }
 
@@ -55,7 +52,7 @@ public class CrawlerServiceImpl implements ICrawlerService {
             return;
         }
         JSONArray hotPhotoInfos = resultObject.getJSONObject("data").getJSONArray("hotPhotoInfos");
-        if (null == hotPhotoInfos || hotPhotoInfos.size() <= 0) {
+        if (null == hotPhotoInfos || hotPhotoInfos.isEmpty()) {
             log.info("热门作品数据获取完成");
             return;
         }
@@ -120,11 +117,10 @@ public class CrawlerServiceImpl implements ICrawlerService {
             return;
         }
         JSONArray effectAds = resultObject.getJSONObject("data").getJSONArray("effectAds");
-        if (null == effectAds || effectAds.size() <= 0) {
+        if (null == effectAds || effectAds.isEmpty()) {
             log.info("效果广告数据获取完成");
             return;
         }
-//        List<KuaishouEffectAdInfo> adInfos = new ArrayList<>();
         for (int i = 0; i < effectAds.size(); i++) {
             JSONObject ad = effectAds.getJSONObject(i);
             Long contentId = ad.getLong("contentId");
@@ -132,12 +128,8 @@ public class CrawlerServiceImpl implements ICrawlerService {
                 JSONObject content = getEffectAdDetail(contentId).getJSONObject("data").getJSONObject("effectAdDetail");
                 KuaishouEffectAdInfo adInfo = new KuaishouEffectAdInfo(content);
                 effectAdInfoService.saveOrUpdate(adInfo);
-                //                adInfos.add(adInfo);
             }
         }
-        /*if (adInfos.size() > 0) {
-            effectAdInfoService.replaceBatch(adInfos);
-        }*/
         loadeffectAdByPage(pageSize, pageNum + 1);
     }
 

+ 5 - 2
jeecg-boot-module-system/src/main/resources/application-prod.yml

@@ -33,8 +33,8 @@ spring:
             enable: true
             required: true
   ## quartz定时任务,采用数据库方式
-  quartz:
-    job-store-type: jdbc
+#  quartz:
+#    job-store-type: jdbc
   #json 时间戳统一转换
   jackson:
     date-format:   yyyy-MM-dd HH:mm:ss
@@ -136,6 +136,8 @@ mybatis-plus:
       id-type: 4
       # 默认数据库表下划线命名
       table-underline: true
+  configuration:
+    log-impl: org.apache.ibatis.logging.stdout.StdOutImpl
 #jeecg专用配置
 jeecg:
   ffmpeg:
@@ -148,6 +150,7 @@ jeecg:
     video-upload: /mnt/upload/video/
     image-upload: /mnt/upload/image/
     chrome-driver: /usr/bin/chromedriver
+#    chrome-driver: D://chromedriver.exe
     csv-upload: /mnt/upload/csv/
     bak-database-file: /mnt/data/bak/
     report-history: /mnt/upload/report/history/

+ 35 - 1
jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java

@@ -1,13 +1,18 @@
 package org.jeecg;
 
 import cn.com.ctop.common.module.entity.BindAccountLogin;
+import cn.com.ctop.common.module.entity.CtopOauthToken;
 import cn.com.ctop.common.module.service.IBindAccountLoginService;
+import cn.com.ctop.common.module.service.ICtopOauthTokenService;
 import cn.com.ctop.common.module.utils.CtopAdConstant;
 import cn.com.ctop.crawler.modules.oceanengine.entity.HotMaterial;
 import cn.com.ctop.crawler.modules.oceanengine.service.IHotMaterialService;
 import cn.com.ctop.crawler.modules.oceanengine.service.IOceanEngineService;
 import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouWebInterfaceService;
+import cn.com.ctop.toutiao.modules.report.service.IBytedanceFundDailyService;
+import com.xxl.job.core.log.XxlJobLogger;
 import lombok.extern.slf4j.Slf4j;
+import org.jeecg.common.util.DateUtils;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -105,12 +110,41 @@ public class SampleTest {
         Long end = System.currentTimeMillis();
         log.info("总用时:{}毫秒",end-start);
     }
+    @Autowired
+    private ICtopOauthTokenService tokenService;
+    @Autowired
+    private IBytedanceFundDailyService bytedanceFundDailyService;
     @Test
     public void testBase64(){
+        List<CtopOauthToken> tokens = tokenService.getTokenListByType(CtopAdConstant.PLATFORM_TYPE_BYTEDANCE);
+        if (null == tokens || tokens.size() <= 0) {
+            XxlJobLogger.log("头条账户流水数据异常:未获取到可用的token");
+            return ;
+        }
+        executorService = Executors.newFixedThreadPool(4);
+        countDownLatch = new CountDownLatch(tokens.size());
+        tokens.forEach(token -> {
+            executorService.submit(() -> {
+                try {
+                    for(int i=0;i<12;i++){
+                        String getDate = DateUtils.addDay("2020-05-28",-i);
+                        bytedanceFundDailyService.loadFundDataByPage(token, getDate, getDate,1);
+                    }
+                }catch (Exception e){
+
+                }finally {
+                    countDownLatch.countDown();
+                }
 
+            });
+        });
+        try {
+            countDownLatch.await();
+        } catch (InterruptedException e) {
+        }
+        System.out.println("执行完成");
     }
 }
-//刷新token:885936毫秒
 
 
 

+ 11 - 0
module-common/src/main/java/cn/com/ctop/common/module/mapper/BindAccountLoginMapper.java

@@ -1,7 +1,12 @@
 package cn.com.ctop.common.module.mapper;
 
 import cn.com.ctop.common.module.entity.BindAccountLogin;
+import cn.com.ctop.common.module.vo.PangolinAccountVO;
+import cn.com.ctop.common.module.vo.PangolinProjectVO;
 import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import org.apache.ibatis.annotations.Param;
+
+import java.util.List;
 
 /**
  * @Description: 账号绑定-登陆
@@ -11,4 +16,10 @@ import com.baomidou.mybatisplus.core.mapper.BaseMapper;
  */
 public interface BindAccountLoginMapper extends BaseMapper<BindAccountLogin> {
 
+    List<PangolinProjectVO> getProjectByParams(@Param(value = "userId") String userId,
+                                               @Param(value = "loginType") String loginType,
+                                               @Param(value = "projectId") Long projectId);
+    List<PangolinAccountVO> getAccountByParams(@Param(value = "userId") String userId,
+                                              @Param(value = "loginType") String loginType,
+                                              @Param(value = "projectId") Long projectId);
 }

+ 34 - 0
module-common/src/main/java/cn/com/ctop/common/module/mapper/xml/BindAccountLoginMapper.xml

@@ -2,4 +2,38 @@
 <!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
 <mapper namespace="cn.com.ctop.common.module.mapper.BindAccountLoginMapper">
 
+    <select id="getProjectByParams" resultType="cn.com.ctop.common.module.vo.PangolinProjectVO">
+        select
+            project.project_name,
+            project.id as 'project_id'
+        from
+            ctop_bind_account_login login
+            left join ctop_project project on login.project_id = project.id
+        where 1=1
+        <if test="userId!=null">
+            and user_id = #{userId}
+        </if>
+        <if test="loginType!=null">
+            and login_type = #{loginType}
+        </if>
+        group by project_id
+    </select>
+    <select id="getAccountByParams" resultType="cn.com.ctop.common.module.vo.PangolinAccountVO">
+        select
+        login.account_id,
+        login.account_name
+        from
+        ctop_bind_account_login login
+        left join ctop_project project on login.project_id = project.id
+        where 1=1
+        <if test="userId!=null">
+            and login.user_id = #{userId}
+        </if>
+        <if test="loginType!=null">
+            and login.login_type = #{loginType}
+        </if>
+        <if test="projectId!=null">
+            and project.id = #{projectId}
+        </if>
+    </select>
 </mapper>

+ 4 - 0
module-common/src/main/java/cn/com/ctop/common/module/service/IBindAccountLoginService.java

@@ -1,6 +1,7 @@
 package cn.com.ctop.common.module.service;
 
 import cn.com.ctop.common.module.entity.BindAccountLogin;
+import cn.com.ctop.common.module.vo.PangolinProjectVO;
 import com.baomidou.mybatisplus.extension.service.IService;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import org.apache.http.client.CookieStore;
@@ -32,4 +33,7 @@ public interface IBindAccountLoginService extends IService<BindAccountLogin> {
     CookieStore setCookie(BindAccountLogin bindAccountLogin,String domain) throws JsonProcessingException;
 
     BindAccountLogin getByParams(String accountName, String password, String advertiserId);
+
+    List<PangolinProjectVO> getProjectByParams(String userId, String loginType);
+
 }

+ 20 - 1
module-common/src/main/java/cn/com/ctop/common/module/service/impl/BindAccountLoginServiceImpl.java

@@ -6,6 +6,8 @@ import cn.com.ctop.common.module.service.IBindAccountLoginService;
 import cn.com.ctop.common.module.utils.CtopAdConstant;
 import cn.com.ctop.common.module.utils.ResultMapUtils;
 import cn.com.ctop.common.module.utils.StatusCode;
+import cn.com.ctop.common.module.vo.PangolinAccountVO;
+import cn.com.ctop.common.module.vo.PangolinProjectVO;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.fasterxml.jackson.core.JsonProcessingException;
@@ -15,7 +17,7 @@ import lombok.extern.slf4j.Slf4j;
 import org.apache.http.client.CookieStore;
 import org.apache.http.impl.client.BasicCookieStore;
 import org.apache.http.impl.cookie.BasicClientCookie;
-import org.springframework.context.annotation.Primary;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
 import java.util.ArrayList;
@@ -65,6 +67,23 @@ public class BindAccountLoginServiceImpl extends ServiceImpl<BindAccountLoginMap
     }
 
     @Override
+    public List<PangolinProjectVO> getProjectByParams(String userId, String loginType) {
+        List<PangolinProjectVO> projectVOS =  bindAccountLoginMapper.getProjectByParams(userId,loginType,null);
+        List<PangolinProjectVO> setProjectVOS = new ArrayList<>();
+        if(null!=projectVOS&&!projectVOS.isEmpty()){
+            projectVOS.forEach(pangolinProjectVO -> {
+                List<PangolinAccountVO>accountVOS = bindAccountLoginMapper.getAccountByParams(userId,loginType,pangolinProjectVO.getProjectId());
+                pangolinProjectVO.setAccountVOList(accountVOS);
+                setProjectVOS.add(pangolinProjectVO);
+            });
+        }
+        return setProjectVOS;
+    }
+
+    @Autowired
+    private BindAccountLoginMapper bindAccountLoginMapper;
+
+    @Override
     public Map<String, Object> getListByUserId(String userId) {
         Map<String,Object>result = new HashMap<>();
         QueryWrapper<BindAccountLogin> queryWrapper = new QueryWrapper<>();

+ 22 - 0
module-common/src/main/java/cn/com/ctop/common/module/vo/PangolinAccountVO.java

@@ -0,0 +1,22 @@
+package cn.com.ctop.common.module.vo;
+
+public class PangolinAccountVO {
+    private Long accountId;
+    private String accountName;
+
+    public Long getAccountId() {
+        return accountId;
+    }
+
+    public void setAccountId(Long accountId) {
+        this.accountId = accountId;
+    }
+
+    public String getAccountName() {
+        return accountName;
+    }
+
+    public void setAccountName(String accountName) {
+        this.accountName = accountName;
+    }
+}

+ 37 - 0
module-common/src/main/java/cn/com/ctop/common/module/vo/PangolinProjectVO.java

@@ -0,0 +1,37 @@
+package cn.com.ctop.common.module.vo;
+
+import java.util.List;
+
+/**
+ * 2020年5月29日13:14:49
+ * @author 宋英豪
+ */
+public class PangolinProjectVO {
+    private String projectName;
+    private Long projectId;
+    private List<PangolinAccountVO> accountList;
+
+    public String getProjectName() {
+        return projectName;
+    }
+
+    public void setProjectName(String projectName) {
+        this.projectName = projectName;
+    }
+
+    public Long getProjectId() {
+        return projectId;
+    }
+
+    public void setProjectId(Long projectId) {
+        this.projectId = projectId;
+    }
+
+    public List<PangolinAccountVO> getAccountList() {
+        return accountList;
+    }
+
+    public void setAccountVOList(List<PangolinAccountVO> accountList) {
+        this.accountList = accountList;
+    }
+}

+ 1 - 1
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/vo/PangolinReportAppHourVO.java

@@ -1,4 +1,4 @@
-package cn.com.ctop.crawler.modules.pangolin.vo;
+package cn.com.ctop.common.module.vo;
 
 import lombok.Data;
 

+ 1 - 1
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/vo/PangolinReportAppMonthVO.java

@@ -1,4 +1,4 @@
-package cn.com.ctop.crawler.modules.pangolin.vo;
+package cn.com.ctop.common.module.vo;
 
 import lombok.Data;
 

+ 13 - 11
module-crawler/src/main/java/cn/com/ctop/crawler/modules/core/util/FfmpegUtils.java

@@ -11,7 +11,7 @@ import java.io.ByteArrayOutputStream;
 import java.util.Base64;
 
 public class FfmpegUtils {
-    public static String getVideoFirstImage(String url){
+    public static String getVideoFirstImage(String url,int frame){
         try {
             FFmpegFrameGrabber ff = new FFmpegFrameGrabber(url);
             ff.start();
@@ -19,33 +19,35 @@ public class FfmpegUtils {
             int i = 0;
             Frame f = null;
             while (i < lenght) {
-// 过滤前5帧,避免出现全黑的图片,依自己情况而定
+                // 过滤帧,避免出现全黑的图片,依自己情况而定
                 f = ff.grabFrame();
-                if ((i > 5) && (f.image != null)) {
+                if ((i > frame) && (f.image != null)) {
                     break;
                 }
                 i++;
             }
-//            opencv_core.IplImage img = f.image;
             int owidth = f.imageWidth;
             int oheight = f.imageHeight;
             Java2DFrameConverter converter =new Java2DFrameConverter();
             BufferedImage fecthedImage =converter.getBufferedImage(f);
-// 对截取的帧进行等比例缩放
+            // 对截取的帧进行等比例缩放
             int width = 800;
             int height = (int) (((double) width / owidth) * oheight);
             BufferedImage bi = new BufferedImage(width, height, BufferedImage.TYPE_3BYTE_BGR);
             bi.getGraphics().drawImage(fecthedImage.getScaledInstance(width, height, Image.SCALE_SMOOTH),
                     0, 0, null);
-            ByteArrayOutputStream baos = new ByteArrayOutputStream();//io流
+            //io流
+            ByteArrayOutputStream baos = new ByteArrayOutputStream();
             ImageIO.write(bi, "jpg", baos);
-            byte[] bytes = baos.toByteArray();//转换成字节
-            String png_base64 = Base64.getEncoder().encodeToString(bytes).trim();//转换成base64串
-            png_base64 = png_base64.replaceAll("\n", "").replaceAll("\r", "");//删除 \r\n
-            //        ImageIO.write(bufferedImage, "png", new File("D:/qrcode1.png"));
+            //转换成字节
+            byte[] bytes = baos.toByteArray();
+            //转换成base64串
+            String pngBase64 = Base64.getEncoder().encodeToString(bytes).trim();
+            //删除 \r\n
+            pngBase64 = pngBase64.replaceAll("\n", "").replaceAll("\r", "");
             baos.close();
             ff.close();
-            return "data:image/jpg;base64,"+png_base64;
+            return pngBase64;
         }catch (Exception e){
             e.printStackTrace();
             return null;

+ 11 - 3
module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/service/impl/EffectCaseServiceImpl.java

@@ -38,15 +38,23 @@ public class EffectCaseServiceImpl extends ServiceImpl<EffectCaseMapper, EffectC
     public IPage<EffectCase> queryHotFeeds(String userName, String start, String end, String orderByColumn, String orderByType, Integer pageNo, Integer pageSize) {
         QueryWrapper<EffectCase> queryWrapper = new QueryWrapper<>();
         if (null != userName && !"".equals(userName)) {
-            queryWrapper.like("author", "%" + userName + "%");
+            queryWrapper.like("author", userName);
         }
         if (null != start && !"".equals(start) && null != end && !"".equals(end)) {
             queryWrapper.between("create_time", start, end);
         }
         if (null != orderByType && "desc".equals(orderByType)) {
-            queryWrapper.orderByDesc(orderByColumn);
+            if(orderByColumn.equals("create_time")){
+                queryWrapper.orderByAsc(orderByColumn);
+            }else{
+                queryWrapper.orderByDesc(orderByColumn);
+            }
         } else {
-            queryWrapper.orderByAsc(orderByColumn);
+            if(orderByColumn.equals("create_time")){
+                queryWrapper.orderByDesc(orderByColumn);
+            }else{
+                queryWrapper.orderByAsc(orderByColumn);
+            }
         }
         Page<EffectCase> page = new Page<>(pageNo, pageSize);
         return this.page(page, queryWrapper);

+ 35 - 18
module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/service/impl/OceanEngineServiceImpl.java

@@ -2,8 +2,10 @@ package cn.com.ctop.crawler.modules.oceanengine.service.impl;
 
 import cn.com.ctop.common.module.utils.Base64Utils;
 import cn.com.ctop.common.module.utils.HttpUtils2;
+import cn.com.ctop.common.module.utils.OSSUtils;
 import cn.com.ctop.crawler.modules.core.util.FateadmHttpUtil;
 import cn.com.ctop.crawler.modules.core.util.FateadmUtil;
+import cn.com.ctop.crawler.modules.core.util.FfmpegUtils;
 import cn.com.ctop.crawler.modules.oceanengine.entity.DouyinHot;
 import cn.com.ctop.crawler.modules.oceanengine.entity.EffectCase;
 import cn.com.ctop.crawler.modules.oceanengine.entity.HotMaterial;
@@ -27,6 +29,8 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
+import java.io.FileInputStream;
+import java.io.InputStream;
 import java.net.HttpURLConnection;
 import java.net.URL;
 import java.util.*;
@@ -63,7 +67,6 @@ public class OceanEngineServiceImpl implements IOceanEngineService {
     @Override
     public void hotMaterialHandler(int page, int appCode, String appName){
         String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_material/list?material_type=3&order_by=click_show_rate&period_type=30&aggr_app_code="+appCode+"&page="+page+"&limit=20");
-        System.out.println(result);
         ObjectMapper mapper = new ObjectMapper();
         mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
         try {
@@ -81,12 +84,18 @@ public class OceanEngineServiceImpl implements IOceanEngineService {
                             if(material.getVideoUrl() == null){
                                 material.setVideoUrl(getVideoUrl(material.getVid()));
                             }
-//                            material.setVideoUrl(getVideoUrl(material.getVid()));
-//                            material.setCoverImage(FfmpegUtils.getVideoFirstImage(material.getVideoUrl()));
-//                            if(material.getCoverImage() == null){
-//                                Thread.sleep(1000);
-//                                material.setCoverImage(FfmpegUtils.getVideoFirstImage(material.getVideoUrl()));
-//                            }
+
+                            String imageCode = FfmpegUtils.getVideoFirstImage(material.getVideoUrl(),20);
+                            if(null == imageCode){
+                                Thread.sleep(1000L);
+                                imageCode = FfmpegUtils.getVideoFirstImage(material.getVideoUrl(),20);
+                            }
+                            String destPath = "D:\\data\\effectCase\\hot-material\\";
+                            String fileName = material.getVid()+".jpg";
+                            EffectCase.base64ToFile(destPath,imageCode,fileName);
+                            InputStream inputStream = new FileInputStream(destPath+fileName);
+                            String finalUrl = OSSUtils.uploadFile2OSS(inputStream, "ocean-engine/hot-material/", fileName);
+                            material.setCoverUrl(finalUrl);
                             hotMaterialService.saveOrUpdate(material);
                         }
                         hotMaterialHandler(page+1,appCode,appName);
@@ -149,11 +158,17 @@ public class OceanEngineServiceImpl implements IOceanEngineService {
                     if (effectCase.getVideoUrl() == null){
                         effectCase.setVideoUrl(getVideoUrl(effectCase.getVid()));
                     }
-//                    effectCase.setCoverImage(FfmpegUtils.getVideoFirstImage(effectCase.getVideoUrl()));
-//                    if (effectCase.getCoverImage() == null){
-//                        Thread.sleep(1000);
-//                        effectCase.setCoverImage(FfmpegUtils.getVideoFirstImage(effectCase.getVideoUrl()));
-//                    }
+                    String imageCode = FfmpegUtils.getVideoFirstImage(effectCase.getVideoUrl(),20);
+                    if(null == imageCode){
+                        Thread.sleep(1000L);
+                        imageCode = FfmpegUtils.getVideoFirstImage(effectCase.getVideoUrl(),20);
+                    }
+                    String destPath = "D:\\data\\effectCase\\effect-case\\";
+                    String fileName = effectCase.getVid()+".jpg";
+                    EffectCase.base64ToFile(destPath,imageCode,fileName);
+                    InputStream inputStream = new FileInputStream(destPath+fileName);
+                    String finalUrl = OSSUtils.uploadFile2OSS(inputStream, "ocean-engine/effect-case/", fileName);
+                    effectCase.setCoverUrl(finalUrl);
                     effectCase.setCreativeLabel(getCase.getString("creative_label"));
                 }
                 effectCaseService.saveOrUpdate(effectCase);
@@ -180,17 +195,19 @@ public class OceanEngineServiceImpl implements IOceanEngineService {
                 if(node.get("data")!= null){
                     String dataJson = node.get("data").get("items").toString();
                     List<DouyinHot> list = mapper.readValue(dataJson,new TypeReference<List<DouyinHot>>() {});
-                    if (list != null && list.size() > 0){
+                    if (list != null && !list.isEmpty()){
                         for (DouyinHot douyinHot : list){
                             douyinHot.setUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+douyinHot.getVideoId()+"&line=0"));
                             if(douyinHot.getUrl() == null){
                                 douyinHot.setUrl(getVideoUrl(douyinHot.getVideoId()));
                             }
-//                            douyinHot.setCoverImage(FfmpegUtils.getVideoFirstImage(douyinHot.getUrl()));
-//                            if(douyinHot.getCoverImage() == null){
-//                                Thread.sleep(1000);
-//                                douyinHot.setCoverImage(FfmpegUtils.getVideoFirstImage(douyinHot.getUrl()));
-//                            }
+                            String imageCode = FfmpegUtils.getVideoFirstImage(douyinHot.getUrl(),20);
+                            String destPath = "D:\\data\\effectCase\\douyin-hot\\";
+                            String fileName = douyinHot.getId()+".jpg";
+                            EffectCase.base64ToFile(destPath,imageCode,fileName);
+                            InputStream inputStream = new FileInputStream(destPath+fileName);
+                            String finalUrl = OSSUtils.uploadFile2OSS(inputStream, "ocean-engine/douyin-hot/", fileName);
+                            douyinHot.setCoverUrl(finalUrl);
                             douyinHotService.saveOrUpdate(douyinHot);
                         }
                         douyinHotHandler(page+1,listType);

+ 3 - 1
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/PangolinAppMapper.java

@@ -7,5 +7,7 @@ import org.apache.ibatis.annotations.Param;
 import java.util.List;
 
 public interface PangolinAppMapper extends BaseMapper<PangolinApp> {
-    List<PangolinApp> getListByUserId(@Param(value = "userId") String userId,@Param(value = "projectId") Long projectID);
+    List<PangolinApp> getListByParams(@Param(value = "userId") String userId,
+                                      @Param(value = "projectIds") List<Long> projectIds,
+                                      @Param(value = "accountNames") List<String> accountNames);
 }

+ 13 - 1
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/PangolinReportAppHourMapper.java

@@ -5,6 +5,18 @@ import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.core.mapper.BaseMapper;
 import org.apache.ibatis.annotations.Param;
 
+import java.util.List;
+
 public interface PangolinReportAppHourMapper extends BaseMapper<PangolinReportAppHour> {
-    JSONObject getComparedDate(@Param(value = "appId") Integer appId,@Param(value = "hour") Integer hour, @Param(value = "userId")String userId, @Param(value = "type")String type, @Param(value = "today")String today, @Param(value = "yesterday")String yesterday, @Param(value = "sevenDaysAgo")String sevenDaysAgo, @Param(value = "compareTime1")String compareTime1, @Param(value = "compareTime2")String compareTime2);
+    JSONObject getComparedDate(@Param(value = "appId") Integer appId,
+                               @Param(value = "hour") Integer hour,
+                               @Param(value = "userId")String userId,
+                               @Param(value = "type")String type,
+                               @Param(value = "today")String today,
+                               @Param(value = "yesterday")String yesterday,
+                               @Param(value = "sevenDaysAgo")String sevenDaysAgo,
+                               @Param(value = "compareTime1")String compareTime1,
+                               @Param(value = "compareTime2")String compareTime2,
+                               @Param(value = "projectIds")List<Long> projectIds,
+                               @Param(value = "accountNames")List<String> accountNames);
 }

+ 8 - 3
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/PangolinReportBillMonthMapper.java

@@ -1,13 +1,18 @@
 package cn.com.ctop.crawler.modules.pangolin.mapper;
 
+import cn.com.ctop.common.module.vo.PangolinReportAppMonthVO;
 import cn.com.ctop.crawler.modules.pangolin.entity.PangolinReportBillMonth;
-import cn.com.ctop.crawler.modules.pangolin.vo.PangolinReportAppMonthVO;
 import com.baomidou.mybatisplus.core.mapper.BaseMapper;
 import org.apache.ibatis.annotations.Param;
 
 import java.util.List;
 
 public interface PangolinReportBillMonthMapper extends BaseMapper<PangolinReportBillMonth> {
-    List<PangolinReportAppMonthVO> checkAppList(@Param(value = "appId") Integer appId, @Param(value = "userId") String userId, @Param(value = "startDate") String startDate,
-                                                @Param(value = "endDate") String endDate, @Param(value = "operateSystem") String operateSystem);
+    List<PangolinReportAppMonthVO> checkAppList(@Param(value = "appId") Integer appId,
+                                                @Param(value = "userId") String userId,
+                                                @Param(value = "startDate") String startDate,
+                                                @Param(value = "endDate") String endDate,
+                                                @Param(value = "operateSystem") String operateSystem,
+                                                @Param(value = "projectIds")List<Long>projectIds,
+                                                @Param(value = "accountNames")List<String>accountNames);
 }

+ 14 - 2
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/PangolinReportChannelDailyMapper.java

@@ -7,7 +7,19 @@ import org.apache.ibatis.annotations.Param;
 import java.util.List;
 
 public interface PangolinReportChannelDailyMapper extends BaseMapper<PangolinReportChannelDaily> {
-    Integer sumAppList(@Param(value = "appId") Integer appId,@Param(value = "startDate") String startDate, @Param(value = "endDate") String endDate, @Param(value = "userId") String userId, @Param(value = "type") String type);
+    Integer sumAppList(@Param(value = "appId") Integer appId,
+                       @Param(value = "startDate") String startDate,
+                       @Param(value = "endDate") String endDate,
+                       @Param(value = "userId") String userId,
+                       @Param(value = "type") String type,
+                       @Param(value = "projectIds") List<Long>projectIds,
+                       @Param(value = "accountNames")List<String> accountNames);
 
-    List<PangolinReportChannelDaily> getActivationList(@Param(value = "appId") Integer appId,@Param(value = "startDate") String startDate, @Param(value = "endDate") String endDate, @Param(value = "userId") String userId, @Param(value = "type") String type);
+    List<PangolinReportChannelDaily> getActivationList(@Param(value = "appId") Integer appId,
+                                                       @Param(value = "startDate") String startDate,
+                                                       @Param(value = "endDate") String endDate,
+                                                       @Param(value = "userId") String userId,
+                                                       @Param(value = "type") String type,
+                                                       @Param(value = "projectIds") List<Long>projectIds,
+                                                       @Param(value = "accountNames")List<String> accountNames);
 }

+ 18 - 5
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/xml/PangolinAppMapper.xml

@@ -1,17 +1,30 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
 <mapper namespace="cn.com.ctop.crawler.modules.pangolin.mapper.PangolinAppMapper">
-    <select id="getListByUserId" resultType="cn.com.ctop.crawler.modules.pangolin.entity.PangolinApp">
+    <select id="getListByParams" resultType="cn.com.ctop.crawler.modules.pangolin.entity.PangolinApp">
         select app.*
             from ctop_pangolin_app app
         left join ctop_bind_account_login login on login.account_name = app.account_name
-        left join ctop_user_allocation allocation on allocation.account_id = login.id
-        where 1=1
+        where
+        1=1
         <if test="userId!=null">
             and login.user_id = #{userId}
         </if>
-        <if test="projectId!=null">
-            and allocation.project_id = #{projectId}
+        <if test="accountNames!=null or projectIds!=null">
+            and  ( 1=0
+              <if test="accountNames!=null">
+                  or login.account_name in
+                  <foreach item="item" index="index" collection="accountNames" open="(" separator="," close=")">
+                      #{item}
+                  </foreach>
+              </if>
+            <if test="projectIds!=null">
+                or login.project_id in
+                <foreach item="item" index="index" collection="projectIds" open="(" separator="," close=")">
+                    #{item}
+                </foreach>
+            </if>
+              )
         </if>
     </select>
 </mapper>

+ 80 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/xml/PangolinReportAppHourMapper.xml

@@ -25,6 +25,22 @@ and date =  #{today} and hour = #{hour}
 <if test="userId!=null">
     and login.user_id = #{userId}
 </if>
+        <if test="accountNames!=null or projectIds!=null">
+            and  ( 1=0
+            <if test="accountNames!=null">
+                or login.account_name in
+                <foreach item="item" index="index" collection="accountNames" open="(" separator="," close=")">
+                    #{item}
+                </foreach>
+            </if>
+            <if test="projectIds!=null">
+                or login.project_id in
+                <foreach item="item" index="index" collection="projectIds" open="(" separator="," close=")">
+                    #{item}
+                </foreach>
+            </if>
+            )
+        </if>
 group by appHourActive.date, appHourActive.hour
 union all select num,date,hour
 from ctop_pangolin_report_app_hour appHourActive
@@ -38,6 +54,22 @@ where appHourActive.type = #{type}
         <if test="userId!=null">
             and login.user_id = #{userId}
         </if>
+        <if test="accountNames!=null or projectIds!=null">
+            and  ( 1=0
+            <if test="accountNames!=null">
+                or login.account_name in
+                <foreach item="item" index="index" collection="accountNames" open="(" separator="," close=")">
+                    #{item}
+                </foreach>
+            </if>
+            <if test="projectIds!=null">
+                or login.project_id in
+                <foreach item="item" index="index" collection="projectIds" open="(" separator="," close=")">
+                    #{item}
+                </foreach>
+            </if>
+            )
+        </if>
         group by appHourActive.date, appHourActive.hour
 union all select num,date,hour
 from ctop_pangolin_report_app_hour appHourActive
@@ -51,6 +83,22 @@ where appHourActive.type = #{type}
         <if test="userId!=null">
             and login.user_id = #{userId}
         </if>
+        <if test="accountNames!=null or projectIds!=null">
+            and  ( 1=0
+            <if test="accountNames!=null">
+                or login.account_name in
+                <foreach item="item" index="index" collection="accountNames" open="(" separator="," close=")">
+                    #{item}
+                </foreach>
+            </if>
+            <if test="projectIds!=null">
+                or login.project_id in
+                <foreach item="item" index="index" collection="projectIds" open="(" separator="," close=")">
+                    #{item}
+                </foreach>
+            </if>
+            )
+        </if>
         group by appHourActive.date, appHourActive.hour
 <if test="compareTime1!=null">
     union all
@@ -66,6 +114,22 @@ where appHourActive.type = #{type}
     <if test="userId!=null">
         and login.user_id = #{userId}
     </if>
+    <if test="accountNames!=null or projectIds!=null">
+        and  ( 1=0
+        <if test="accountNames!=null">
+            or login.account_name in
+            <foreach item="item" index="index" collection="accountNames" open="(" separator="," close=")">
+                #{item}
+            </foreach>
+        </if>
+        <if test="projectIds!=null">
+            or login.project_id in
+            <foreach item="item" index="index" collection="projectIds" open="(" separator="," close=")">
+                #{item}
+            </foreach>
+        </if>
+        )
+    </if>
     group by appHourActive.date, appHourActive.hour
 </if>
         <if test="compareTime2!=null">
@@ -82,6 +146,22 @@ where appHourActive.type = #{type}
             <if test="userId!=null">
                 and login.user_id = #{userId}
             </if>
+            <if test="accountNames!=null or projectIds!=null">
+                and  ( 1=0
+                <if test="accountNames!=null">
+                    or login.account_name in
+                    <foreach item="item" index="index" collection="accountNames" open="(" separator="," close=")">
+                        #{item}
+                    </foreach>
+                </if>
+                <if test="projectIds!=null">
+                    or login.project_id in
+                    <foreach item="item" index="index" collection="projectIds" open="(" separator="," close=")">
+                        #{item}
+                    </foreach>
+                </if>
+                )
+            </if>
             group by appHourActive.date, appHourActive.hour
         </if>
         )a group by hour

+ 18 - 2
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/xml/PangolinReportBillMonthMapper.xml

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
 <mapper namespace="cn.com.ctop.crawler.modules.pangolin.mapper.PangolinReportBillMonthMapper">
-    <select id="checkAppList" resultType="cn.com.ctop.crawler.modules.pangolin.vo.PangolinReportAppMonthVO">
+    <select id="checkAppList" resultType="cn.com.ctop.common.module.vo.PangolinReportAppMonthVO">
         select
        bill.date,
         <if test="appId == null">
@@ -15,7 +15,7 @@
        sum(bill.estimated_total_price) as 'estimated_total_price',
        round(avg(bill.account_quality_score),2) as 'account_quality_score',
        sum(bill.actual_total_price) as 'actual_total_price'
-from ctop_pangolin_report_bill_month bill
+    from ctop_pangolin_report_bill_month bill
          left join ctop_pangolin_app app on app.id = bill.app_id
          left join ctop_bind_account_login login on login.account_name = app.account_name
          where 1=1
@@ -34,6 +34,22 @@ from ctop_pangolin_report_bill_month bill
         <if test="operateSystem!=null">
            and bill.os = #{operateSystem}
         </if>
+        <if test="accountNames!=null or projectIds!=null">
+            and  ( 1=0
+            <if test="accountNames!=null">
+                or login.account_name in
+                <foreach item="item" index="index" collection="accountNames" open="(" separator="," close=")">
+                    #{item}
+                </foreach>
+            </if>
+            <if test="projectIds!=null">
+                or login.project_id in
+                <foreach item="item" index="index" collection="projectIds" open="(" separator="," close=")">
+                    #{item}
+                </foreach>
+            </if>
+            )
+        </if>
         group by bill.date,bill.os order by bill.date desc
     </select>
 </mapper>

+ 32 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/xml/PangolinReportChannelDailyMapper.xml

@@ -37,6 +37,22 @@
         <if test="type!=null">
             and daily.os = #{type}
         </if>
+        <if test="accountNames!=null or projectIds!=null">
+            and  ( 1=0
+            <if test="accountNames!=null">
+                or login.account_name in
+                <foreach item="item" index="index" collection="accountNames" open="(" separator="," close=")">
+                    #{item}
+                </foreach>
+            </if>
+            <if test="projectIds!=null">
+                or login.project_id in
+                <foreach item="item" index="index" collection="projectIds" open="(" separator="," close=")">
+                    #{item}
+                </foreach>
+            </if>
+            )
+        </if>
         group by daily.date
     </select>
     <select id="sumAppList" resultType="java.lang.Integer">
@@ -63,6 +79,22 @@
         <if test="type!=null">
             and daily.os = #{type}
         </if>
+        <if test="accountNames!=null or projectIds!=null">
+            and  ( 1=0
+            <if test="accountNames!=null">
+                or login.account_name in
+                <foreach item="item" index="index" collection="accountNames" open="(" separator="," close=")">
+                    #{item}
+                </foreach>
+            </if>
+            <if test="projectIds!=null">
+                or login.project_id in
+                <foreach item="item" index="index" collection="projectIds" open="(" separator="," close=")">
+                    #{item}
+                </foreach>
+            </if>
+            )
+        </if>
 
     </select>
 

+ 2 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/PangolinAppService.java

@@ -8,4 +8,6 @@ import java.util.Map;
 
 public interface PangolinAppService extends IService<PangolinApp> {
     Map<String, Object> getListByUserId(String userId, JSONObject data);
+
+    Map<String, Object> getMemberList(String userId);
 }

+ 1 - 1
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/PangolinReportBillMonthService.java

@@ -1,7 +1,7 @@
 package cn.com.ctop.crawler.modules.pangolin.service;
 
+import cn.com.ctop.common.module.vo.PangolinReportAppMonthVO;
 import cn.com.ctop.crawler.modules.pangolin.entity.PangolinReportBillMonth;
-import cn.com.ctop.crawler.modules.pangolin.vo.PangolinReportAppMonthVO;
 import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.extension.service.IService;
 import com.github.pagehelper.PageInfo;

+ 34 - 4
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinAppServiceImpl.java

@@ -1,15 +1,20 @@
 package cn.com.ctop.crawler.modules.pangolin.service.impl;
 
+import cn.com.ctop.common.module.service.IBindAccountLoginService;
+import cn.com.ctop.common.module.utils.CtopAdConstant;
 import cn.com.ctop.common.module.utils.ResultMapUtils;
 import cn.com.ctop.common.module.utils.StatusCode;
+import cn.com.ctop.common.module.vo.PangolinProjectVO;
 import cn.com.ctop.crawler.modules.pangolin.entity.PangolinApp;
 import cn.com.ctop.crawler.modules.pangolin.mapper.PangolinAppMapper;
 import cn.com.ctop.crawler.modules.pangolin.service.PangolinAppService;
+import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -18,16 +23,41 @@ import java.util.Map;
 public class PangolinAppServiceImpl extends ServiceImpl<PangolinAppMapper, PangolinApp> implements PangolinAppService {
     @Autowired
     private PangolinAppMapper pangolinAppMapper;
+    @Autowired
+    private IBindAccountLoginService bindAccountLoginService;
     @Override
     public Map<String, Object> getListByUserId(String userId, JSONObject data) {
         Map<String,Object>result = new HashMap<>();
-        Long projectId = data.getLong("projectId");
-        if(null == projectId||projectId == 0){
-            projectId = null;
+        System.out.println(data.toJSONString());
+        List<Long>projectIds = null;
+        List<String>accountNames = null;
+        JSONArray accountNameArray = data.getJSONArray("accountName");
+        JSONArray projectIdArray = data.getJSONArray("projectId");
+        if(null!=accountNameArray&&!accountNameArray.isEmpty()){
+            accountNames = new ArrayList<>();
+            for(int i=0;i<accountNameArray.size();i++){
+                accountNames.add(accountNameArray.getString(i));
+            }
+        }
+        if(null!=projectIdArray&&!projectIdArray.isEmpty()){
+            projectIds = new ArrayList<>();
+            for(int i=0;i<projectIdArray.size();i++){
+                projectIds.add(projectIdArray.getLong(i));
+            }
         }
-        List<PangolinApp>appList = pangolinAppMapper.getListByUserId(userId,projectId);
+        List<PangolinApp>appList = pangolinAppMapper.getListByParams(userId,projectIds,accountNames);
         result.put("data",appList);
         ResultMapUtils.setResultMap(result, StatusCode.COMMON_SUCCESS);
         return result;
     }
+
+    @Override
+    public Map<String, Object> getMemberList(String userId) {
+        Map<String,Object>resultMap = new HashMap<>();
+        //1:查询所有项目信息
+        List<PangolinProjectVO> projectVOS = bindAccountLoginService.getProjectByParams(userId, CtopAdConstant.PLATFORM_TYPE_PANGOLIN_PY);
+        resultMap.put("data",projectVOS);
+        ResultMapUtils.setResultMap(resultMap,StatusCode.COMMON_SUCCESS);
+        return resultMap;
+    }
 }

+ 20 - 3
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinReportAppHourServiceImpl.java

@@ -5,6 +5,7 @@ import cn.com.ctop.common.module.utils.StatusCode;
 import cn.com.ctop.crawler.modules.pangolin.entity.PangolinReportAppHour;
 import cn.com.ctop.crawler.modules.pangolin.mapper.PangolinReportAppHourMapper;
 import cn.com.ctop.crawler.modules.pangolin.service.PangolinReportAppHourService;
+import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import org.jeecg.common.util.DateUtils;
@@ -32,12 +33,28 @@ public class PangolinReportAppHourServiceImpl extends ServiceImpl<PangolinReport
         String compareTime1 = data.getString("compareTime1");
         String compareTime2 = data.getString("compareTime2");
         Integer appId = data.getInteger("appId");
+        List<Long>projectIds = null;
+        List<String>accountNames = null;
+        JSONArray accountNameArray = data.getJSONArray("accountName");
+        JSONArray projectIdArray = data.getJSONArray("projectId");
+        if(null!=accountNameArray&&!accountNameArray.isEmpty()){
+            accountNames = new ArrayList<>();
+            for(int i=0;i<accountNameArray.size();i++){
+                accountNames.add(accountNameArray.getString(i));
+            }
+        }
+        if(null!=projectIdArray&&!projectIdArray.isEmpty()){
+            projectIds = new ArrayList<>();
+            for(int i=0;i<projectIdArray.size();i++){
+                projectIds.add(projectIdArray.getLong(i));
+            }
+        }
         if(null == appId||appId == 0){
             appId = null;
         }
         List<JSONObject> dataList = new ArrayList<>();
         for(int i=0;i<24;i++){
-            JSONObject getData = this.getComparedDate(appId,i,userId,type,today,yesterday,sevenDaysAgo,compareTime1,compareTime2);
+            JSONObject getData = this.getComparedDate(appId,i,userId,type,today,yesterday,sevenDaysAgo,compareTime1,compareTime2,projectIds,accountNames);
             dataList.add(getData);
         }
         resultMap.put("data",dataList);
@@ -45,7 +62,7 @@ public class PangolinReportAppHourServiceImpl extends ServiceImpl<PangolinReport
         return resultMap;
     }
 
-    private JSONObject getComparedDate(Integer appId,Integer hour,String userId,String type,String today, String yesterday, String sevenDaysAgo, String compareTime1, String compareTime2) {
-        return pangolinReportAppHourMapper.getComparedDate(appId,hour,userId,type,today, yesterday,sevenDaysAgo,compareTime1,compareTime2);
+    private JSONObject getComparedDate(Integer appId,Integer hour,String userId,String type,String today, String yesterday, String sevenDaysAgo, String compareTime1, String compareTime2,List<Long>projectIds,List<String>  accountNames) {
+        return pangolinReportAppHourMapper.getComparedDate(appId,hour,userId,type,today, yesterday,sevenDaysAgo,compareTime1,compareTime2,projectIds,accountNames);
     }
 }

+ 20 - 2
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinReportBillMonthServiceImpl.java

@@ -1,9 +1,10 @@
 package cn.com.ctop.crawler.modules.pangolin.service.impl;
 
+import cn.com.ctop.common.module.vo.PangolinReportAppMonthVO;
 import cn.com.ctop.crawler.modules.pangolin.entity.PangolinReportBillMonth;
 import cn.com.ctop.crawler.modules.pangolin.mapper.PangolinReportBillMonthMapper;
 import cn.com.ctop.crawler.modules.pangolin.service.PangolinReportBillMonthService;
-import cn.com.ctop.crawler.modules.pangolin.vo.PangolinReportAppMonthVO;
+import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.github.pagehelper.PageHelper;
@@ -11,6 +12,7 @@ import com.github.pagehelper.PageInfo;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
+import java.util.ArrayList;
 import java.util.List;
 
 @Service
@@ -32,6 +34,22 @@ public class PangolinReportBillMonthServiceImpl extends ServiceImpl<PangolinRepo
         if(null == appId||appId == 0){
             appId = null;
         }
+        List<Long>projectIds = null;
+        List<String>accountNames = null;
+        JSONArray accountNameArray = data.getJSONArray("accountName");
+        JSONArray projectIdArray = data.getJSONArray("projectId");
+        if(null!=accountNameArray&&!accountNameArray.isEmpty()){
+            accountNames = new ArrayList<>();
+            for(int i=0;i<accountNameArray.size();i++){
+                accountNames.add(accountNameArray.getString(i));
+            }
+        }
+        if(null!=projectIdArray&&!projectIdArray.isEmpty()){
+            projectIds = new ArrayList<>();
+            for(int i=0;i<projectIdArray.size();i++){
+                projectIds.add(projectIdArray.getLong(i));
+            }
+        }
         String startDate = data.getString("startDate");
         String endDate = data.getString("endDate");
         String operateSystem = data.getString("operateSystem");
@@ -39,7 +57,7 @@ public class PangolinReportBillMonthServiceImpl extends ServiceImpl<PangolinRepo
             operateSystem = null;
         }
         PageHelper.startPage(pageNo,pageSize);
-        List<PangolinReportAppMonthVO> list = pangolinReportBillMonthMapper.checkAppList(appId,userId,startDate,endDate,operateSystem);
+        List<PangolinReportAppMonthVO> list = pangolinReportBillMonthMapper.checkAppList(appId,userId,startDate,endDate,operateSystem,projectIds,accountNames);
         PageInfo<PangolinReportAppMonthVO> pageInfo = new PageInfo<>(list);
         return pageInfo;
     }

+ 20 - 2
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinReportChannelDailyServiceImpl.java

@@ -5,6 +5,7 @@ import cn.com.ctop.common.module.utils.StatusCode;
 import cn.com.ctop.crawler.modules.pangolin.entity.PangolinReportChannelDaily;
 import cn.com.ctop.crawler.modules.pangolin.mapper.PangolinReportChannelDailyMapper;
 import cn.com.ctop.crawler.modules.pangolin.service.PangolinReportChannelDailyService;
+import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.github.pagehelper.PageHelper;
@@ -13,6 +14,7 @@ import org.jeecg.common.util.DateUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -41,13 +43,29 @@ public class PangolinReportChannelDailyServiceImpl extends ServiceImpl<PangolinR
         if(null == appId||appId == 0){
             appId = null;
         }
+        List<Long>projectIds = null;
+        List<String>accountNames = null;
+        JSONArray accountNameArray = data.getJSONArray("accountName");
+        JSONArray projectIdArray = data.getJSONArray("projectId");
+        if(null!=accountNameArray&&!accountNameArray.isEmpty()){
+            accountNames = new ArrayList<>();
+            for(int i=0;i<accountNameArray.size();i++){
+                accountNames.add(accountNameArray.getString(i));
+            }
+        }
+        if(null!=projectIdArray&&!projectIdArray.isEmpty()){
+            projectIds = new ArrayList<>();
+            for(int i=0;i<projectIdArray.size();i++){
+                projectIds.add(projectIdArray.getLong(i));
+            }
+        }
         String startDate = data.getString("startDate");
         String endDate = data.getString("endDate");
         PageHelper.startPage(pageNo,pageSize);
-        List<PangolinReportChannelDaily> dailyList = pangolinReportChannelDailyMapper.getActivationList(appId,startDate, endDate, userId, operateSystem);
+        List<PangolinReportChannelDaily> dailyList = pangolinReportChannelDailyMapper.getActivationList(appId,startDate, endDate, userId, operateSystem,projectIds,accountNames);
         PageInfo<PangolinReportChannelDaily> pageInfo = new PageInfo<>(dailyList);
         result.put("data", pageInfo);
-        Integer totalActive = pangolinReportChannelDailyMapper.sumAppList(appId,startDate,endDate,userId,operateSystem);
+        Integer totalActive = pangolinReportChannelDailyMapper.sumAppList(appId,startDate,endDate,userId,operateSystem,projectIds,accountNames);
         Long totalDays = 1L;
         if(!startDate.equals(endDate)){
             totalDays = DateUtils.getDiscrepantDays(startDate,endDate)+1;