Преглед изворни кода

重构穿山甲爬虫获取逻辑

syh пре 5 година
родитељ
комит
0497d73852
15 измењених фајлова са 113 додато и 95 уклоњено
  1. 5 13
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/mapper/xml/KuaishouMatcostAccountMayMapper.xml
  2. 1 1
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/service/impl/KuaishouMatcostAccountMayServiceImpl.java
  3. 27 5
      jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java
  4. 1 1
      module-common/src/main/java/cn/com/ctop/common/module/mapper/xml/BindAccountLoginMapper.xml
  5. 13 11
      module-common/src/main/java/cn/com/ctop/common/module/service/impl/BindAccountLoginServiceImpl.java
  6. 2 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/entity/PangolinReportBillMonth.java
  7. 13 13
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/xml/PangolinAppMapper.xml
  8. 19 19
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/xml/PangolinReportAppHourMapper.xml
  9. 4 4
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/xml/PangolinReportBillMonthMapper.xml
  10. 10 12
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/xml/PangolinReportChannelDailyMapper.xml
  11. 1 1
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinAppServiceImpl.java
  12. 13 11
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinCrawlerServiceImpl.java
  13. 0 1
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinReportAppHourServiceImpl.java
  14. 0 2
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/material/service/impl/KuaishouBidWarningServiceImpl.java
  15. 4 1
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/material/service/impl/BytedanceBidWarningServiceImpl.java

+ 5 - 13
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/mapper/xml/KuaishouMatcostAccountMayMapper.xml

@@ -10,24 +10,16 @@
         where video_code = #{code}
     </update>
     <select id="getMinDateByCode" resultType="java.lang.String">
-        SELECT
-            min(s.stat_date)
-        FROM
-            ctop_kuaishou_report_daily_creative s
-            LEFT JOIN ctop_kuaishou_creative creative2 ON s.creative_id = creative2.creative_id
-            left join ctop_kuaishou_video_get video on video.photo_id = creative2.photo_id and creative2.account_id = video.account_id
-        WHERE
-            video.signature = #{code}
+    select substr(min(tmp.stat_date),1,10) from kuaishou_video_new_v2 video
+        left join tmp_kuaishou_creative_statdate2 tmp on tmp.photo_id = video.photo_id
+        where video.video_code = #{code}
     </select>
     <select id="getCostByParams" resultType="java.math.BigDecimal">
         SELECT
             sum(report.charge)
         FROM
-        ctop_kuaishou_report_daily_creative report
-        LEFT JOIN ctop_kuaishou_creative creative2 ON report.creative_id = creative2.creative_id
-        left join ctop_kuaishou_video_get video on video.photo_id = creative2.photo_id and creative2.account_id = video.account_id
+        tmp_kuaishou_creative_report report
         WHERE
-        video.signature =#{code} and report.stat_date &gt;= #{startDate} and report.stat_date &lt;= #{endDate};
-
+        report.video_code =#{code} and report.state_date &gt;= #{startDate} and report.state_date &lt; #{endDate};
     </select>
 </mapper>

+ 1 - 1
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/service/impl/KuaishouMatcostAccountMayServiceImpl.java

@@ -17,7 +17,7 @@ public class KuaishouMatcostAccountMayServiceImpl extends ServiceImpl<KuaishouMa
     @Override
     public KuaishouMatcostAccountMay getOne() {
         QueryWrapper<KuaishouMatcostAccountMay>queryWrapper = new QueryWrapper<>();
-        queryWrapper.isNotNull("end_date").isNotNull("video_code").isNull("twcost").last("limit 1");
+        queryWrapper.isNull("twcost").isNotNull("end_date").last("limit 1");
         return this.getOne(queryWrapper);
     }
 

+ 27 - 5
jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java

@@ -10,6 +10,7 @@ import cn.com.ctop.crawler.modules.pangolin.service.PangolinCrawlerService;
 import cn.com.ctop.crawler.modules.pangolin.service.PangolinLoginService;
 import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouWebInterfaceService;
 import cn.com.ctop.toutiao.modules.material.service.IByteDanceAdvertiserDataService;
+import cn.com.ctop.toutiao.modules.report.service.IReportService;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import lombok.extern.slf4j.Slf4j;
 import org.jeecg.common.util.DateUtils;
@@ -22,6 +23,7 @@ import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.test.context.junit4.SpringRunner;
 
 import java.math.BigDecimal;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
@@ -36,10 +38,23 @@ public class SampleTest {
     private ICtopOauthTokenService oauthTokenService;
     @Autowired
     private IByteDanceAdvertiserDataService advertiserDataService;
+    @Autowired
+    private IReportService reportService;
     @Test
     public void testOceanEngineJob() {
-        CtopOauthToken token = oauthTokenService.getTokenByAccountId(1629785592929294L);
-        advertiserDataService.getMaterialList(token);
+        CtopOauthToken token1 = oauthTokenService.getTokenByAccountId(1670725854459911L);
+        CtopOauthToken token2 = oauthTokenService.getTokenByAccountId(1670827164313607L);
+        CtopOauthToken token3 = oauthTokenService.getTokenByAccountId(1670801593147405L);
+        CtopOauthToken token4 = oauthTokenService.getTokenByAccountId(1670827163327501L);
+        CtopOauthToken token5 = oauthTokenService.getTokenByAccountId(1670827163853832L);
+        for(int i=0;i<100;i++){
+            Date startDate = DateUtils.addDay(new Date(),-i);
+            reportService.getAdvertiserReport(token1,startDate,startDate,CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
+            reportService.getAdvertiserReport(token2,startDate,startDate,CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
+            reportService.getAdvertiserReport(token3,startDate,startDate,CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
+            reportService.getAdvertiserReport(token4,startDate,startDate,CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
+            reportService.getAdvertiserReport(token5,startDate,startDate,CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
+        }
     }
 
     @Autowired
@@ -133,6 +148,13 @@ public class SampleTest {
             }else{
                 matcostAccountMayService.updateCostByCode(matcostAccountMay.getVideoCode(),BigDecimal.ZERO);
             }
+//            String minDate = matcostAccountMayService.getMinDate(matcostAccountMay.getVideoCode());
+//            if(null == minDate||minDate.trim().equals("")){
+//                matcostAccountMayService.updateByCode(matcostAccountMay.getVideoCode(),"-");
+//            }else{
+//                matcostAccountMayService.updateByCode(matcostAccountMay.getVideoCode(),minDate);
+//            }
+
             loadData();
         }
     }
@@ -146,9 +168,9 @@ public class SampleTest {
         List<BindAccountLogin> list = bindAccountLoginService.list(queryWrapper);
         if (list != null && !list.isEmpty()) {
             for (BindAccountLogin bindAccountLogin : list) {
-//                pangolinCrawlerService.getChannelList(bindAccountLogin);
-                pangolinCrawlerService.getActivationList(bindAccountLogin, DateUtils.getNowDate("yyyy-MM-dd"), 1);
-                pangolinCrawlerService.getCheckList(bindAccountLogin);
+                pangolinCrawlerService.getChannelList(bindAccountLogin);
+//                pangolinCrawlerService.getActivationList(bindAccountLogin, DateUtils.getNowDate("yyyy-MM-dd"), 1);
+//                pangolinCrawlerService.getCheckList(bindAccountLogin);
 //                QueryWrapper<PangolinApp> queryWrapper1 = new QueryWrapper<>();
 //                queryWrapper1.eq("account_name", bindAccountLogin.getAccountName());
 //                List<PangolinApp> appList = pangolinAppService.list(queryWrapper1);

+ 1 - 1
module-common/src/main/java/cn/com/ctop/common/module/mapper/xml/BindAccountLoginMapper.xml

@@ -25,7 +25,7 @@
         from
         ctop_bind_account_login login
         left join ctop_project project on login.project_id = project.id
-        where 1=1
+        where login.status = 1
         <if test="userId!=null">
             and login.user_id = #{userId}
         </if>

+ 13 - 11
module-common/src/main/java/cn/com/ctop/common/module/service/impl/BindAccountLoginServiceImpl.java

@@ -34,6 +34,8 @@ import java.util.Map;
 @Slf4j
 @Service
 public class BindAccountLoginServiceImpl extends ServiceImpl<BindAccountLoginMapper, BindAccountLogin> implements IBindAccountLoginService {
+    @Autowired
+    private BindAccountLoginMapper bindAccountLoginMapper;
     @Override
     public boolean checkLogin(BindAccountLogin bindAccountLogin, String response) {
         if (response.contains("<html>")) {
@@ -43,6 +45,7 @@ public class BindAccountLoginServiceImpl extends ServiceImpl<BindAccountLoginMap
         }
         return true;
     }
+
     @Override
     public BindAccountLogin selectByAccountId(String accountId) {
         QueryWrapper<BindAccountLogin> queryWrapper = new QueryWrapper<>();
@@ -68,21 +71,20 @@ public class BindAccountLoginServiceImpl extends ServiceImpl<BindAccountLoginMap
 
     @Override
     public List<PangolinProjectVO> getProjectByParams(String userId, String loginType) {
-        List<PangolinProjectVO> projectVOS =  bindAccountLoginMapper.getProjectByParams(userId,loginType,null);
-        List<PangolinProjectVO> setProjectVOS = new ArrayList<>();
-        if(null!=projectVOS&&!projectVOS.isEmpty()){
-            projectVOS.forEach(pangolinProjectVO -> {
-                List<PangolinAccountVO>accountVOS = bindAccountLoginMapper.getAccountByParams(userId,loginType,pangolinProjectVO.getProjectId());
-                pangolinProjectVO.setAccountVOList(accountVOS);
-                setProjectVOS.add(pangolinProjectVO);
+        List<PangolinProjectVO> projectVos =  bindAccountLoginMapper.getProjectByParams(userId,loginType,null);
+        List<PangolinProjectVO> setProjectVos = new ArrayList<>();
+        if(null!=projectVos&&!projectVos.isEmpty()){
+            projectVos.forEach(pangolinProjectVO -> {
+                List<PangolinAccountVO>accountVos = bindAccountLoginMapper.getAccountByParams(userId,loginType,pangolinProjectVO.getProjectId());
+                if(null!=accountVos&&!accountVos.isEmpty()){
+                    pangolinProjectVO.setAccountVOList(accountVos);
+                    setProjectVos.add(pangolinProjectVO);
+                }
             });
         }
-        return setProjectVOS;
+        return setProjectVos;
     }
 
-    @Autowired
-    private BindAccountLoginMapper bindAccountLoginMapper;
-
     @Override
     public Map<String, Object> getListByUserId(String userId) {
         Map<String,Object>result = new HashMap<>();

+ 2 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/entity/PangolinReportBillMonth.java

@@ -44,4 +44,6 @@ public class PangolinReportBillMonth {
      * 账户质量分
      */
     private BigDecimal accountQualityScore;
+
+    private String accountName;
 }

+ 13 - 13
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/xml/PangolinAppMapper.xml

@@ -2,28 +2,28 @@
 <!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
 <mapper namespace="cn.com.ctop.crawler.modules.pangolin.mapper.PangolinAppMapper">
     <select id="getListByParams" resultType="cn.com.ctop.crawler.modules.pangolin.entity.PangolinApp">
-        select app.*
+        select distinct app.app_id as 'id',name
             from ctop_pangolin_app app
         left join ctop_bind_account_login login on login.account_name = app.account_name
         where
-        1=1
+        login.status = 1
         <if test="userId!=null">
             and login.user_id = #{userId}
         </if>
         <if test="accountNames!=null or projectIds!=null">
             and  ( 1=0
-              <if test="accountNames!=null">
-                  or login.account_name in
-                  <foreach item="item" index="index" collection="accountNames" open="(" separator="," close=")">
+                <if test="accountNames!=null">
+                    or login.account_name in
+                    <foreach item="item" index="index" collection="accountNames" open="(" separator="," close=")">
                       #{item}
-                  </foreach>
-              </if>
-            <if test="projectIds!=null">
-                or login.project_id in
-                <foreach item="item" index="index" collection="projectIds" open="(" separator="," close=")">
-                    #{item}
-                </foreach>
-            </if>
+                    </foreach>
+                </if>
+                <if test="projectIds!=null">
+                    or login.project_id in
+                    <foreach item="item" index="index" collection="projectIds" open="(" separator="," close=")">
+                        #{item}
+                    </foreach>
+                </if>
               )
         </if>
     </select>

+ 19 - 19
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/xml/PangolinReportAppHourMapper.xml

@@ -15,12 +15,12 @@
             hour
        from (select num,date,hour
 from ctop_pangolin_report_app_hour appHourActive
-         left join ctop_pangolin_app app on app.id = appHourActive.app_id
-         left join ctop_bind_account_login login on login.account_name = app.account_name
+         left join ctop_pangolin_app app on app.app_id = appHourActive.app_id
+         left join ctop_bind_account_login login on login.account_name = appHourActive.account_name
 where appHourActive.type = #{type}
 and date =  #{today} and hour = #{hour}
 <if test="appId!=null">
-    and app.id = #{appId}
+    and app.app_id = #{appId}
 </if>
 <if test="userId!=null">
     and login.user_id = #{userId}
@@ -28,7 +28,7 @@ and date =  #{today} and hour = #{hour}
         <if test="accountNames!=null or projectIds!=null">
             and  ( 1=0
             <if test="accountNames!=null">
-                or login.account_name in
+                or appHourActive.account_name in
                 <foreach item="item" index="index" collection="accountNames" open="(" separator="," close=")">
                     #{item}
                 </foreach>
@@ -44,12 +44,12 @@ and date =  #{today} and hour = #{hour}
 group by appHourActive.date, appHourActive.hour
 union all select num,date,hour
 from ctop_pangolin_report_app_hour appHourActive
-         left join ctop_pangolin_app app on app.id = appHourActive.app_id
-         left join ctop_bind_account_login login on login.account_name = app.account_name
+         left join ctop_pangolin_app app on app.app_id = appHourActive.app_id
+         left join ctop_bind_account_login login on login.account_name = appHourActive.account_name
 where appHourActive.type = #{type}
         and date =  #{yesterday} and hour = #{hour}
         <if test="appId!=null">
-            and app.id = #{appId}
+            and app.app_id = #{appId}
         </if>
         <if test="userId!=null">
             and login.user_id = #{userId}
@@ -57,7 +57,7 @@ where appHourActive.type = #{type}
         <if test="accountNames!=null or projectIds!=null">
             and  ( 1=0
             <if test="accountNames!=null">
-                or login.account_name in
+                or appHourActive.account_name in
                 <foreach item="item" index="index" collection="accountNames" open="(" separator="," close=")">
                     #{item}
                 </foreach>
@@ -73,12 +73,12 @@ where appHourActive.type = #{type}
         group by appHourActive.date, appHourActive.hour
 union all select num,date,hour
 from ctop_pangolin_report_app_hour appHourActive
-         left join ctop_pangolin_app app on app.id = appHourActive.app_id
-         left join ctop_bind_account_login login on login.account_name = app.account_name
+         left join ctop_pangolin_app app on app.app_id = appHourActive.app_id
+         left join ctop_bind_account_login login on login.account_name = appHourActive.account_name
 where appHourActive.type = #{type}
         and date =  #{sevenDaysAgo} and hour = #{hour}
         <if test="appId!=null">
-            and app.id = #{appId}
+            and app.app_id = #{appId}
         </if>
         <if test="userId!=null">
             and login.user_id = #{userId}
@@ -86,7 +86,7 @@ where appHourActive.type = #{type}
         <if test="accountNames!=null or projectIds!=null">
             and  ( 1=0
             <if test="accountNames!=null">
-                or login.account_name in
+                or appHourActive.account_name in
                 <foreach item="item" index="index" collection="accountNames" open="(" separator="," close=")">
                     #{item}
                 </foreach>
@@ -104,8 +104,8 @@ where appHourActive.type = #{type}
     union all
     select num,date,hour
     from ctop_pangolin_report_app_hour appHourActive
-    left join ctop_pangolin_app app on app.id = appHourActive.app_id
-    left join ctop_bind_account_login login on login.account_name = app.account_name
+    left join ctop_pangolin_app app on app.app_id = appHourActive.app_id
+    left join ctop_bind_account_login login on login.account_name = appHourActive.account_name
     where appHourActive.type = #{type}
     and date =  #{compareTime1} and hour = #{hour}
     <if test="appId!=null">
@@ -117,7 +117,7 @@ where appHourActive.type = #{type}
     <if test="accountNames!=null or projectIds!=null">
         and  ( 1=0
         <if test="accountNames!=null">
-            or login.account_name in
+            or appHourActive.account_name in
             <foreach item="item" index="index" collection="accountNames" open="(" separator="," close=")">
                 #{item}
             </foreach>
@@ -136,12 +136,12 @@ where appHourActive.type = #{type}
             union all
             select num,date,hour
             from ctop_pangolin_report_app_hour appHourActive
-            left join ctop_pangolin_app app on app.id = appHourActive.app_id
-            left join ctop_bind_account_login login on login.account_name = app.account_name
+            left join ctop_pangolin_app app on app.app_id = appHourActive.app_id
+            left join ctop_bind_account_login login on login.account_name = appHourActive.account_name
             where appHourActive.type = #{type}
             and date =  #{compareTime2} and hour = #{hour}
             <if test="appId!=null">
-                and app.id = #{appId}
+                and app.app_id = #{appId}
             </if>
             <if test="userId!=null">
                 and login.user_id = #{userId}
@@ -149,7 +149,7 @@ where appHourActive.type = #{type}
             <if test="accountNames!=null or projectIds!=null">
                 and  ( 1=0
                 <if test="accountNames!=null">
-                    or login.account_name in
+                    or appHourActive.account_name in
                     <foreach item="item" index="index" collection="accountNames" open="(" separator="," close=")">
                         #{item}
                     </foreach>

+ 4 - 4
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/xml/PangolinReportBillMonthMapper.xml

@@ -16,14 +16,14 @@
        round(avg(bill.account_quality_score),2) as 'account_quality_score',
        sum(bill.actual_total_price) as 'actual_total_price'
     from ctop_pangolin_report_bill_month bill
-         left join ctop_pangolin_app app on app.id = bill.app_id
-         left join ctop_bind_account_login login on login.account_name = app.account_name
+         left join ctop_pangolin_app app on app.app_id = bill.app_id
+         left join ctop_bind_account_login login on login.account_name = bill.account_name
          where 1=1
          <if test="userId!=null">
              and login.user_id = #{userId}
          </if>
          <if test="appId!=null">
-             and app.id = #{appId}
+             and app.app_id = #{appId}
          </if>
         <if test="startDate!=null">
             and bill.date &gt;= #{startDate}
@@ -37,7 +37,7 @@
         <if test="accountNames!=null or projectIds!=null">
             and  ( 1=0
             <if test="accountNames!=null">
-                or login.account_name in
+                or bill.account_name in
                 <foreach item="item" index="index" collection="accountNames" open="(" separator="," close=")">
                     #{item}
                 </foreach>

+ 10 - 12
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/mapper/xml/PangolinReportChannelDailyMapper.xml

@@ -6,7 +6,8 @@
             resultType="cn.com.ctop.crawler.modules.pangolin.entity.PangolinReportChannelDaily">
         select
             daily.date,
-            channel.name as 'channel_name',
+--             daily.channel_name as 'channel_name',
+            '-' as 'channel_name',
             <if test="appId == null">
                 '-' as 'app_name',
             </if>
@@ -18,15 +19,14 @@
             sum(daily.total_price)as 'total_price'
         from
         ctop_pangolin_report_channel_daily daily
-        left join ctop_pangolin_channel channel on channel.name = daily.channel_name
-        left join ctop_bind_account_login login on login.account_name = channel.account_name
-        left join ctop_pangolin_app app on app.account_name = channel.account_name
+        left join ctop_bind_account_login login on login.account_name = daily.account_name
+        left join ctop_pangolin_app app on app.name = daily.app_name and app.account_name = daily.account_name
         where 1=1
         <if test="userId!=null">
             and login.user_id = #{userId}
         </if>
         <if test="appId!=null">
-            and app.id = #{appId}
+            and app.app_id = #{appId}
         </if>
         <if test="startDate!=null">
             and daily.date &gt;= #{startDate}
@@ -40,7 +40,7 @@
         <if test="accountNames!=null or projectIds!=null">
             and  ( 1=0
             <if test="accountNames!=null">
-                or login.account_name in
+                or daily.account_name in
                 <foreach item="item" index="index" collection="accountNames" open="(" separator="," close=")">
                     #{item}
                 </foreach>
@@ -60,15 +60,14 @@
             sum(daily.external_activation)
         from
         ctop_pangolin_report_channel_daily daily
-        left join ctop_pangolin_channel channel on channel.name = daily.channel_name
-        left join ctop_bind_account_login login on login.account_name = channel.account_name
-        left join ctop_pangolin_app app on app.account_name = channel.account_name
+        left join ctop_bind_account_login login on login.account_name = daily.account_name
+        left join ctop_pangolin_app app on app.account_name = daily.account_name and daily.app_name = app.name
         where 1=1
         <if test="userId!=null">
             and login.user_id = #{userId}
         </if>
         <if test="appId!=null">
-            and app.id = #{appId}
+            and app.app_id = #{appId}
         </if>
         <if test="startDate!=null">
             and daily.date &gt;= #{startDate}
@@ -82,7 +81,7 @@
         <if test="accountNames!=null or projectIds!=null">
             and  ( 1=0
             <if test="accountNames!=null">
-                or login.account_name in
+                or daily.account_name in
                 <foreach item="item" index="index" collection="accountNames" open="(" separator="," close=")">
                     #{item}
                 </foreach>
@@ -95,7 +94,6 @@
             </if>
             )
         </if>
-
     </select>
 
 </mapper>

+ 1 - 1
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinAppServiceImpl.java

@@ -71,7 +71,7 @@ public class PangolinAppServiceImpl extends ServiceImpl<PangolinAppMapper, Pango
         if(null == getApp){
             this.save(app);
         }else{
-            getApp.setName(app.getAccountName());
+            getApp.setName(app.getName());
             this.updateById(getApp);
         }
     }

+ 13 - 11
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinCrawlerServiceImpl.java

@@ -80,11 +80,13 @@ public class PangolinCrawlerServiceImpl implements PangolinCrawlerService {
                     pangolinReportBillMonth.setYear(Integer.parseInt(date.split("-")[0]));
                     pangolinReportBillMonth.setMonth(Integer.parseInt(date.split("-")[1]));
                     pangolinReportBillMonth.setDate(date);
+                    pangolinReportBillMonth.setAccountName(bindAccountLogin.getAccountName());
                     QueryWrapper<PangolinReportBillMonth> queryWrapper = new QueryWrapper<>();
                     queryWrapper.eq("os", pangolinReportBillMonth.getOs());
                     queryWrapper.eq("app_id", pangolinReportBillMonth.getAppId());
                     queryWrapper.eq("month", pangolinReportBillMonth.getMonth());
                     queryWrapper.eq("year", pangolinReportBillMonth.getYear());
+                    queryWrapper.eq("account_name", pangolinReportBillMonth.getAccountName());
                     if (pangolinReportBillMonthService.getOne(queryWrapper) == null) {
                         pangolinReportBillMonthService.save(pangolinReportBillMonth);
                     } else {
@@ -135,17 +137,17 @@ public class PangolinCrawlerServiceImpl implements PangolinCrawlerService {
                     app.setName(name);
                     pangolinAppService.saveOrUpdateApp(app);
                 }
-                Iterator<JsonNode> channelIterator = resultNode.get("data").get("ChannelNameList").elements();
-                while (channelIterator.hasNext()) {
-                    JsonNode channelNode = channelIterator.next();
-                    Long id = channelNode.get("Id").asLong();
-                    String name = channelNode.get("Name").asText();
-                    PangolinChannel channel = new PangolinChannel();
-                    channel.setAccountName(bindAccountLogin.getAccountName());
-                    channel.setId(id);
-                    channel.setName(name);
-                    pangolinChannelService.saveOrUpdate(channel);
-                }
+//                Iterator<JsonNode> channelIterator = resultNode.get("data").get("ChannelNameList").elements();
+//                while (channelIterator.hasNext()) {
+//                    JsonNode channelNode = channelIterator.next();
+//                    Long id = channelNode.get("Id").asLong();
+//                    String name = channelNode.get("Name").asText();
+//                    PangolinChannel channel = new PangolinChannel();
+//                    channel.setAccountName(bindAccountLogin.getAccountName());
+//                    channel.setId(id);
+//                    channel.setName(name);
+//                    pangolinChannelService.saveOrUpdate(channel);
+//                }
             }
         } catch (IOException e) {
             e.printStackTrace();

+ 0 - 1
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinReportAppHourServiceImpl.java

@@ -24,7 +24,6 @@ public class PangolinReportAppHourServiceImpl extends ServiceImpl<PangolinReport
 
     @Override
     public Map<String, Object> getRealTimeData(String userId, JSONObject data) {
-
         Map<String, Object> resultMap = new HashMap<>();
         String today = DateUtils.formatDate();
         String yesterday = DateUtils.addDay(today,-1);

+ 0 - 2
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/material/service/impl/KuaishouBidWarningServiceImpl.java

@@ -15,7 +15,6 @@ import cn.com.ctop.kuaishou.modules.material.service.IKuaishouBidWarningService;
 import cn.com.feishu.provider.MessageProvider;
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONArray;
-import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.xxl.job.core.log.XxlJobLogger;
 import org.jeecg.common.util.DateUtils;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -45,7 +44,6 @@ public class KuaishouBidWarningServiceImpl implements IKuaishouBidWarningService
         XxlJobLogger.log("快手出价预警定时任务开始");
         long start = System.currentTimeMillis();
         try {
-            QueryWrapper<Project> projectQueryWrapper = new QueryWrapper<>();
             List<Project> projects = projectService.list();
             if (Check.isNull(projects)) {
                 return;

+ 4 - 1
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/material/service/impl/BytedanceBidWarningServiceImpl.java

@@ -56,11 +56,14 @@ public class BytedanceBidWarningServiceImpl implements IBytedanceBidWarningServi
                 }
                 Long maxBid = project.getMaxBid();
                 Long projectId = project.getId();
-                List<UserAllocation> userAllocations = userAllocationService.getByParams(projectId,null,1);
+                List<UserAllocation> userAllocations = userAllocationService.getByParams(projectId,null,null);
                 if (Check.isNull(userAllocations)) {
                     continue;
                 }
                 for (UserAllocation userAllocation : userAllocations) {
+                    if (userAllocation.getAccountStatus() == 1) {
+                        continue;
+                    }
                     Long accountId = userAllocation.getAccountId();
                     List<String> unitWarning = new ArrayList<>();
                     if ("1".equals(userAllocation.getMediaId())) {