Преглед изворни кода

添加头条全量计划获取任务

syh пре 5 година
родитељ
комит
700a86710d

+ 90 - 78
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/TestController.java

@@ -435,84 +435,6 @@ public class TestController {
         return result;
     }
 
-    @GetMapping("formatYddData")
-    public Map<String, Object> formatYddData() throws Exception {
-        Map<String, Object> result = new HashMap<>();
-        //清洗关于缘多多数据
-        //1:查询缘多多相关的用户数据
-        QueryWrapper<UserAllocation> yddWrapper = new QueryWrapper<>();
-        yddWrapper.eq("project_id", 113);
-        List<UserAllocation> yddallocations = userAllocationService.list(yddWrapper);
-        if (null != yddallocations && yddallocations.size() > 0) {
-            yddallocations.forEach(allocation -> {
-                //根据accountId和时间,查询相关的日报表信息
-                QueryWrapper<BytedancePlanDailyReport> queryWrapper = new QueryWrapper<>();
-                queryWrapper.eq("advertiser_id", allocation.getAccountId())
-                //.eq("stat_datetime",dateString)
-                ;
-                List<BytedancePlanDailyReport> bytedancePlanDailyReports = planDailyReportService.list(queryWrapper);
-                if (null != bytedancePlanDailyReports && bytedancePlanDailyReports.size() > 0) {
-                    bytedancePlanDailyReports.forEach(report -> {
-                        String adName = report.getAdName();
-                        String[] tags = adName.split("-");
-                        report.setChannelCode(tags[0]);
-                        planDailyReportService.updateById(report);
-                    });
-                }
-            });
-        }
-
-        //清洗关于缘多多数据
-        //1:查询缘多多相关的用户数据
-        QueryWrapper<UserAllocation> dsjyWrapper = new QueryWrapper<>();
-        yddWrapper.eq("project_id", 112);
-        List<UserAllocation> dsjyAllocations = userAllocationService.list(dsjyWrapper);
-        if (null != dsjyAllocations && dsjyAllocations.size() > 0) {
-            dsjyAllocations.forEach(allocation -> {
-                //根据accountId和时间,查询相关的日报表信息
-                QueryWrapper<BytedancePlanDailyReport> queryWrapper = new QueryWrapper<>();
-                queryWrapper.eq("advertiser_id", allocation.getAccountId())
-                //.eq("stat_datetime",dateString)
-                ;
-                List<BytedancePlanDailyReport> bytedancePlanDailyReports = planDailyReportService.list(queryWrapper);
-                if (null != bytedancePlanDailyReports && bytedancePlanDailyReports.size() > 0) {
-                    bytedancePlanDailyReports.forEach(report -> {
-                        String adName = report.getAdName();
-                        String[] tags = adName.split("-");
-                        report.setChannelCode(tags[0]);
-                        planDailyReportService.updateById(report);
-                    });
-                }
-            });
-        }
-
-        //清洗关于缘多多数据
-        //1:查询缘多多相关的用户数据
-        QueryWrapper<UserAllocation> fjyhWrapper = new QueryWrapper<>();
-        yddWrapper.eq("project_id", 222);
-        List<UserAllocation> fjyhAllocations = userAllocationService.list(fjyhWrapper);
-        if (null != fjyhAllocations && fjyhAllocations.size() > 0) {
-            fjyhAllocations.forEach(allocation -> {
-                //根据accountId和时间,查询相关的日报表信息
-                QueryWrapper<BytedancePlanDailyReport> queryWrapper = new QueryWrapper<>();
-                queryWrapper.eq("advertiser_id", allocation.getAccountId())
-                //.eq("stat_datetime",dateString)
-                ;
-                List<BytedancePlanDailyReport> bytedancePlanDailyReports = planDailyReportService.list(queryWrapper);
-                if (null != bytedancePlanDailyReports && bytedancePlanDailyReports.size() > 0) {
-                    bytedancePlanDailyReports.forEach(report -> {
-                        String adName = report.getAdName();
-                        String[] tags = adName.split("-");
-                        report.setChannelCode(tags[0]);
-                        planDailyReportService.updateById(report);
-                    });
-                }
-            });
-        }
-        ResultMapUtils.setResultMap(result, StatusCode.COMMON_SUCCESS);
-        return result;
-    }
-
     //线程计数器
     static CountDownLatch countDownLatch = null;
 
@@ -562,6 +484,18 @@ public class TestController {
         return result;
     }
 
+    @GetMapping("formatYzData")
+    public Map<String, Object> formatYzData() {
+        Map<String, Object> result = new HashMap<>();
+        formatYzData(151, new Date());
+        ResultMapUtils.setResultMap(result, StatusCode.COMMON_SUCCESS);
+        return result;
+    }
+
+    private void formatYzData(int projectId, Date getDate) {
+        planDailyReportService.cleanYzData(projectId);
+    }
+
     private void formatData(Integer projectId, Date finalGetDate) {
         QueryWrapper<UserAllocation> queryWrapper = new QueryWrapper<>();
         queryWrapper.eq("project_id", projectId);
@@ -591,6 +525,84 @@ public class TestController {
     @Autowired
     private IBytedancePlanHourlyReportService planHourlyReportService;
 
+    @GetMapping("loadCreativeData")
+    public Map<String, Object> loadCreaticeData() {
+        Map<String, Object> result = new HashMap<>();
+        List<CtopOauthToken> tokens = tokenService.getTokenListByType(CtopAdConstant.PLATFORM_TYPE_BYTEDANCE);
+        log.info("头条创意素材数据开始获取");
+        //查询当日数据
+        if (null == tokens || tokens.size() <= 0) {
+            log.info("定时获取头条数据异常:wei获取到可用的token");
+            ResultMapUtils.setResultMap(result, StatusCode.COMMON_SERVER_ERROR);
+            return result;
+        }
+        executorService = Executors.newFixedThreadPool(20);
+        countDownLatch = new CountDownLatch(tokens.size());
+        tokens.forEach(token -> {
+            executorService.submit(new Runnable() {
+                @Override
+                public void run() {
+                    //4: 获取广告创意信息数据
+                    try {
+                        advertiserDataService.getAdvertiserCreative(token, null);
+                    } catch (Exception e) {
+                        e.printStackTrace();
+                    } finally {
+                        countDownLatch.countDown();
+                    }
+                }
+            });
+        });
+        try {
+            countDownLatch.await();
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+        }
+        log.info("头条创意素材数据获取完成");
+        ResultMapUtils.setResultMap(result, StatusCode.COMMON_SUCCESS);
+        return result;
+    }
+
+    @GetMapping("loadCreaticeReportData")
+    public Map<String, Object> loadCreaticeReportData() {
+        Map<String, Object> result = new HashMap<>();
+        List<CtopOauthToken> tokens = tokenService.getTokenListByType(CtopAdConstant.PLATFORM_TYPE_BYTEDANCE);
+        for (int i = 47; i < 180; i++) {
+            Date getDate = DateUtils.addDay(new Date(), -i);
+            log.info(DateUtils.formatDate(getDate) + "头条创意日报数据开始获取");
+            //查询当日数据
+            if (null == tokens || tokens.size() <= 0) {
+                log.info("定时获取头条数据异常:为获取到可用的token");
+                continue;
+            }
+            executorService = Executors.newFixedThreadPool(20);
+            countDownLatch = new CountDownLatch(tokens.size());
+            tokens.forEach(token -> {
+                executorService.submit(new Runnable() {
+                    @Override
+                    public void run() {
+                        //4: 获取广告创意信息数据
+                        try {
+                            reportService.getAdvertiserCreativeReport(token, getDate, getDate, CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
+                        } catch (Exception e) {
+                            e.printStackTrace();
+                        } finally {
+                            countDownLatch.countDown();
+                        }
+                    }
+                });
+            });
+            try {
+                countDownLatch.await();
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+            log.info(DateUtils.formatDate(getDate) + "头条创意日报数据获取完成");
+        }
+        ResultMapUtils.setResultMap(result, StatusCode.COMMON_SUCCESS);
+        return result;
+    }
+
     @GetMapping("loadVideoData")
     public Map<String, Object> loadVideoData() {
         log.info("素材数据获取开始");

+ 10 - 5
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/BytedancePlanDailyReportLoadJob.java

@@ -45,10 +45,18 @@ public class BytedancePlanDailyReportLoadJob implements Job {
             //获取广告计划信息数据
             reportService.getAdvertiserPlanReport(token, getDate, getDate, CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
         });
+        formatZybData(215, getDate);
+        formatYddData(113, getDate);
+        formatYddData(112, getDate);
+        formatYddData(222, getDate);
+        planDailyReportService.cleanYzData(151);
+    }
+
+    private void formatZybData(Integer projectId, Date getDate) {
         //清洗关于作业帮数据
         //1:查询作业帮相关的用户数据
         QueryWrapper<UserAllocation> wrapper = new QueryWrapper<>();
-        wrapper.eq("project_id",215);
+        wrapper.eq("project_id", projectId);
         List<UserAllocation> allocations = userAllocationService.list(wrapper);
         if(null!=allocations&&allocations.size()>0){
             allocations.forEach(allocation->{
@@ -90,12 +98,9 @@ public class BytedancePlanDailyReportLoadJob implements Job {
                 }
             });
         }
-        formatData(113,getDate);
-        formatData(112,getDate);
-        formatData(222,getDate);
     }
 
-    private void formatData(Integer projectId,Date getDate){
+    private void formatYddData(Integer projectId, Date getDate) {
         //清洗关于缘多多数据
         QueryWrapper<UserAllocation> dsjyWrapper = new QueryWrapper<>();
         dsjyWrapper.eq("project_id", projectId);

+ 2 - 1
jeecg-boot-module-system/src/main/java/org/jeecg/modules/mq/Receiver.java

@@ -1,13 +1,14 @@
 package org.jeecg.modules.mq;
 
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.amqp.rabbit.annotation.Queue;
 import org.springframework.amqp.rabbit.annotation.RabbitHandler;
 import org.springframework.amqp.rabbit.annotation.RabbitListener;
 import org.springframework.stereotype.Component;
 
 @Slf4j
 @Component
-@RabbitListener(queues = "hello")
+@RabbitListener(queuesToDeclare = @Queue("hello"))
 public class Receiver {
     /**
      * 通过@RabbitHandler声明的方法,对hello队列中的消息进行处理

+ 12 - 0
module-report/src/main/java/cn/com/ctop/bytedance/mapper/BytedancePlanDailyReportMapper.java

@@ -3,6 +3,7 @@ package cn.com.ctop.bytedance.mapper;
 import cn.com.ctop.bytedance.entity.BytedancePlanDailyReport;
 import com.baomidou.mybatisplus.core.mapper.BaseMapper;
 import org.apache.ibatis.annotations.Param;
+import org.apache.ibatis.annotations.Update;
 
 import java.util.List;
 
@@ -15,4 +16,15 @@ import java.util.List;
 public interface BytedancePlanDailyReportMapper extends BaseMapper<BytedancePlanDailyReport> {
 
     void replaceBatch(@Param(value = "reports") List<BytedancePlanDailyReport> dailyReports);
+
+    @Update("update ctop_bytedance_report_plan_daily " +
+            "set channel_code = replace(substring_index(ad_name, '-', 1), '.apk', '') " +
+            "where advertiser_id in (select account_id from ctop_user_allocation where project_id = #{projectId})")
+    void cleanYzData(@Param(value = "projectId") Integer projectId);
+
+    @Update("update ctop_bytedance_report_plan_daily " +
+            "set channel_code = replace(replace(replace(substring_index(ad_name, '-', 2), '衍生计划-', ''), '.apk',''),'-','') " +
+            "where advertiser_id in (select account_id from ctop_user_allocation where project_id = #{projectId}) " +
+            "  and channel_code = '衍生计划' ")
+    void cleanYzDataFuck(@Param(value = "projectId") Integer projectId);
 }

+ 2 - 0
module-report/src/main/java/cn/com/ctop/bytedance/service/IBytedancePlanDailyReportService.java

@@ -16,4 +16,6 @@ public interface IBytedancePlanDailyReportService extends IService<BytedancePlan
     void loadData(String data, String date);
 
     void replaceBatch(List<BytedancePlanDailyReport> dailyReports);
+
+    void cleanYzData(Integer projectId);
 }

+ 6 - 0
module-report/src/main/java/cn/com/ctop/bytedance/service/impl/BytedancePlanDailyReportServiceImpl.java

@@ -48,4 +48,10 @@ public class BytedancePlanDailyReportServiceImpl extends ServiceImpl<BytedancePl
     public void replaceBatch(List<BytedancePlanDailyReport> dailyReports) {
         dailyReportMapper.replaceBatch(dailyReports);
     }
+
+    @Override
+    public void cleanYzData(Integer projectId) {
+        dailyReportMapper.cleanYzData(projectId);
+        dailyReportMapper.cleanYzDataFuck(projectId);
+    }
 }

+ 0 - 4
module-report/src/main/java/cn/com/ctop/bytedance/service/impl/ReportServiceImpl.java

@@ -327,7 +327,6 @@ public class ReportServiceImpl implements IReportService {
         }
         JSONArray dataArray = getObject.getJSONObject("data").getJSONArray("list");
         if (null == dataArray || dataArray.size() <= 0) {
-            log.info("广告创意报表信息为空=》accountId:{}", token.getAccountId());
             return;
         }
         List<BytedanceCreativeHourlyReport> hourlyReports = new ArrayList<>();
@@ -336,11 +335,9 @@ public class ReportServiceImpl implements IReportService {
             var data = dataArray.getJSONObject(i);
             if (null != conditions.getTimeGranularity() && CtopAdConstant.BYTEDANCE_REPORT_TYPE_HOURLY.equals(conditions.getTimeGranularity())) {
                 var hourlyReport = new BytedanceCreativeHourlyReport(data, token.getAccountId());
-//                creativeHourlyReportService.saveOrUpdate(hourlyReport);
                 hourlyReports.add(hourlyReport);
             } else {
                 var dailyReport = new BytedanceCreativeDailyReport(data, token.getAccountId());
-//                creativeDailyReportService.saveOrUpdate(dailyReport);
                 if (!Check.isNull(dailyReport)) {
                     Long creativeId = dailyReport.getCreativeId();
                     String signature = creativeHourlyReportService.getSignatureByCreativeId(creativeId);
@@ -348,7 +345,6 @@ public class ReportServiceImpl implements IReportService {
                         dailyReport.setSignature(signature);
                     }
                 }
-
                 dailyReports.add(dailyReport);
             }
         }

+ 4 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/mapper/ByteDanceCreativeMapper.java

@@ -2,6 +2,9 @@ package cn.com.ctop.toutiao.mapper;
 
 import cn.com.ctop.toutiao.entity.ByteDanceCreative;
 import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import org.apache.ibatis.annotations.Param;
+
+import java.util.List;
 
 /**
  * @Description: 今日头条创意信息
@@ -11,4 +14,5 @@ import com.baomidou.mybatisplus.core.mapper.BaseMapper;
  */
 public interface ByteDanceCreativeMapper extends BaseMapper<ByteDanceCreative> {
 
+    void replaceBatch(@Param(value = "creatives") List<ByteDanceCreative> creatives);
 }

+ 44 - 1
module-toutiao/src/main/java/cn/com/ctop/toutiao/mapper/xml/ByteDanceCreativeMapper.xml

@@ -1,5 +1,48 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
 <mapper namespace="cn.com.ctop.toutiao.mapper.ByteDanceCreativeMapper">
-
+    <insert id="replaceBatch">
+        replace into ctop_bytedance_creative(
+        id,
+        plan_id,
+        toutiao_id,
+        account_id,
+        title,
+        creative_word_ids,
+        status,
+        opt_status,
+        image_mode,
+        image_ids,
+        image_id,
+        video_id,
+        audit_reject_reason,
+        third_party_id,
+        creative_create_time,
+        creative_modify_time,
+        create_time,
+        update_time)
+        values
+        <foreach collection="creatives" item="report" separator=",">
+            (
+            #{report.id},
+            #{report.planId},
+            #{report.toutiaoId},
+            #{report.accountId},
+            #{report.title},
+            #{report.creativeWordIds},
+            #{report.status},
+            #{report.optStatus},
+            #{report.imageMode},
+            #{report.imageIds},
+            #{report.imageId},
+            #{report.videoId},
+            #{report.auditRejectReason},
+            #{report.thirdPartyId},
+            #{report.creativeCreateTime},
+            #{report.creativeModifyTime},
+            #{report.createTime},
+            #{report.updateTime}
+            )
+        </foreach>
+    </insert>
 </mapper>

+ 2 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/service/IByteDanceAdvertiserDataService.java

@@ -20,6 +20,8 @@ public interface IByteDanceAdvertiserDataService {
 
     Map<String, Object> getAdvertiserCreativeMaterial(String accountId, String creativeIds);
 
+    Map<String, Object> getAdvertiserCreative(CtopOauthToken token, String date);
+
     Map<String, Object> advertiserCampaignUpdateStatus(String accountId, String campaignIds, String optStatus);
 
     Map<String, Object> advertiserCampaignUpdate(String accountId, Long campaignId, String budgetMode, Integer budget, String campaignName);

+ 3 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/service/IByteDanceCreativeService.java

@@ -4,6 +4,7 @@ import cn.com.ctop.toutiao.entity.ByteDanceCreative;
 import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.extension.service.IService;
 
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -23,4 +24,6 @@ public interface IByteDanceCreativeService extends IService<ByteDanceCreative> {
     Map<String, Object> getAdvertiserCreative(String accountId, String ids, String date);
 
     Map<String, Object> advertiserCreativeUpdateStatus(String accountId, String creativeIds, String optStatus);
+
+    void replaceBatch(List<ByteDanceCreative> creatives);
 }

+ 54 - 2
module-toutiao/src/main/java/cn/com/ctop/toutiao/service/impl/ByteDanceAdvertiserDataServiceImpl.java

@@ -4,6 +4,7 @@ import cn.com.ctop.common.module.entity.CtopOauthToken;
 import cn.com.ctop.common.module.service.ICtopOauthTokenService;
 import cn.com.ctop.common.module.utils.*;
 import cn.com.ctop.toutiao.entity.*;
+import cn.com.ctop.toutiao.mapper.ByteDanceCreativeMapper;
 import cn.com.ctop.toutiao.service.*;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
@@ -27,12 +28,12 @@ public class ByteDanceAdvertiserDataServiceImpl implements IByteDanceAdvertiserD
     @Autowired
     private IByteDanceCreativeMaterialService creativeMaterialService;
     @Autowired
+    private ByteDanceCreativeMapper creativeMapper;
+    @Autowired
     private IByteDanceAdvertiserBudgetService advertiserBudgetService;
     @Autowired
     private IByteDanceCampaignService campaignService;
     @Autowired
-    private IByteDanceAdvertiserService advertiserService;
-    @Autowired
     private IByteDanceImageInfoService imageInfoService;
     @Autowired
     private IByteDanceVideoInfoService videoInfoService;
@@ -234,6 +235,56 @@ public class ByteDanceAdvertiserDataServiceImpl implements IByteDanceAdvertiserD
     }
 
     @Override
+    public Map<String, Object> getAdvertiserCreative(CtopOauthToken token, String date) {
+        log.info("开始获取创意信息=>accountId:{}", token.getAccountId());
+        Map<String, Object> result = new HashMap<>();
+        ResultMapUtils.setResultMap(result, StatusCode.COMMON_SUCCESS);
+        getCreativeByPage(token, date, 1);
+        log.info("创意信息获取结束=>accountId:{}", token.getAccountId());
+        return result;
+    }
+
+    private void getCreativeByPage(CtopOauthToken token, String date, int pageNum) {
+        JSONObject resultObject = getCreative(token, date, pageNum);
+        Integer code = resultObject.getInteger("code");
+        if (null == code || !code.equals(0)) {
+            log.error("头条创意素材获取接口异常==》accountId:{},message:{}", token.getAccountId(), resultObject.getString("message"));
+            return;
+        }
+        JSONArray data = resultObject.getJSONObject("data").getJSONArray("list");
+        if (null == data || data.size() <= 0) {
+            return;
+        }
+        List<ByteDanceCreative> creatives = new ArrayList<>();
+        for (int i = 0; i < data.size(); i++) {
+            JSONObject dataObject = data.getJSONObject(i);
+            ByteDanceCreative creative = new ByteDanceCreative(dataObject, token.getAccountId() + "");
+            creatives.add(creative);
+        }
+        creativeMapper.replaceBatch(creatives);
+        getVideoByPage(token, date, pageNum + 1);
+    }
+
+
+    public JSONObject getCreative(CtopOauthToken token, String date, int pageNumber) {
+        final Long advertiser_id = token.getAccountId();  //操作的广告主id
+        // 请求地址
+        String url = "https://ad.oceanengine.com/open_api/2/creative/get/";
+
+        // 请求参数
+        JSONObject param = new JSONObject();
+        param.put("advertiser_id", advertiser_id);
+        param.put("page", pageNumber);
+        param.put("page_size", 100);
+        if (null != date && !date.trim().equals("")) {
+            JSONObject filtering = new JSONObject();
+            filtering.put("creative_create_time", date);
+            param.put("filtering", filtering);
+        }
+        return HttpUtils.bytedanceGetRequest(token.getAccessToken(), url, param);
+    }
+
+    @Override
     public Map<String, Object> advertiserCampaignUpdateStatus(String accountId, String campaignIds, String optStatus) {
         Map<String, Object> resultMap = new HashMap<>();
         JSONArray ids = new JSONArray();
@@ -495,6 +546,7 @@ public class ByteDanceAdvertiserDataServiceImpl implements IByteDanceAdvertiserD
         return resultMap;
     }
 
+
     /**
      * 查询人群包信息
      *

+ 8 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/service/impl/ByteDanceCreativeServiceImpl.java

@@ -27,6 +27,7 @@ import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
 
@@ -47,6 +48,8 @@ public class ByteDanceCreativeServiceImpl extends ServiceImpl<ByteDanceCreativeM
     private IBytedanceAdvertisePlanTemplateService planTemplateService;
     @Autowired
     private ICtopOauthTokenService tokenService;
+    @Autowired
+    private ByteDanceCreativeMapper creativeMapper;
 
     @Override
     public Map<String, Object> advertiserCreativeUpdateStatus(String accountId, String creativeIds, String optStatus) {
@@ -96,6 +99,11 @@ public class ByteDanceCreativeServiceImpl extends ServiceImpl<ByteDanceCreativeM
     }
 
     @Override
+    public void replaceBatch(List<ByteDanceCreative> creatives) {
+        creativeMapper.replaceBatch(creatives);
+    }
+
+    @Override
     public Map<String, Object> getAdvertiserCreative(String accountId, String ids, String date) {
         Map<String, Object> resultMap = new HashMap<>();
         getAdvertiserCreativeByPageNumber(accountId, 1, ids, date);