Ver código fonte

fix of 'v1.1.3' on 2021.02.24

jiequan.bi 4 anos atrás
pai
commit
3b83b7bcf7

+ 49 - 8
jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java

@@ -13,6 +13,7 @@ import cn.com.ctop.kuaishou.modules.ai.service.IAiKuaishouNewCreateCampaign;
 import cn.com.ctop.kuaishou.modules.batch.service.*;
 import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouWebInterfaceService;
 import cn.com.ctop.kuaishou.modules.report.service.IKuaishouReportDailyAgentService;
+import cn.com.ctop.kuaishou.modules.report.service.IKuaishouReportDailyImageService;
 import cn.com.ctop.kuaishou.modules.report.service.IKuaishouVideoEtlInfoService;
 import cn.com.ctop.kuaishou.modules.report.service.IRuleKuaiShouPlanService;
 import cn.com.ctop.oa.modules.service.IWechatCheckinDataService;
@@ -23,6 +24,7 @@ import cn.com.ctop.toutiao.modules.material.service.IByteDanceAdvertiserDataServ
 import cn.com.ctop.toutiao.modules.report.entity.BytedanceAccountReportTaskRecord;
 import cn.com.ctop.toutiao.modules.report.mapper.BytedanceReportMaterialDailyMapper;
 import cn.com.ctop.toutiao.modules.report.service.*;
+import com.alibaba.fastjson.JSONObject;
 import lombok.extern.slf4j.Slf4j;
 import org.jeecg.common.util.DateUtils;
 import org.junit.Test;
@@ -620,10 +622,10 @@ public class SampleTest {
     @Test
     public void etlBytedanceVideoReport() {
 
-//        for (int i = 1; i <= 62; i++) {
-//            etlBytedanceVideo(DateUtils.formatDate(DateUtils.addDay(new Date(), -i)));
-//        }
-        etlBytedanceVideo("2021-02-03");
+        for (int i = 27; i <= 35; i++) {
+            etlBytedanceVideo(DateUtils.formatDate(DateUtils.addDay(new Date(), -i)));
+        }
+        //etlBytedanceVideo("2021-02-03");
 
     }
 
@@ -633,7 +635,9 @@ public class SampleTest {
         countDownLatch = new CountDownLatch(tokens.size());
         tokens.forEach(token -> executorService.submit(() -> {
             try {
-                bytedanceVideoReportService.pullVideoReport(token, statDate, statDate, 1);
+                Long[] materials={6904990807503634445L,6904990807503634445L};
+                bytedanceVideoReportService.pullVideoReportByMaterialId(token, statDate, statDate, materials,1);
+                //bytedanceVideoReportService.pullVideoReport(token, statDate, statDate, 1);
             } catch (Exception e) {
 
             } finally {
@@ -660,11 +664,48 @@ public class SampleTest {
 
     @Test
     public void etlBytedanceVideoReportClean1() {
-        //executorService = Executors.newFixedThreadPool(31);
-        //countDownLatch = new CountDownLatch(31);
+
         for (int i = 3; i <= 31; i++) {
             String statDate = DateUtils.formatDate(DateUtils.addDay(new Date(), -i));
             ietlReportBytedanceVideoService.cleanETL(statDate);
         }
     }
-}
+
+
+    @Autowired
+    IKuaishouReportDailyImageService kuaishouReportDailyImageService;
+
+    @Test
+    public void loadKuaiShouImage() {
+        List<CtopOauthToken> tokens = oauthTokenService.selectKuaiShouToken();
+        executorService = Executors.newFixedThreadPool(22);
+        countDownLatch = new CountDownLatch(tokens.size());
+        tokens.forEach(token -> executorService.submit(() -> {
+            try {
+                kuaishouReportDailyImageService.getMaterialImageReport(token.getAccountId(),token.getAccessToken(), "2021-01-17", "2021-01-31", 1);
+            } catch (Exception e) {
+
+            } finally {
+                countDownLatch.countDown();
+            }
+        }));
+        try {
+            countDownLatch.await();
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+        }
+    }
+
+    @Autowired
+    IBytedanceMaterialReportAsync bytedanceMaterialReportAsync;
+
+    @Test
+    public void testAsyncMaterialReport() {
+        List<CtopOauthToken> tokens = oauthTokenService.selectToutiaoToken();
+        tokens.forEach(it->{
+            JSONObject createResult = bytedanceMaterialReportAsync.asyncTaskCreate(it, "2021-02-23", "2021-02-23");
+            System.out.println(createResult.toJSONString());
+        });
+    }
+
+    }

+ 1 - 1
module-job-kuaishou/src/main/java/cn/com/ctop/job/kuaishou/handler/KuaiShouDailyImageReportLoadJob.java

@@ -19,7 +19,7 @@ public class KuaiShouDailyImageReportLoadJob {
     private ICtopOauthTokenService tokenService;
     @Autowired
     private IKuaishouReportDailyImageService kuaishouReportDailyImageService;
-    static ExecutorService executorService = Executors.newFixedThreadPool(8);
+    private static ExecutorService executorService = Executors.newFixedThreadPool(8);
     @XxlJob("kuaiShouDailyImageReportLoadJob")
     public void execute() throws Exception {
         //增加接口的开始和结束日期

+ 56 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/controller/KsNewGroupReportCtrl.java

@@ -1,10 +1,13 @@
 package cn.com.ctop.kuaishou.modules.report.controller;
 
+import cn.com.ctop.common.module.utils.ExportExcelUtils;
+import cn.com.ctop.common.module.utils.HttpUtils;
 import cn.com.ctop.kuaishou.modules.report.service.IKsNewGroupReportService;
 import cn.com.ctop.kuaishou.modules.report.service.IKsNewPlanReportService;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
 import org.jeecg.common.api.vo.Result;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.PostMapping;
@@ -12,6 +15,11 @@ import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
 
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -93,7 +101,55 @@ public class KsNewGroupReportCtrl {
         }
         return result;
     }
+    @PostMapping("/report/excel")
+    public void getExcelReport(@RequestBody JSONObject requestBody,
+                               HttpServletRequest request,
+                               HttpServletResponse response) {
 
+        JSONArray accountIds = requestBody.getJSONArray("accountIds");
+        String userId = requestBody.getString("userId");
+        //TODO 没有维护快手事业部负责人角色,增加判断
+        if (userId.equals("6b4c821adc414dc8b7718ab63ccbfcaf")) {
+            accountIds = newPlanReportService.getAccountIdsByMedia();
+        } else {
+            if (!userId.isEmpty()) {
+                accountIds = newPlanReportService.getAccountIdsByUserId(userId);
+            }
+            if (accountIds.isEmpty()) {
+                return;
+            }
+        }
+        String startDate = requestBody.getString("startDate");
+        String endDate = requestBody.getString("endDate");
+        String target = requestBody.getString("target");
+        String order = requestBody.getString("order");
+        List<JSONObject> listData = newGroupReportService.getKsNewGroupGroupByAccount(accountIds, startDate, endDate, target, order);
+
+        List<List<Object>> excelList = new ArrayList<>();
+        listData.forEach(data -> {
+            List<Object> temp = new ArrayList<>();
+            temp.add(data.getString("authName"));
+            temp.add(data.getLong("accountId"));
+            temp.add(data.getString("userName"));
+            temp.add(data.getLong("new"));
+            temp.add(data.getLong("valid"));
+            temp.add(data.getString("maxCreateTime"));
+            excelList.add(temp);
+        });
+        try {
+            String[] headers = {"账户名称","账户ID","账户所属人","新上推广组","有效推广组","最近上新时间"};
+            OutputStream os = response.getOutputStream();
+            ExportExcelUtils eeu = new ExportExcelUtils();
+            XSSFWorkbook workbook = new XSSFWorkbook();
+            eeu.exportExcel(workbook, 0, "快手新上推广组", headers, excelList);
+            HttpUtils.setResponseHeader(response, "快手新上推广组.xlsx");
+            workbook.write(os);
+            os.flush();
+            os.close();
+        } catch (IOException e) {
+            log.error(e.getMessage());
+        }
+    }
     /**
      * 查询所有的快手运营
      */

+ 59 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/controller/KsNewPlanReportCtrl.java

@@ -1,16 +1,26 @@
 package cn.com.ctop.kuaishou.modules.report.controller;
 
+import cn.com.ctop.common.module.utils.ExportExcelUtils;
+import cn.com.ctop.common.module.utils.HttpUtils;
 import cn.com.ctop.kuaishou.modules.report.service.IKsNewPlanReportService;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
 import org.jeecg.common.api.vo.Result;
+import org.jeecg.common.constant.AccountReportConstants;
+import org.jeecg.common.util.JsonResourceUtil;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
 
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -92,6 +102,55 @@ public class KsNewPlanReportCtrl {
         return result;
     }
 
+    @PostMapping("/report/excel")
+    public void getExcelReport(@RequestBody JSONObject requestBody,
+                               HttpServletRequest request,
+                               HttpServletResponse response) {
+
+        JSONArray accountIds = requestBody.getJSONArray("accountIds");
+        String userId = requestBody.getString("userId");
+        //TODO 没有维护快手事业部负责人角色,增加判断
+        if (userId.equals("6b4c821adc414dc8b7718ab63ccbfcaf")) {
+            accountIds = newPlanReportService.getAccountIdsByMedia();
+        } else {
+            if (!userId.isEmpty()) {
+                accountIds = newPlanReportService.getAccountIdsByUserId(userId);
+            }
+            if (accountIds.isEmpty()) {
+                return;
+            }
+        }
+        String startDate = requestBody.getString("startDate");
+        String endDate = requestBody.getString("endDate");
+        String target = requestBody.getString("target");
+        String order = requestBody.getString("order");
+        List<JSONObject> listData = newPlanReportService.getKsNewPlanGroupByAccount(accountIds, startDate, endDate, target, order);
+
+        List<List<Object>> excelList = new ArrayList<>();
+        listData.forEach(data -> {
+            List<Object> temp = new ArrayList<>();
+            temp.add(data.getString("authName"));
+            temp.add(data.getLong("accountId"));
+            temp.add(data.getString("userName"));
+            temp.add(data.getLong("new"));
+            temp.add(data.getLong("valid"));
+            temp.add(data.getString("maxCreateTime"));
+            excelList.add(temp);
+        });
+        try {
+            String[] headers = {"账户名称","账户ID","账户所属人","新上计划数","有效计划数","最近上新时间"};
+            OutputStream os = response.getOutputStream();
+            ExportExcelUtils eeu = new ExportExcelUtils();
+            XSSFWorkbook workbook = new XSSFWorkbook();
+            eeu.exportExcel(workbook, 0, "快手新上计划数", headers, excelList);
+            HttpUtils.setResponseHeader(response, "快手新上计划数.xlsx");
+            workbook.write(os);
+            os.flush();
+            os.close();
+        } catch (IOException e) {
+            log.error(e.getMessage());
+        }
+    }
     /**
      * 查询所有的快手运营
      */

+ 2 - 2
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/xml/KsNewPlanReportMapper.xml

@@ -267,7 +267,7 @@
                 )
     </select>
 
-    <select id="queryAccountIdsByUserId" resultType="java.lang.Long">
+    <select id="queryAccountIdsByUserId" resultType="long">
         SELECT
             account_id
         FROM
@@ -277,7 +277,7 @@
         and account_status=0
     </select>
 
-    <select id="queryAccountIdsByMedia" resultType="java.lang.Long">
+    <select id="queryAccountIdsByMedia" resultType="long">
         SELECT
             account_id
         FROM

+ 48 - 2
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/xml/MaterialReportMapper.xml

@@ -71,6 +71,14 @@
         ELSE 0
         END
         ) conversionPrice,
+
+        (
+        CASE
+        WHEN sum(CEIL(t2.play_3s_ratio*t2.photo_show)) / sum(t2.photo_show) IS NULL THEN 0
+        ELSE sum(CEIL(t2.play_3s_ratio*t2.photo_show)) / sum(t2.photo_show)
+        END
+        ) play3sRate,
+
         t2.photo_url as  'url',
         t2.cover_url as  'coverUrl',
         t2.signature signature
@@ -157,7 +165,15 @@
         THEN (sum(charge) / sum(activation))
         ELSE 0
         END
-        ) conversionPrice
+        ) conversionPrice,
+
+        (
+        CASE
+        WHEN sum(CEIL(play_3s_ratio*photo_show)) / sum(photo_show) IS NULL THEN 0
+        ELSE sum(CEIL(play_3s_ratio*photo_show)) / sum(photo_show)
+        END
+        ) play3sRate
+
         from
         ctop_kuaishou_report_hourly_creative_statistic
 
@@ -242,6 +258,14 @@
         ELSE 0
         END
         ) conversionPrice,
+
+        (
+        CASE
+        WHEN sum(CEIL(t2.play_3s_ratio*t2.photo_show)) / sum(t2.photo_show) IS NULL THEN 0
+        ELSE sum(CEIL(t2.play_3s_ratio*t2.photo_show)) / sum(t2.photo_show)
+        END
+        ) play3sRate,
+
         t2.photo_url as  'url',
         t2.cover_url as  'coverUrl',
         t2.signature signature
@@ -330,7 +354,15 @@
         THEN (sum(charge) / sum(activation))
         ELSE 0
         END
-        ) conversionPrice
+        ) conversionPrice,
+
+        (
+        CASE
+        WHEN sum(CEIL(play_3s_ratio*photo_show)) / sum(photo_show) IS NULL THEN 0
+        ELSE sum(CEIL(play_3s_ratio*photo_show)) / sum(photo_show)
+        END
+        ) play3sRate
+
         from
         ctop_kuaishou_report_daily_material
         <where>
@@ -416,6 +448,13 @@
         END
         ) conversionPrice,
 
+        (
+        CASE
+        WHEN sum(CEIL(t2.play_3s_ratio*t2.photo_show)) / sum(t2.photo_show) IS NULL THEN 0
+        ELSE sum(CEIL(t2.play_3s_ratio*t2.photo_show)) / sum(t2.photo_show)
+        END
+        ) play3sRate,
+
         t2.url url,
         t2.signature signature,
         t2.account_id accountId
@@ -505,6 +544,13 @@
         END
         ) conversionPrice,
 
+        (
+        CASE
+        WHEN sum(CEIL(t1.play_3s_ratio*t1.photo_show)) / sum(t1.photo_show) IS NULL THEN 0
+        ELSE sum(CEIL(t1.play_3s_ratio*t1.photo_show)) / sum(t1.photo_show)
+        END
+        ) play3sRate,
+
         t1.photo_url  as  'url',
         t1.cover_url as  'coverUrl',
         t1.signature signature,

+ 2 - 1
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/service/impl/KuaishouReportDailyImageServiceImpl.java

@@ -47,7 +47,7 @@ public class KuaishouReportDailyImageServiceImpl extends ServiceImpl<KuaishouRep
         param.put("end_date",endDate);
         param.put("view_type",7);
         param.put("temporal_granularity","DAILY");
-        param.put("page_size", 1000);
+        param.put("page_size", 500);
         param.put("page", page);
 
         JSONObject resultJson = JSONObject.parseObject(HttpUtils.kuaiShouhttpPostRequest("https://ad.e.kuaishou.com/rest/openapi/v1/report/material_report", param.toJSONString(), headers));
@@ -58,6 +58,7 @@ public class KuaishouReportDailyImageServiceImpl extends ServiceImpl<KuaishouRep
 
         JSONArray details = resultJson.getJSONArray("details");
         if (null == details || details.size() <= 0) {
+            log.info(resultJson.toJSONString());
             return;
         }
         List<KuaishouReportDailyImage> imageReports = new ArrayList<>();

+ 14 - 4
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/service/impl/MaterialReportServiceImpl.java

@@ -133,7 +133,6 @@ public class MaterialReportServiceImpl implements IMaterialReportService {
     @Override
     public JSONObject getKuaiShouChainRatio(JSONObject json) {
         JSONObject returnJson = new JSONObject();
-        long startTime = System.currentTimeMillis();
         try {
             if (Check.isNull(json)) {
                 return null;
@@ -372,7 +371,7 @@ public class MaterialReportServiceImpl implements IMaterialReportService {
                 } else {
                     chainRatioJson.put("impression1kCostProportion", 0);
                 }
-                // 转化
+                // 转化
                 Long contrastConversions = contrastJson.getLong("conversions");
                 Long beContrastConversions = beContrastJson.getLong("conversions");
                 if (!Check.isNull(contrastConversions) && !Check.isNull(beContrastConversions)) {
@@ -398,10 +397,21 @@ public class MaterialReportServiceImpl implements IMaterialReportService {
                 } else {
                     chainRatioJson.put("conversionPriceProportion", 0);
                 }
+                // 3s播放率
+                BigDecimal contrastPlay3sRate = contrastJson.getBigDecimal("play3sRate");
+                BigDecimal beContrastPlay3sRate = beContrastJson.getBigDecimal("play3sRate");
+                if (!Check.isNull(contrastPlay3sRate) && !Check.isNull(beContrastPlay3sRate)) {
+                    if (beContrastPlay3sRate.compareTo(new BigDecimal(0)) != 0) {
+                        BigDecimal conversionPlay3sRate = (contrastPlay3sRate.subtract(beContrastPlay3sRate)).divide(beContrastPlay3sRate, 2, RoundingMode.HALF_UP);
+                        chainRatioJson.put("conversionPlay3sRate", conversionPlay3sRate);
+                    } else {
+                        chainRatioJson.put("conversionPlay3sRate", 0);
+                    }
+                } else {
+                    chainRatioJson.put("conversionPlay3sRate", 0);
+                }
                 returnJson.put("chainRatio", chainRatioJson);
             }
-
-            log.info("素材数据获取环比请求所用时长------------->: {} ms", System.currentTimeMillis() - startTime);
             return returnJson;
         } catch (Exception e) {
             e.printStackTrace();

+ 4 - 0
module-kuaishou/src/main/resources/json_template/kuaishou_filed_mapping.json

@@ -194,5 +194,9 @@
   "submitRate": {
     "filed": "CASE WHEN sum(t1.submit) / sum(t1.bclick) IS NULL THEN 0 ELSE concat(round(sum(t1.submit) / sum(t1.bclick)*100,2),'%') end as submitRate ",
     "comment": "按钮点击率"
+  },
+  "play3sRate": {
+    "filed": "CASE WHEN sum(CEIL(t1.play_3s_ratio*t1.photo_show)) / sum(t1.photo_show) IS NULL THEN 0 ELSE concat(round(sum(CEIL(t1.play_3s_ratio*t1.photo_show)) / sum(t1.photo_show)*100,2),'%') end as play3sRate ",
+    "comment": "3s播放率"
   }
 }

+ 3 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/IBytedanceVideoReportService.java

@@ -16,4 +16,7 @@ public interface IBytedanceVideoReportService {
     List<JSONObject> exportExcelReportBy(JSONArray accountIds, String startDate, String endDate, String filed, String target, String order, String clip, String shot, String plan);
 
     void pullVideoReport(CtopOauthToken oauthToken,String startDate, String endDate,int page);
+
+    void pullVideoReportByMaterialId(CtopOauthToken oauthToken, String startDate, String endDate,Long[] materialIds, int page);
+
 }

+ 56 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/BytedanceVideoReportServiceImpl.java

@@ -101,4 +101,60 @@ public class BytedanceVideoReportServiceImpl implements IBytedanceVideoReportSer
         }
     }
 
+    @Override
+    public void pullVideoReportByMaterialId(CtopOauthToken oauthToken, String startDate, String endDate,Long[] materialIds, int page) {
+        Map<String, String> headers = new HashMap<>();
+        headers.put("Content-Type", " application/json");
+        headers.put("Access-Token", oauthToken.getAccessToken());
+
+        JSONObject param = new JSONObject();
+        param.put("advertiser_id", oauthToken.getAccountId());
+        param.put("start_date",startDate);
+        param.put("end_date",endDate);
+        JSONObject filtering = new JSONObject();
+        filtering.put("material_id",materialIds);
+        param.put("filtering",filtering);
+        param.put("page_size", 200);
+        param.put("page", page);
+
+        JSONObject resultJson = JSONObject.parseObject(HttpUtils.httpGetRequest("https://ad.oceanengine.com/open_api/2/report/video/get/", headers, param));
+        if (Check.isNull(resultJson)) {
+            log.warn("拉取头条视频报表返回结果为空,advertiserId:{}", oauthToken.getAccountId());
+            return;
+        }
+        if(resultJson.getInteger("code")!=0){
+            log.error("拉取头条视频表异常,advertiserId:{},message:{}", oauthToken.getAccountId(),resultJson.getString("message"));
+            return;
+        }
+        JSONObject data = resultJson.getJSONObject("data");
+        if (null == data || data.size() <= 0) {
+            log.error("拉取头条视频表异常,data is null");
+            return;
+        }
+
+        JSONArray list= data.getJSONArray("list");
+        if(list.isEmpty()){
+            log.warn("拉取头条视频表,数据为空advertiserId:{}", oauthToken.getAccountId());
+            return;
+        }
+        List<BaseReportBytedanceVideoDaily> batch =new ArrayList<>();
+        list.forEach(videoReport->{
+            JSONObject videoReportJSONObject= (JSONObject)videoReport;
+            Long materialId= videoReportJSONObject.getJSONObject("dimensions").getLong("material_id");
+            JSONObject metrics= videoReportJSONObject.getJSONObject("metrics");
+            BaseReportBytedanceVideoDaily temp= JSONObject.toJavaObject(metrics,BaseReportBytedanceVideoDaily.class);
+            temp.setMaterialShow(metrics.getLong("show"));
+            temp.setConvertMaterial(metrics.getLong("convert"));
+            temp.setAccountId(oauthToken.getAccountId());
+            temp.setMaterialId(materialId);
+            temp.setStatDatetime(startDate);
+            batch.add(temp);
+        });
+        baseReportBytedanceVideoDailyService.saveBatch(batch);
+
+        if(resultJson.getJSONObject("data").getJSONObject("page_info").getInteger("total_page")>page){
+            pullVideoReport(oauthToken, startDate, endDate, page + 1);
+        }
+    }
+
 }