浏览代码

素材报表改为 自定义报表

zhaoxian 2 年之前
父节点
当前提交
46f8a88e3a

+ 1 - 1
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/constant/BytedanceConstant.java

@@ -28,7 +28,7 @@ public class BytedanceConstant {
 
     public static final String BYTEDANCE_V2_REPORT_INTEGRATED_GET = "/2/report/integrated/get/";
 
-    public static final String BYTEDANCE_V3_REPORT_INTEGRATED_GET = "/v3.0/report/material/get/";
+    public static final String BYTEDANCE_V3_REPORT_INTEGRATED_GET = "/v3.0/report/custom/get/";
 
     public static final String BYTEDANCE_V3_PROMOTION_LIST_GET = "/v3.0/promotion/list/";
 }

+ 6 - 6
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/entity/ByteDanceNewMaterialReport.java

@@ -16,7 +16,7 @@ public class ByteDanceNewMaterialReport implements Serializable {
     //private Integer promotionId;
     //private Integer projectId;
     private Integer showCnt;
-    private Integer cpmPlatform;
+    private BigDecimal cpmPlatform;
     private Integer clickCnt;
     private BigDecimal ctr;
     private BigDecimal cpcPlatform;
@@ -170,14 +170,14 @@ public class ByteDanceNewMaterialReport implements Serializable {
         stringBuilder.append("\"attribution_game_in_app_roi_7days\"").append(",");
         stringBuilder.append("\"attribution_game_in_app_roi_8days\"").append(",");
 
-        //stringBuilder.append("\"attribution_day_active_pay_count\"");
-        //stringBuilder.append("\"attribution_day_active_pay_cost\"");
-//        stringBuilder.append("\"attribution_day_active_pay_rate\"");
+//        stringBuilder.append("\"attribution_day_active_pay_count\"").append(",");
+//        stringBuilder.append("\"attribution_day_active_pay_cost\"").append(",");
+//        stringBuilder.append("\"attribution_day_active_pay_rate\"").append(",");
 
         stringBuilder.append("\"active_pay_intra_day_count\"").append(",");
         stringBuilder.append("\"active_pay_intra_day_cost\"").append(",");
-        stringBuilder.append("\"active_pay_intra_day_rate\"").append(",");
-        stringBuilder.append("\"delivery_mode\"");
+        stringBuilder.append("\"active_pay_intra_day_rate\"");
+//        stringBuilder.append("\"delivery_mode\"");
 
         stringBuilder.append("]");
         return stringBuilder.toString();

+ 2 - 0
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/mapper/OauthTokenMapper.java

@@ -16,4 +16,6 @@ public interface OauthTokenMapper extends BaseMapper<OauthToken> {
   String getByAccountId(@Param("accountId") Long accountId);
 
     List<Long> getAgentByTouTiao();
+
+  List<Long> getAllToutiaoAccountIds();
 }

+ 4 - 0
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/mapper/xml/OauthTokenMapper.xml

@@ -9,4 +9,8 @@
     <select id="getAgentByTouTiao" resultType="Long">
        SELECT agent_id  from  `jeecg-boot`.ctop_cwjs_agent_info WHERE status = 0 and enabled=0
     </select>
+
+    <select id="getAllToutiaoAccountIds" resultType="Long">
+        SELECT account_id FROM `jeecg-boot`.ctop_user_allocation WHERE media_id = 1 and account_status = 0
+    </select>
 </mapper>

+ 2 - 0
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/IOauthTokenService.java

@@ -16,4 +16,6 @@ public interface IOauthTokenService extends IService<OauthToken> {
   String getByAccountId(Long accountId);
 
   List<Long> getAgentByTouTiao();
+
+  List<Long> getAllToutiaoAccountIds();
 }

+ 109 - 85
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/impl/BytedanceMaterialReportDailyServiceImpl.java

@@ -7,21 +7,25 @@ import cn.com.ctop.job.bytedance.data.entity.OauthToken;
 import cn.com.ctop.job.bytedance.data.mapper.BytedanceMaterialReportDailyMapper;
 import cn.com.ctop.job.bytedance.data.mapper.BytedanceNewMaterialReportMapper;
 import cn.com.ctop.job.bytedance.data.service.IBytedanceMaterialReportDailyService;
+import cn.com.ctop.job.bytedance.data.utils.Check;
 import cn.com.ctop.job.bytedance.data.utils.DateUtils;
 import cn.com.ctop.job.bytedance.data.utils.HttpUtils;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
 import javax.annotation.Resource;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 /**
  * 图片与视频素材日报
@@ -41,9 +45,11 @@ public class BytedanceMaterialReportDailyServiceImpl implements IBytedanceMateri
     @Resource
     private BytedanceNewMaterialReportMapper bytedanceNewMaterialReportMapper;
 
+    private static final ObjectMapper mapper = new ObjectMapper();
+
     @Override
     public void bytedanceDailyMaterialReportJob(OauthToken oauthToken, String yesterda, String yesterda1, int page, int pageSize) {
-        log.info("开始获取图片和视频素材日报 accountID:{},page:{} date:{}", oauthToken.getAccountId(), page,yesterda);
+        log.info("开始获取图片和视频素材日报 accountID:{},page:{} date:{}", oauthToken.getAccountId(), page, yesterda);
         // 请求地址
         String url = bytedanceApiUrl + BytedanceConstant.BYTEDANCE_V2_REPORT_INTEGRATED_GET;
         JSONObject param = new JSONObject();
@@ -76,15 +82,15 @@ public class BytedanceMaterialReportDailyServiceImpl implements IBytedanceMateri
                 report.setImageMode((String) dimensions.get("image_mode"));
 
                 //过滤 imageMode 为 MATERIAL_IMAGE_MODE_TITLE
-                if (report.getImageMode()==null || StringUtils.equals("MATERIAL_IMAGE_MODE_TITLE",report.getImageMode())){
+                if (report.getImageMode() == null || StringUtils.equals("MATERIAL_IMAGE_MODE_TITLE", report.getImageMode())) {
                     continue;
                 }
 
                 report.setInventory((String) dimensions.get("inventory"));
-                if (report.getInventory()==null){
+                if (report.getInventory() == null) {
                     continue;
                 }
-                report.setStatDatetime(dimensions.getString("stat_datetime")==null?null:dimensions.getString("stat_datetime").substring(0,10).replace("-",""));
+                report.setStatDatetime(dimensions.getString("stat_datetime") == null ? null : dimensions.getString("stat_datetime").substring(0, 10).replace("-", ""));
                 report.setSubmitCertificationCount(dataObject.get("submit_certification_count") == null ? null : Integer.parseInt((String) dataObject.get("submit_certification_count")));
                 report.setApprovalCount(dataObject.get("approval_count") == null ? null : Integer.parseInt((String) dataObject.get("approval_count")));
                 report.setFirstOrderCount(dataObject.get("first_order_count") == null ? null : Integer.parseInt((String) dataObject.get("first_order_count")));
@@ -98,7 +104,7 @@ public class BytedanceMaterialReportDailyServiceImpl implements IBytedanceMateri
         }
         try {
             bytedanceMaterialReportDailyMapper.insert(list);
-        }catch (Exception e){
+        } catch (Exception e) {
             log.info(e.toString());
         }
 
@@ -110,127 +116,123 @@ public class BytedanceMaterialReportDailyServiceImpl implements IBytedanceMateri
     }
 
 
-
     //头条-体验版素材日报--视频
     @Override
     public void bytedanceDailyMaterialReportJobV3Video(OauthToken oauthToken, String yesterda, String yesterda1, int page, int pageSize) throws Exception {
-        log.info("开始获取体验版-视频-素材日报 accountID:{},page:{} date:{}", oauthToken.getAccountId(), page,yesterda);
+        log.info("开始获取体验版-视频-素材日报 accountID:{},page:{} date:{}", oauthToken.getAccountId(), page, yesterda);
         // 请求地址
         String url = bytedanceApiUrl + BytedanceConstant.BYTEDANCE_V3_REPORT_INTEGRATED_GET;
-        Map<String,Object> param = new HashMap<>();
-        param.put("advertiser_id",oauthToken.getAccountId());
-        param.put("start_date",yesterda);
-        param.put("end_date",yesterda1);
-        param.put("page",page);
-        param.put("page_size",pageSize);
-        //List<String> fields = Arrays.asList("stat_cost");
-        //String fields1 = "[\"stat_cost\",\"show_cnt\",\"image_mode\"]";
-        // fields 入参 为拼接字符串数组形式 与入参 video_image_mode数组 && image_image_mode数组 格式不同
-        String fields = ByteDanceNewMaterialReport.getParamsFields();
-        param.put("fields",fields);
-        param.put("time_granularity","STAT_TIME_GRANULARITY_DAILY");
-        //param.put("group_by","STAT_GROUP_BY_DELIVERY_MODE");
-
-        Map<String,String> header = new HashMap<>();
-        header.put("Access-Token",oauthToken.getAccessToken());
-
-        //String video_image_mode = "[\"CREATIVE_IMAGE_MODE_VIDEO\",\"CREATIVE_IMAGE_MODE_VIDEO_VERTICAL\"]";
-        String[] video_image_mode = new String[]{"CREATIVE_IMAGE_MODE_VIDEO","CREATIVE_IMAGE_MODE_VIDEO_VERTICAL"};
-        //String image_image_mode = "[\"CREATIVE_IMAGE_MODE_LARGE\",\"CREATIVE_IMAGE_MODE_LARGE_VERTICAL\",\"CREATIVE_IMAGE_MODE_SMALL\"]";
-        String[] image_image_mode = new String[]{"CREATIVE_IMAGE_MODE_LARGE","CREATIVE_IMAGE_MODE_LARGE_VERTICAL","CREATIVE_IMAGE_MODE_SMALL"};
-
-        JSONObject filtering = new JSONObject();
-        Map<String,Object> video_material = new HashMap<>();
-        video_material.put("image_mode",video_image_mode);
-
-        Map<String,Object> image_material = new HashMap<>();
-        image_material.put("image_mode",image_image_mode);
-
-        filtering.put("video_material",video_material);
-        //filtering.put("image_material",image_material);
-
-        param.put("filtering",filtering);
-        JSONObject resultObject = JSONObject.parseObject(HttpUtils.httpGet(url, param,header));
+        Map<String, String> header = new HashMap<>();
+        header.put("Access-Token", oauthToken.getAccessToken());
 
+        Map<String, Object> param = new HashMap<>();
+        param.put("advertiser_id", oauthToken.getAccountId());
+        param.put("start_time", yesterda);
+        param.put("end_time", yesterda1);
+        param.put("page", page);
+        param.put("page_size", pageSize);
+        //维度:每日数据
+        List<String> dimensions = new ArrayList<>();
+        dimensions.add("material_id");
+        dimensions.add("stat_time_day");
+        dimensions.add("image_mode");
+        param.put("dimensions", mapper.writeValueAsString(dimensions));
+
+        //指标列表
+        param.put("metrics", ByteDanceNewMaterialReport.getParamsFields());
+        List<JSONObject> filters = new ArrayList<>();
+        JSONObject filter = new JSONObject();
+        filter.put("field", "stat_cost");
+        filter.put("type", 3);
+        filter.put("operator", 4);
+        filter.put("values", Arrays.asList("0"));
+        filters.add(filter);
+        param.put("filters", mapper.writeValueAsString(filters));
+
+        List<JSONObject> orderBy = new ArrayList<>();
+        JSONObject ord = new JSONObject();
+        ord.put("field", "material_id");
+        ord.put("type", "ASC");
+        orderBy.add(ord);
+        param.put("order_by", mapper.writeValueAsString(orderBy));
+        String result = HttpUtils.httpGet(url, param, header);
+        JSONObject resultObject = JSONObject.parseObject(result);
         Integer code = resultObject.getInteger("code");
         if (null == code || !code.equals(0)) {
             log.error("获取体验版-视频-素材日报接口异常==》accountId:{},message:{}", oauthToken.getAccountId(), resultObject.getString("message"));
             return;
         }
-        JSONArray data = resultObject.getJSONObject("data").getJSONArray("list");
+        JSONArray data = resultObject.getJSONObject("data").getJSONArray("rows");
         if (null == data || data.isEmpty()) {
             log.error("获取体验版-视频-素材数据为空==》accountId:{}", oauthToken.getAccountId());
             return;
         }
-        List<ByteDanceNewMaterialReport> list = new ArrayList<>();
-
+        List<ByteDanceNewMaterialReport> videolist = new ArrayList<>();
+        List<ByteDanceNewMaterialReport> imageList = new ArrayList<>();
         for (int i = 0; i < data.size(); i++) {
             JSONObject dataObject = data.getJSONObject(i);
-            Long materialId = dataObject.getLong("material_id");
-            String statDatetime = dataObject.getString("stat_datetime");
-            Integer dateTime = DateUtils.getDateInteger(statDatetime);
-            JSONArray data1 = dataObject.getJSONArray("data");
-            for (Object o : data1) {
-                JSONObject j = JSONObject.parseObject(o.toString());
-                ByteDanceNewMaterialReport report = JSONObject.parseObject(j.toJSONString(), ByteDanceNewMaterialReport.class);
-                report.setAdvertiserId(oauthToken.getAccountId());
-                report.setMaterialId(materialId);
-                report.setStatDatetime(dateTime);
-                list.add(report);
+            JSONObject dim = dataObject.getJSONObject("dimensions");
+            ByteDanceNewMaterialReport report = JSONObject.parseObject(dataObject.getJSONObject("metrics").toJSONString(), ByteDanceNewMaterialReport.class);
+            report.setAdvertiserId(oauthToken.getAccountId());
+            report.setMaterialId(dim.getLong("material_id"));
+            report.setStatDatetime(DateUtils.getDateInteger(dim.getString("stat_time_day")));
+            if (!Check.isNull(dim.getString("image_mode")) && dim.getString("image_mode").contains("视频")) {
+                videolist.add(report);
+            }
+            if (!Check.isNull(dim.getString("image_mode")) && dim.getString("image_mode").contains("图")) {
+                imageList.add(report);
             }
         }
-
-        bytedanceNewMaterialReportMapper.replaceBytedanceNewMaterialVideoDaily(list);
-
+        if (!Check.isNull(videolist)) {
+            bytedanceNewMaterialReportMapper.replaceBytedanceNewMaterialVideoDaily(videolist);
+        }
+        if (!Check.isNull(imageList)) {
+            bytedanceNewMaterialReportMapper.replaceBytedanceNewMaterialImageDaily(imageList);
+        }
         page++;
         int totalPage = resultObject.getJSONObject("data").getJSONObject("page_info").getInteger("total_page");
         if (page <= totalPage) {
             bytedanceDailyMaterialReportJobV3Video(oauthToken, yesterda, yesterda1, page, 100);
         }
-        log.error("获取体验版-视频-素材数据完成==》accountId:{}", oauthToken.getAccountId());
-
+        log.info("获取体验版-视频-素材数据完成==》accountId:{}", oauthToken.getAccountId());
     }
 
     //头条-体验版素材日报--图片
     @Override
     public void bytedanceDailyMaterialReportJobV3Image(OauthToken oauthToken, String yesterda, String yesterda1, int page, int pageSize) throws Exception {
-        log.info("开始获取体验版-图片-素材日报 accountID:{},page:{} date:{}", oauthToken.getAccountId(), page,yesterda);
+        log.info("开始获取体验版-图片-素材日报 accountID:{},page:{} date:{}", oauthToken.getAccountId(), page, yesterda);
         // 请求地址
         String url = bytedanceApiUrl + BytedanceConstant.BYTEDANCE_V3_REPORT_INTEGRATED_GET;
-        Map<String,Object> param = new HashMap<>();
-        param.put("advertiser_id",oauthToken.getAccountId());
-        param.put("start_date",yesterda);
-        param.put("end_date",yesterda1);
-        param.put("page",page);
-        param.put("page_size",pageSize);
-        //List<String> fields = Arrays.asList("stat_cost");
-        //String fields1 = "[\"stat_cost\",\"show_cnt\",\"image_mode\"]";
-        // fields 入参 为拼接字符串数组形式 与入参 video_image_mode数组 && image_image_mode数组 格式不同
+        Map<String, Object> param = new HashMap<>();
+        param.put("advertiser_id", oauthToken.getAccountId());
+        param.put("start_date", yesterda);
+        param.put("end_date", yesterda1);
+        param.put("page", page);
+        param.put("page_size", pageSize);
         String fields = ByteDanceNewMaterialReport.getParamsFields();
-        param.put("fields",fields);
-        param.put("time_granularity","STAT_TIME_GRANULARITY_DAILY");
-        //param.put("group_by","STAT_GROUP_BY_DELIVERY_MODE");
+        param.put("fields", fields);
+        param.put("time_granularity", "STAT_TIME_GRANULARITY_DAILY");
 
-        Map<String,String> header = new HashMap<>();
-        header.put("Access-Token",oauthToken.getAccessToken());
+        Map<String, String> header = new HashMap<>();
+        header.put("Access-Token", oauthToken.getAccessToken());
 
         //String video_image_mode = "[\"CREATIVE_IMAGE_MODE_VIDEO\",\"CREATIVE_IMAGE_MODE_VIDEO_VERTICAL\"]";
-        String[] video_image_mode = new String[]{"CREATIVE_IMAGE_MODE_VIDEO","CREATIVE_IMAGE_MODE_VIDEO_VERTICAL"};
+        String[] video_image_mode = new String[]{"CREATIVE_IMAGE_MODE_VIDEO", "CREATIVE_IMAGE_MODE_VIDEO_VERTICAL"};
         //String image_image_mode = "[\"CREATIVE_IMAGE_MODE_LARGE\",\"CREATIVE_IMAGE_MODE_LARGE_VERTICAL\",\"CREATIVE_IMAGE_MODE_SMALL\"]";
-        String[] image_image_mode = new String[]{"CREATIVE_IMAGE_MODE_LARGE","CREATIVE_IMAGE_MODE_LARGE_VERTICAL","CREATIVE_IMAGE_MODE_SMALL"};
+        String[] image_image_mode = new String[]{"CREATIVE_IMAGE_MODE_LARGE", "CREATIVE_IMAGE_MODE_LARGE_VERTICAL", "CREATIVE_IMAGE_MODE_SMALL"};
 
         JSONObject filtering = new JSONObject();
-        Map<String,Object> video_material = new HashMap<>();
-        video_material.put("image_mode",video_image_mode);
+        Map<String, Object> video_material = new HashMap<>();
+        video_material.put("image_mode", video_image_mode);
 
-        Map<String,Object> image_material = new HashMap<>();
-        image_material.put("image_mode",image_image_mode);
+        Map<String, Object> image_material = new HashMap<>();
+        image_material.put("image_mode", image_image_mode);
 
 //        filtering.put("video_material",video_material);
-        filtering.put("image_material",image_material);
+        filtering.put("image_material", image_material);
 
-        param.put("filtering",filtering);
-        JSONObject resultObject = JSONObject.parseObject(HttpUtils.httpGet(url, param,header));
+        param.put("filtering", filtering);
+        JSONObject resultObject = JSONObject.parseObject(HttpUtils.httpGet(url, param, header));
 
         Integer code = resultObject.getInteger("code");
         if (null == code || !code.equals(0)) {
@@ -271,4 +273,26 @@ public class BytedanceMaterialReportDailyServiceImpl implements IBytedanceMateri
 
     }
 
+    //Unicode转中文方法
+    public static String unicodeToCn(String unicode) {
+        /** 以 \ u 分割,因为java注释也能识别unicode,因此中间加了一个空格*/
+        String[] strs = unicode.split("\\\\u");
+        String returnStr = "";
+        // 由于unicode字符串以 \ u 开头,因此分割出的第一个字符是""。
+        for (int i = 1; i < strs.length; i++) {
+            returnStr += (char) Integer.valueOf(strs[i], 16).intValue();
+        }
+        return returnStr;
+    }
+
+    public String unicodeDecode(String unicode) {
+        Pattern pattern = Pattern.compile("(\\\\u(\\p{XDigit}{4}))");
+        Matcher matcher = pattern.matcher(unicode);
+        char ch = 0;
+        while (matcher.find()) {
+            ch = (char) Integer.parseInt(matcher.group(2), 16);
+            unicode = unicode.replace(matcher.group(1), ch + "");
+        }
+        return unicode;
+    }
 }

+ 5 - 0
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/impl/OauthTokenServiceImpl.java

@@ -29,4 +29,9 @@ public class OauthTokenServiceImpl extends ServiceImpl<OauthTokenMapper, OauthTo
   public List<Long> getAgentByTouTiao() {
     return tokenMapper.getAgentByTouTiao();
   }
+
+  @Override
+  public List<Long> getAllToutiaoAccountIds() {
+    return tokenMapper.getAllToutiaoAccountIds();
+  }
 }

+ 4 - 9
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/utils/HttpUtils.java

@@ -24,6 +24,7 @@ import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.impl.client.HttpClients;
 import org.apache.http.impl.cookie.BasicClientCookie;
+import org.apache.http.protocol.HTTP;
 import org.apache.http.ssl.SSLContextBuilder;
 import org.apache.http.util.EntityUtils;
 
@@ -152,7 +153,7 @@ public class HttpUtils {
                 return JSONObject.parseObject(result.toString());
             }
         } catch (Exception e) {
-            log.info("地址:{},---参数{},---调用发生了异常=======》{}",url,e,params);
+            log.info("地址:{},---参数{},---调用发生了异常=======》{}", url, e, params);
             e.printStackTrace();
         } finally {
             try {
@@ -170,8 +171,6 @@ public class HttpUtils {
     }
 
 
-
-
     public static HttpClient getHttpclient() {
         RequestConfig defaultRequestConfig = RequestConfig.custom().setSocketTimeout(180000).setConnectTimeout(30000)
                 .setConnectionRequestTimeout(30000).setStaleConnectionCheckEnabled(true).build();
@@ -212,12 +211,10 @@ public class HttpUtils {
                     httpget.setHeader(key, headers.get(key));
                 }
             }
-            HttpEntity respEntity;
             HttpResponse response = httpclient.execute(httpget);
-            respEntity = response.getEntity();
+            HttpEntity entity = response.getEntity();
             Header[] responseHeaders = response.getHeaders("Set-Cookie");
             if (responseHeaders != null) {
-
                 for (Header header : responseHeaders) {
                     String[] ckArray = header.getValue().split(";");
                     BasicClientCookie ck = new BasicClientCookie(ckArray[0].split("=")[0], ckArray[0].split("=")[1]);
@@ -226,13 +223,11 @@ public class HttpUtils {
                     HttpUtils.COOKIESTORE.addCookie(ck);
                 }
             }
-            result = EntityUtils.toString(respEntity);
+            result = EntityUtils.toString(entity, HTTP.UTF_8);
         } catch (Exception e) {
             log.error("http get请求异常");
             e.printStackTrace();
         }
         return result;
     }
-
-
 }