Explorar o código

素材异步报表

hcst_sunzhen %!s(int64=5) %!d(string=hai) anos
pai
achega
383935c1f7

+ 59 - 2
jeecg-boot-base-common/src/main/java/org/jeecg/common/util/DateUtils.java

@@ -1204,8 +1204,65 @@ public class DateUtils extends PropertyEditorSupport {
         return betweenDate;
     }
 
-    public static void main(String[] args) {
-        getDiscrepantDays("2020-01-01","2020-03-31");
+    /***
+     * 两个日期之间间隔的月数(不考虑日期的情况)
+     */
+    public static int calDiffMonth(Date start, Date end){
+        GregorianCalendar startCalendar = new GregorianCalendar();
+        startCalendar.setTime(start);
+        GregorianCalendar endCalendar = new GregorianCalendar();
+        endCalendar.setTime(end);
+
+        int startYear=startCalendar.get(Calendar.YEAR);
+        int startMonth=startCalendar.get(Calendar.MONTH)+1;
+        int startDay=startCalendar.get(Calendar.DAY_OF_MONTH);
+        int endYear=endCalendar.get(Calendar.YEAR);
+        int endMonth=endCalendar.get(Calendar.MONTH)+1;
+        int endDay=endCalendar.get(Calendar.DAY_OF_MONTH);
+
+        return (endYear-startYear)*12 + (endMonth - startMonth);
+    }
+
+    public static int getDaysOfMonth(int year, int month) {
+        Calendar calendar = Calendar.getInstance();
+        calendar.set(year, month - 1, 1);
+        return calendar.getActualMaximum(Calendar.DAY_OF_MONTH);
+    }
+
+    public static int getFirstDateOfMonth(int year, int month) {
+        Calendar calendar = Calendar.getInstance();
+        calendar.set(year, month - 1, 1);
+        return calendar.getActualMinimum(Calendar.DAY_OF_MONTH);
+    }
+
+    /**
+     * 获取指定日期下个月的第一天
+     * @param dateStr
+     * @param format
+     * @return
+     */
+    public static Map<String, String> getMaxMinDaysOfAddMonth(String dateStr,String format, int addMonth){
+        SimpleDateFormat sdf = new SimpleDateFormat(format);
+        Map<String, String> dateMap = new HashMap<>();
+        try {
+            Date date = sdf.parse(dateStr);
+            Calendar calendar = Calendar.getInstance();
+            calendar.setTime(date);
+            calendar.add(Calendar.MONTH, addMonth);
+            calendar.set(Calendar.DAY_OF_MONTH,1);
+            dateMap.put("startDate", sdf.format(calendar.getTime()));
+
+            calendar.set(Calendar.DAY_OF_MONTH, calendar.getActualMaximum(Calendar.DATE));
+            dateMap.put("endDate", sdf.format(calendar.getTime()));
+            return dateMap;
+        } catch (ParseException e) {
+            e.printStackTrace();
+        }
+        return null;
+    }
+
+    public static void main(String[] args) throws ParseException {
+        System.out.println(calDiffMonth(DateUtils.parseDate("2019-04-01","yyyy-MM-dd"),new Date()));
     }
 
 }

+ 23 - 4
module-report/src/main/java/cn/com/ctop/bytedance/controller/BytedanceReportController.java

@@ -103,26 +103,45 @@ public class BytedanceReportController {
     }
 
     @PostMapping("/bytedance/async")
-    public Result<List<JSONObject>> asyncTaskCreate(@RequestBody JSONObject requestJson) {
+    public Result<List<JSONObject>> asyncTaskCreate(@RequestParam(name = "startDate")String startDate,
+                                                    @RequestParam(name = "endDate")String endDate) {
 
         Result<List<JSONObject>> result = new Result<>();
         try {
-            bytedanceReportService.bytedanceAsyncTaskCreate();
+            log.info("头条获取素材报表数据任务执行开始");
+            Long starttime = System.currentTimeMillis();
+            List<CtopOauthToken> tokens = tokenService.selectToutiaoToken();
+            if (null == tokens || tokens.size() <= 0) {
+                log.info("头条获取异步报表数据任务执行失败:未获取到可用的token");
+                return result;
+            }
 
+            for (CtopOauthToken token : tokens) {
+                if(1629785592929294L == token.getAccountId()){
+                    bytedanceReportService.bytedanceAsyncTaskCreate(startDate, endDate, token);
+                }
+             }
+            Long endtime = System.currentTimeMillis();
+            log.info("头条异步报表数据任务执行结束,执行耗时:{}秒", (endtime - starttime) / 1000);
         } catch (Exception e) {
+            log.error("头条获取异步报表数据任务执行报错");
             e.printStackTrace();
             result.setSuccess(false);
         }
         return result;
-
     }
 
     @PostMapping("/bytedance/async/taskGet")
     public Result<List<JSONObject>> bytedanceAsyncTaskGet(@RequestBody JSONObject requestJson) {
         Result<List<JSONObject>> result = new Result<>();
         try {
+            log.info("头条获取素材报表数据定时任务执行开始");
+            Long starttime = System.currentTimeMillis();
             bytedanceReportService.bytedanceAsyncTaskGet();
+            Long endtime = System.currentTimeMillis();
+            log.info("头条异步报表数据定时任务执行结束,执行耗时:{}秒", (endtime - starttime) / 1000);
         } catch (Exception e) {
+            log.error("头条获取异步报表数据定时任务执行报错");
             e.printStackTrace();
             result.setSuccess(false);
         }
@@ -247,7 +266,7 @@ public class BytedanceReportController {
             result.setSuccess(false);
         }
 
-        log.info("头条获取素材报表失败数据任务重试结束");
+        log.info("头条获取素材报表数据任务重试结束");
         return result;
     }
 

+ 2 - 0
module-report/src/main/java/cn/com/ctop/bytedance/entity/BytedanceDailyReportTask.java

@@ -70,4 +70,6 @@ public class BytedanceDailyReportTask {
 	private Date updateTime;
 
     private String taskStatus;
+
+    private Integer isDownload;
 }

+ 2 - 0
module-report/src/main/java/cn/com/ctop/bytedance/mapper/BytedanceDailyReportTaskMapper.java

@@ -14,4 +14,6 @@ import com.baomidou.mybatisplus.core.mapper.BaseMapper;
  */
 public interface BytedanceDailyReportTaskMapper extends BaseMapper<BytedanceDailyReportTask> {
 
+    void loadBytedanceDailyCreativeFileAsync(@Param("localPath")String localPath);
+
 }

+ 31 - 1
module-report/src/main/java/cn/com/ctop/bytedance/mapper/xml/BytedanceDailyReportTaskMapper.xml

@@ -2,4 +2,34 @@
 <!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
 <mapper namespace="cn.com.ctop.bytedance.mapper.BytedanceDailyReportTaskMapper">
 
-</mapper>
+    <insert id="loadBytedanceDailyCreativeFileAsync">
+		LOAD DATA local INFILE  #{localPath}
+		REPLACE INTO TABLE ctop_bytedance_report_creative_daily_async CHARACTER SET utf8mb4
+		FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n'
+		IGNORE 1 LINES
+		(
+		campaign_id,
+        deep_convert_rate,
+        ad_id,
+        ctr,
+        show_material,
+        avg_show_cost,
+        creative_id,
+        convert_cost,
+        convert_material,
+        deep_convert_cost,
+        account_id,
+        cost,
+        deep_convert,
+        avg_click_cost,
+        ad_name,
+        campaign_name,
+        convert_rate,
+        creative_material_mode,
+        click,
+        stat_datetime
+        )
+    </insert>
+
+
+</mapper>

+ 2 - 3
module-report/src/main/java/cn/com/ctop/bytedance/service/IBytedanceReportService.java

@@ -4,6 +4,7 @@ import cn.com.ctop.bytedance.entity.BytedanceReportMaterialRetry;
 import cn.com.ctop.common.module.entity.CtopOauthToken;
 import com.alibaba.fastjson.JSONObject;
 
+import java.text.ParseException;
 import java.util.List;
 
 public interface IBytedanceReportService {
@@ -35,9 +36,7 @@ public interface IBytedanceReportService {
      */
     List<JSONObject> getReportList(JSONObject requestJson);
 
-
-
-    void bytedanceAsyncTaskCreate();
+    void bytedanceAsyncTaskCreate(String startDate, String endDate, CtopOauthToken token) throws ParseException;
 
     void bytedanceAsyncTaskGet();
 

+ 52 - 89
module-report/src/main/java/cn/com/ctop/bytedance/service/impl/BytedanceReportServiceImpl.java

@@ -42,6 +42,8 @@ import java.io.InputStreamReader;
 import java.math.BigDecimal;
 import java.math.RoundingMode;
 import java.net.URI;
+import java.sql.Timestamp;
+import java.text.ParseException;
 import java.util.*;
 
 @Service
@@ -674,18 +676,42 @@ public class BytedanceReportServiceImpl implements IBytedanceReportService {
 
     /////////////////////////////////////////////////////////////////////////////////////////////////
     //创建异步任务
-    public void bytedanceAsyncTaskCreate () {
-        String startDate = DateUtils.formatDate(DateUtils.addDay(new Date(), -1));
-        String endDate = DateUtils.formatDate(DateUtils.addDay(new Date(), -31));
-        String type = "BYTEDANCE_LOAD_JOB_TYPE_ASYNC";
-        Long accountId = 1662649623517198L;
+    public void bytedanceAsyncTaskCreate (String startDate, String endDate, CtopOauthToken token) throws ParseException {
+        int diffMonths = DateUtils.calDiffMonth(DateUtils.parseDate(startDate,"yyyy-MM-dd"),DateUtils.parseDate(endDate,"yyyy-MM-dd"));
+        if (diffMonths == 0) {
+            bytedanceAsyncTaskCreateByMonth(startDate, endDate, token);
+        }else{
+            for(int i=0; i<=diffMonths ; i++){
+                if (i == 0){
+                    Map<String,String> dateMap = DateUtils.getMaxMinDaysOfAddMonth(startDate,"yyyy-MM-dd", i);
+                    String end = dateMap.get("endDate");
+                    //System.out.println(startDate + " ----- " + end);
+                    bytedanceAsyncTaskCreateByMonth(startDate, end, token);
+                }else if(i == diffMonths){
+                    Map<String,String> dateMap = DateUtils.getMaxMinDaysOfAddMonth(startDate,"yyyy-MM-dd", i);
+                    String start = dateMap.get("startDate");
+                    //System.out.println(start + " ----- " + endDate);
+                    bytedanceAsyncTaskCreateByMonth(start, endDate, token);
+                }else{
+                    Map<String,String> dateMap = DateUtils.getMaxMinDaysOfAddMonth(startDate,"yyyy-MM-dd", i);
+                    String start = dateMap.get("startDate");
+                    String end = dateMap.get("endDate");
+                    //System.out.println(start + " ----- " + end);
+                    bytedanceAsyncTaskCreateByMonth(start, end, token);
+                }
+            }
+        }
+    }
 
-        CtopOauthToken token = ctopOauthTokenService.getTokenByAccountId(accountId);
+    /***
+     * 由于异步拉取数据不能跨月,故只能按月循环拉取
+     * @param startDate
+     * @param endDate
+     * @param token
+     */
+    private void bytedanceAsyncTaskCreateByMonth(String startDate, String endDate, CtopOauthToken token){
         String access_token = token.getAccessToken();
-        Long advertiser_id = accountId;
-        //final Long[] ad_ids = new Long[]{1L};
-        //List<String> fields_list = FIELDS;
-        //ist<String> fields_list = new ArrayList<>();
+        Long accountId = token.getAccountId();
 
         // 请求地址
         String open_api_domain = "https://ad.toutiao.com";
@@ -695,16 +721,14 @@ public class BytedanceReportServiceImpl implements IBytedanceReportService {
         // 请求参数
         final Map filtering = new HashMap() {
             {
-                //put("start_date", startDate);
-                //put("end_date", endDate);
-                put("start_date", "2020-04-01");
-                put("end_date", "2020-04-15");
+                put("start_date", startDate);
+                put("end_date", endDate);
                 put("group_by", new String[]{"STAT_GROUP_BY_CREATIVE_ID","STAT_GROUP_BY_TIME_DAY","STAT_GROUP_BY_CREATIVE_MATERIAL_MODE"});   //按照创意分组、天分组、创意类型分组查询
             }
         };
-        Map data = new HashMap() {
+        final Map data = new HashMap() {
             {
-                put("advertiser_id", advertiser_id);
+                put("advertiser_id", accountId);
                 put("task_name", taskName);
                 put("task_type", "REPORT");
                 put("task_params", filtering);
@@ -760,6 +784,7 @@ public class BytedanceReportServiceImpl implements IBytedanceReportService {
                     task.setJson(jsonString);
                     task.setTaskStatus(dataJson.getString("task_status"));
                     task.setTaskType(dataJson.getString("task_type"));
+                    task.setIsDownload(0);
                     bytedanceDailyReportTaskMapper.insert(task);
                 }
 
@@ -786,7 +811,9 @@ public class BytedanceReportServiceImpl implements IBytedanceReportService {
     public void bytedanceAsyncTaskGet(){
         QueryWrapper<BytedanceDailyReportTask> taskQueryWrapper = new QueryWrapper<>();
         //查询三种状态的信息--任务已创建、任务执行中、任务处理中
-        taskQueryWrapper.in("task_status", "ASYNC_TASK_STATUS_CREATED","ASYNC_TASK_STATUS_EXECUTING","ASYNC_TASK_STATUS_EXECUTING_STAGE2");
+        taskQueryWrapper.eq("is_download",0);
+        //taskQueryWrapper.in("task_status", "ASYNC_TASK_STATUS_CREATED","ASYNC_TASK_STATUS_EXECUTING","ASYNC_TASK_STATUS_EXECUTING_STAGE2");
+        taskQueryWrapper.notIn("task_status", "ASYNC_TASK_STATUS_FAILED");
         taskQueryWrapper.orderByDesc("create_time");
         List<BytedanceDailyReportTask> taskList = bytedanceDailyReportTaskService.list(taskQueryWrapper);
 
@@ -802,7 +829,6 @@ public class BytedanceReportServiceImpl implements IBytedanceReportService {
                     String access_token = token.getAccessToken();
                     Long accountId = task.getAccountId();
                     final Long advertiser_id = accountId;
-                    //Long taskId = 682327L;
 
                     // 请求参数
                     final Map filtering = new HashMap() {
@@ -871,6 +897,7 @@ public class BytedanceReportServiceImpl implements IBytedanceReportService {
 
                                         if (taskStatus .equals( "ASYNC_TASK_STATUS_COMPLETED")) {
                                             bytedanceAsyncTaskDownload(task,token);
+                                            task.setIsDownload(1);
                                         }
                                             task.setTaskStatus(taskStatus);
                                             bytedanceDailyReportTaskMapper.updateById(task);
@@ -908,73 +935,18 @@ public class BytedanceReportServiceImpl implements IBytedanceReportService {
         String open_api_domain = "https://ad.toutiao.com";
         String path = "/open_api/2/async_task/download/";
 
-        Long accountId = task.getAccountId();
-        final Long advertiser_id = accountId;
-        String access_token = token.getAccessToken();
-        Long taskId = task.getTaskId();
-
-        //Map data = new HashMap(){
-        //    {
-        //        put("advertiser_id", advertiser_id);
-        //        put("task_id", taskId);
-        //    }
-        //};
-        //
-        //// 构造请求
-        //HttpEntityEnclosingRequestBase httpEntity = new HttpEntityEnclosingRequestBase() {
-        //    @Override
-        //    public String getMethod() {
-        //        return "GET";
-        //    }
-        //};
-        //
-        //httpEntity.setHeader("Access-Token",access_token);
-        //
-        //CloseableHttpResponse response = null;
-        //CloseableHttpClient client = null;
-
-
-
-        try {
-            //client = HttpClientBuilder.create().build();
-            //httpEntity.setURI(URI.create(open_api_domain + path));
-            //httpEntity.setEntity(new StringEntity(JSONObject.toJSONString(data), ContentType.APPLICATION_JSON));
-            //
-            //response = client.execute(httpEntity);
-            //log.info("response为:" +  response);
-            //if (response != null && response.getStatusLine().getStatusCode() == 200) {
-            //    BufferedReader bufferedReader  = new BufferedReader(new InputStreamReader(response.getEntity().getContent()));
-            //    StringBuffer result = new StringBuffer();
-            //    String line = "";
-            //    while((line = bufferedReader.readLine()) != null) {
-            //        result.append(line);
-            //    }
-            //    bufferedReader.close();
-
-            String url = open_api_domain + path;
-            String fileName = task.getAccountId() + "_" +  ".csv";
-            Map<String, Object> requestMap = new HashMap<>();
-            requestMap.put("advertiser_id", task.getAccountId());
-            requestMap.put("task_id", task.getTaskId());
-            String localPath = DownloadUtils.downloadByUrl(requestMap, downloadPath, url, token.getAccessToken(), fileName);
-
-
-
-        } catch (Exception e) {
-            e.printStackTrace();
-            //task.setTaskStatus(5);
-        }
-
-        return;
+        String url = open_api_domain + path;
+        String fileName = task.getAccountId() + "_" + System.currentTimeMillis() + ".csv";
+        Map<String, Object> requestMap = new HashMap<>();
+        requestMap.put("advertiser_id", task.getAccountId());
+        requestMap.put("task_id", task.getTaskId());
+        String localPath = DownloadUtils.downloadByUrl(requestMap, downloadPath, url, token.getAccessToken(), fileName);
+        bytedanceDailyReportTaskMapper.loadBytedanceDailyCreativeFileAsync(localPath);
     }
 
-
-
-
     /////////////////////////////////////////////////////////\
     //素材报表
     public void bytedanceMaterialReportRetry(CtopOauthToken token, String startDate, String endDate){
-        //CtopOauthToken token = ctopOauthTokenService.getTokenByAccountId(accountId);
         Long accountId = token.getAccountId();
         log.info("头条素材报表当前accountId为:" + accountId);
         Integer page = 1;
@@ -996,7 +968,6 @@ public class BytedanceReportServiceImpl implements IBytedanceReportService {
     }
 
     public void bytedanceMaterialReport(CtopOauthToken token, String startDate, String endDate){
-        //CtopOauthToken token = ctopOauthTokenService.getTokenByAccountId(accountId);
         Long accountId = token.getAccountId();
         log.info("头条素材报表当前accountId为:" + accountId);
         Integer page = 1;
@@ -1232,15 +1203,7 @@ public class BytedanceReportServiceImpl implements IBytedanceReportService {
                     return -1;
                 }
             }else{
-                //BytedanceReportMaterialRetry retry = new BytedanceReportMaterialRetry();
-                //retry.setAccountId(accountId);
-                //retry.setStatus(0);
-                //retry.setStartDate(startDate);
-                //retry.setEndDate(endDate);
-                //bytedanceReportMaterialDailyMapper.replaceMaterialRetry(retry);
-
                 log.error("头条素材报表请求有误:accountId:" + accountId + ",开始时间:" + startDate + "结束时间:" + endDate + "错误返回:" +response);
-
                 returnCode = 0;  //504等状态0;-2连接服务器报错 ;-1对方服务器返回值为空;200正常;1没有数据(也属于正常)
                 return 0;
             }