Explorar o código

优化代理商数据清洗

zhaoxian hai 1 ano
pai
achega
f3b6eef5a6

+ 37 - 32
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/controller/BytedanceController.java

@@ -68,35 +68,6 @@ public class BytedanceController {
     @Autowired
     private IByteDanceAdvertiserDataService advertiserDataService;
 
-
-    /**
-     * 代理商日报
-     *
-     * @throws Exception
-     */
-    @GetMapping(value = "/bytedanceAdvertiserDailyReport")
-    public void bytedanceAdvertiserDailyReport() throws Exception {
-        List<Long> agentByTouTiao = tokenService.getAgentByTouTiao();
-        List<String> allDatesOfTwoTimes = DateUtils.getAllDatesOfTwoTimes("2022-09-01", "2022-09-14");
-        for (Long aLong : agentByTouTiao) {
-            String token = tokenService.getByAccountId(Long.valueOf(aLong));
-            OauthToken oauthToken = new OauthToken();
-            oauthToken.setAccessToken(token);
-            oauthToken.setAccountId(aLong);
-
-            allDatesOfTwoTimes.forEach(date -> dailyExecutorService.submit(() -> {
-                advertiserDataService.getAgentReport(oauthToken, date, date, CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
-                try {
-                    Thread.sleep(1000);
-                } catch (InterruptedException e) {
-                    e.printStackTrace();
-                }
-                advertiserDataService.getAgentReport(oauthToken, date, date, CtopAdConstant.BYTEDANCE_REPORT_TYPE_HOURLY);
-
-            }));
-        }
-    }
-
     @Autowired
     IBytedanceCreativeReportDailyService creativeReportDailyService;
     @Autowired
@@ -132,7 +103,7 @@ public class BytedanceController {
                         public void run() {
                             try {
                                 Thread.sleep(500);
-    //                            materialReportDailyService.bytedanceDailyMaterialReportJobV3Video(oauthToken, yesterda2, yesterda, 1, 100);
+                                //                            materialReportDailyService.bytedanceDailyMaterialReportJobV3Video(oauthToken, yesterda2, yesterda, 1, 100);
 //                                listService.getProjectList(oauthToken, start, yesterda, 1);
 //                                advertiserDataService.getV3AdvertiserReport(oauthToken, start, yesterda, CtopAdConstant.BYTEDANCE_REPORT_V3_TYPE_DAILY);
 //                                materialReportDailyService.bytedanceDailyMaterialReportJobV3Video(oauthToken, yesterda, yesterda, 1, 100);
@@ -151,7 +122,8 @@ public class BytedanceController {
 
     @GetMapping(value = "/testOneReport")
     public String testOneReport(Long accountId) {
-        String token = tokenService.getByAccountId(accountId);
+//        String token = tokenService.getByAccountId(accountId);
+        String token = tokenService.getAgentToken(accountId);
         if (Check.isNull(token)) {
             log.error("此账户未获取到相关token,accountId:{}", accountId);
             return "no";
@@ -179,7 +151,9 @@ public class BytedanceController {
 //                            advertiserDataService.getV3AdvertiserReport(oauthToken, nowDate, nowDate, CtopAdConstant.BYTEDANCE_REPORT_V3_TYPE_HOURLY);
 //                            listService.getProjectList(oauthToken, start, yesterda, 1);
 //                            materialReportDailyService.bytedanceDailyMaterialReportJobV3Image(oauthToken, yesterda2, yesterda, 1, 100);
-                            advertiserDataServices.getPositionAdvertiserReport(oauthToken, yesterda, yesterda, CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
+//                            advertiserDataServices.getPositionAdvertiserReport(oauthToken, yesterda, yesterda, CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
+//                            advertiserDataService.getAdvertiserReport(oauthToken, nowDate, nowDate, CtopAdConstant.BYTEDANCE_REPORT_V3_TYPE_HOURLY);
+//                            advertiserDataService.getAgentReport(oauthToken, yesterda, yesterda, CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
 
                         } catch (Exception e) {
                             e.printStackTrace();
@@ -189,5 +163,36 @@ public class BytedanceController {
         return "ok";
     }
 
+    @GetMapping(value = "/test1")
+    public void test1(Long accountId) {
+        String token = tokenService.getByAccountId(accountId);
+        OauthToken oauthToken = new OauthToken();
+        oauthToken.setAccessToken(token);
+        oauthToken.setAccountId(accountId);
+        log.info("获取头条视频全量素材---账户--{}======》》》", oauthToken.getAccountId());
+        fileVideoService.getFileVideoData(oauthToken, null, null, 1, 100);
+    }
+
+
+    @GetMapping(value = "/getReports")
+    public void getReports() {
+        List<JSONObject> tokens = tokenService.getAgentTokens();
+        //获取两个时间的每一天
+        List<String> perDay = DateUtils.getPerDay("2023-12-01", "2023-12-26");
+        for (String date : perDay) {
+            for (JSONObject obj : tokens) {
+                OauthToken oauthToken = new OauthToken();
+                oauthToken.setAccessToken(obj.getString("access_token"));
+                oauthToken.setAccountId(obj.getLong("agent_id"));
+                dailyExecutorService.submit(
+                        new Runnable() {
+                            @Override
+                            public void run() {
+                                advertiserDataService.getAgentReport(oauthToken, date, date);
+                            }
+                        });
+            }
+        }
+    }
 
 }

+ 7 - 2
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/mapper/OauthTokenMapper.java

@@ -1,6 +1,7 @@
 package cn.com.ctop.job.bytedance.data.mapper;
 
 import cn.com.ctop.job.bytedance.data.entity.OauthToken;
+import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.core.mapper.BaseMapper;
 import org.apache.ibatis.annotations.Param;
 
@@ -13,9 +14,13 @@ import java.util.List;
  * @version V1.0
  */
 public interface OauthTokenMapper extends BaseMapper<OauthToken> {
-  String getByAccountId(@Param("accountId") Long accountId);
+    String getByAccountId(@Param("accountId") Long accountId);
 
     List<Long> getAgentByTouTiao();
 
-  List<Long> getAllToutiaoAccountIds();
+    List<Long> getAllToutiaoAccountIds();
+
+    String getAgentToken(@Param("agentId") Long agentId);
+
+    List<JSONObject> getAgentTokens();
 }

+ 25 - 4
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/mapper/xml/OauthTokenMapper.xml

@@ -3,14 +3,35 @@
 <mapper namespace="cn.com.ctop.job.bytedance.data.mapper.OauthTokenMapper">
 
     <select id="getByAccountId" resultType="java.lang.String">
-       SELECT access_token  from  `jeecg-boot`.ctop_oauth_token WHERE account_id = #{accountId}
+        SELECT access_token
+        from `jeecg-boot`.ctop_oauth_token
+        WHERE account_id = #{accountId}
     </select>
 
+
     <select id="getAgentByTouTiao" resultType="Long">
-       SELECT agent_id  from  `jeecg-boot`.ctop_cwjs_agent_info WHERE status = 0 and enabled=0
+        SELECT agent_id
+        from `jeecg-boot`.ctop_cwjs_agent_info
+        WHERE status = 0
+          and enabled = 0
     </select>
 
     <select id="getAllToutiaoAccountIds" resultType="Long">
-        SELECT account_id FROM `jeecg-boot`.ctop_user_allocation WHERE media_id = 1 and account_status = 0
+        SELECT account_id
+        FROM `jeecg-boot`.ctop_user_allocation
+        WHERE media_id = 1
+          and account_status = 0
+    </select>
+
+    <select id="getAgentToken" resultType="java.lang.String">
+        SELECT access_token
+        from `jeecg-boot`.ctop_bytedance_agent_token
+        WHERE agent_id = #{agentId}
     </select>
-</mapper>
+
+    <select id="getAgentTokens" resultType="com.alibaba.fastjson.JSONObject">
+        SELECT access_token, agent_id
+        from `jeecg-boot`.ctop_bytedance_agent_token
+    </select>
+
+</mapper>

+ 1 - 1
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/IByteDanceAdvertiserDataService.java

@@ -18,7 +18,7 @@ public interface IByteDanceAdvertiserDataService {
 
     void getPositionAdvertiserReport(OauthToken oauthToken, String yesterda, String yesterda1, String bytedanceReportTypeDaily);
 
-    void getAgentReport(OauthToken oauthToken, String yesterda, String yesterda1, String bytedanceReportTypeDaily);
+    void getAgentReport(OauthToken oauthToken, String startDate, String endDate);
 
     void getV3AdvertiserConfig(OauthToken oauthToken);
 

+ 8 - 3
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/IOauthTokenService.java

@@ -1,6 +1,7 @@
 package cn.com.ctop.job.bytedance.data.service;
 
 import cn.com.ctop.job.bytedance.data.entity.OauthToken;
+import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.extension.service.IService;
 
 import java.util.List;
@@ -13,9 +14,13 @@ import java.util.List;
  */
 public interface IOauthTokenService extends IService<OauthToken> {
 
-  String getByAccountId(Long accountId);
+    String getByAccountId(Long accountId);
 
-  List<Long> getAgentByTouTiao();
+    List<Long> getAgentByTouTiao();
 
-  List<Long> getAllToutiaoAccountIds();
+    List<Long> getAllToutiaoAccountIds();
+
+    String getAgentToken(Long accountId);
+
+    List<JSONObject> getAgentTokens();
 }

+ 29 - 58
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/impl/ByteDanceAdvertiserDataServiceImpl.java

@@ -11,7 +11,6 @@ import cn.com.ctop.job.bytedance.data.entity.BytedanceAdReportHourly;
 import cn.com.ctop.job.bytedance.data.entity.BytedanceAdvertiserReportDaily;
 import cn.com.ctop.job.bytedance.data.entity.BytedanceAdvertiserReportHourly;
 import cn.com.ctop.job.bytedance.data.entity.BytedanceAgentReportDaily;
-import cn.com.ctop.job.bytedance.data.entity.BytedanceAgentReportHourly;
 import cn.com.ctop.job.bytedance.data.entity.BytedanceV3ProjectReportDaily;
 import cn.com.ctop.job.bytedance.data.entity.BytedanceV3ProjectReportHourly;
 import cn.com.ctop.job.bytedance.data.entity.BytedanceV3PromotionProjectReportDaily;
@@ -50,7 +49,6 @@ import javax.annotation.Resource;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -387,7 +385,7 @@ public class ByteDanceAdvertiserDataServiceImpl implements IByteDanceAdvertiserD
         config.propertyNamingStrategy = PropertyNamingStrategy.SnakeCase;
         JSONObject jsonObject = JSONObject.parseObject(JSON.toJSONString(conditions, config));
         JSONObject getObject = getAdvertiserStat(token, jsonObject);
-//        System.out.println("返回结果:" + getObject);
+        System.out.println("返回结果:" + getObject);
         if (null == getObject) {
             XxlJobHelper.log("头条广告主数据获取异常");
             log.error("头条广告主数据获取信息为空==》accountId:{}", token.getAccountId());
@@ -640,21 +638,13 @@ public class ByteDanceAdvertiserDataServiceImpl implements IByteDanceAdvertiserD
      */
     @Override
     public void getAgentReport(
-            OauthToken token, String startDate, String endDate, String bytedanceReportTypePl) {
+            OauthToken token, String startDate, String endDate) {
         JSONObject jsonObject = new JSONObject();
-        if (CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY.equals(bytedanceReportTypePl)) {
-            // 日报
-            jsonObject.put("agent_id", token.getAccountId());
-            jsonObject.put("start_date", startDate);
-            jsonObject.put("end_date", endDate);
-            jsonObject.put("page_size", 1000);
-            jsonObject.put("timeGranularity", bytedanceReportTypePl);
-        } else {
-            // 时报
-            jsonObject.put("agent_id", token.getAccountId());
-            jsonObject.put("page_size", 1000);
-            jsonObject.put("timeGranularity", bytedanceReportTypePl);
-        }
+        // 日报
+        jsonObject.put("agent_id", token.getAccountId());
+        jsonObject.put("start_date", startDate);
+        jsonObject.put("end_date", endDate);
+        jsonObject.put("page_size", 1000);
 
         getAgentReportByPage(token, jsonObject, 1);
     }
@@ -663,61 +653,41 @@ public class ByteDanceAdvertiserDataServiceImpl implements IByteDanceAdvertiserD
      * 获取代理商
      */
     private void getAgentReportByPage(OauthToken token, JSONObject conditions, Integer page) {
-        log.info("获取代理商数据 accountID:{},page:{}", token.getAccountId(), page);
-        SimpleDateFormat simpleDateFormat = new SimpleDateFormat("HH");
-        String hour = simpleDateFormat.format(new Date());
-        String nowDate = DateUtils.getNowDate("yyyyMMdd");
         conditions.put("page", page);
         SerializeConfig config = new SerializeConfig();
         config.propertyNamingStrategy = PropertyNamingStrategy.SnakeCase;
         JSONObject jsonObject = JSONObject.parseObject(JSON.toJSONString(conditions, config));
+//        System.out.println("----------------入参:" + jsonObject);
+//        System.out.println("-");
         JSONObject getObject = getAgentStat(token, jsonObject);
+//        System.out.println("----------------结果:" + getObject);
         if (null == getObject) {
             XxlJobHelper.log("头条代理商数据获取异常");
             return;
         }
         Integer code = getObject.getInteger("code");
-        String message = getObject.getString("message");
-        if (null == code || code != 0) {
-            XxlJobHelper.log("头条代理商数据获取异常:{}", message);
-            return;
-        }
-        JSONArray dataArray = getObject.getJSONObject("data").getJSONArray("list");
-        if (null == dataArray || dataArray.isEmpty()) {
-            return;
-        }
-        List<BytedanceAgentReportHourly> hourlyList = new ArrayList<>();
-        List<BytedanceAgentReportDaily> dailyList = new ArrayList<>();
-        try {
-            for (int i = 0; i < dataArray.size(); i++) {
-                JSONObject data = dataArray.getJSONObject(i);
-                if (null != conditions.get("timeGranularity")
-                        && conditions
-                        .get("timeGranularity")
-                        .equals(CtopAdConstant.BYTEDANCE_REPORT_TYPE_HOURLY)) {
-                    // 时报
-                    BytedanceAgentReportHourly hourlyReport =
-                            JSONObject.parseObject(data.toJSONString(), BytedanceAgentReportHourly.class);
-                    hourlyReport.setStatDatetime(nowDate);
-                    hourlyReport.setHour(Integer.parseInt(hour));
-                    hourlyList.add(hourlyReport);
-                } else {
+        if (code == 0) {
+            JSONArray dataArray = getObject.getJSONObject("data").getJSONArray("list");
+            if (null == dataArray || dataArray.isEmpty()) {
+                return;
+            }
+            List<BytedanceAgentReportDaily> dailyList = new ArrayList<>();
+            try {
+                Integer startDate = DateUtils.getDateInteger(conditions.getString("start_date"));
+                for (int i = 0; i < dataArray.size(); i++) {
+                    JSONObject data = dataArray.getJSONObject(i);
                     // 日报
-                    BytedanceAgentReportDaily dailyReport =
-                            JSONObject.parseObject(data.toJSONString(), BytedanceAgentReportDaily.class);
-                    dailyReport.setStatDatetime(((String) conditions.get("start_date")).replace("-", ""));
+                    BytedanceAgentReportDaily dailyReport = JSONObject.parseObject(data.toJSONString(), BytedanceAgentReportDaily.class);
+                    dailyReport.setStatDatetime(String.valueOf(startDate));
                     dailyList.add(dailyReport);
                 }
+                if (dailyList.size() > 0) {
+                    bytedanceAgentReportDailyMapper.insert(dailyList);
+                }
+            } catch (Exception e) {
+                e.printStackTrace();
+                log.info("获取代理商数据错误:{}", e.toString());
             }
-            if (hourlyList.size() > 0) {
-                bytedanceAgentReportHourlyMapper.insert(hourlyList);
-            }
-            if (dailyList.size() > 0) {
-                bytedanceAgentReportDailyMapper.insert(dailyList);
-            }
-        } catch (Exception e) {
-            e.printStackTrace();
-            log.info("获取代理商数据错误:{}", e.toString());
         }
         page++;
         int totalPage =
@@ -725,6 +695,7 @@ public class ByteDanceAdvertiserDataServiceImpl implements IByteDanceAdvertiserD
         if (page <= totalPage) {
             getAgentReportByPage(token, conditions, page);
         }
+
     }
 
     /**

+ 3 - 0
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/impl/BytedanceFileVideoGetServiceImpl.java

@@ -42,6 +42,8 @@ public class BytedanceFileVideoGetServiceImpl implements IBytedanceFileVideoGetS
         JSONObject param = new JSONObject();
 
         param.put("advertiser_id", oauthToken.getAccountId());
+        List<Long> materialIds = new ArrayList<>();
+        materialIds.add(7296782053222498341l);
         param.put("page", page);
         param.put("page_size", pageSize);
         JSONObject param2 = new JSONObject();
@@ -51,6 +53,7 @@ public class BytedanceFileVideoGetServiceImpl implements IBytedanceFileVideoGetS
         if (!Check.isNull(endDate)){
             param2.put("end_time",endDate);
         }
+        param2.put("material_ids",materialIds);
         param.put("filtering",param2);
         JSONObject resultObject = HttpUtils.bytedanceGetRequest(oauthToken.getAccessToken(), url, param);
 

+ 28 - 17
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/impl/OauthTokenServiceImpl.java

@@ -3,6 +3,7 @@ package cn.com.ctop.job.bytedance.data.service.impl;
 import cn.com.ctop.job.bytedance.data.entity.OauthToken;
 import cn.com.ctop.job.bytedance.data.mapper.OauthTokenMapper;
 import cn.com.ctop.job.bytedance.data.service.IOauthTokenService;
+import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
@@ -17,21 +18,31 @@ import java.util.List;
  */
 @Service
 public class OauthTokenServiceImpl extends ServiceImpl<OauthTokenMapper, OauthToken> implements IOauthTokenService {
-  @Autowired
-  private OauthTokenMapper tokenMapper;
-
-  @Override
-  public String getByAccountId(Long accountId) {
-    return tokenMapper.getByAccountId(accountId);
-  }
-
-  @Override
-  public List<Long> getAgentByTouTiao() {
-    return tokenMapper.getAgentByTouTiao();
-  }
-
-  @Override
-  public List<Long> getAllToutiaoAccountIds() {
-    return tokenMapper.getAllToutiaoAccountIds();
-  }
+    @Autowired
+    private OauthTokenMapper tokenMapper;
+
+    @Override
+    public String getByAccountId(Long accountId) {
+        return tokenMapper.getByAccountId(accountId);
+    }
+
+    @Override
+    public List<Long> getAgentByTouTiao() {
+        return tokenMapper.getAgentByTouTiao();
+    }
+
+    @Override
+    public List<Long> getAllToutiaoAccountIds() {
+        return tokenMapper.getAllToutiaoAccountIds();
+    }
+
+    @Override
+    public String getAgentToken(Long accountId) {
+        return tokenMapper.getAgentToken(accountId);
+    }
+
+    @Override
+    public List<JSONObject> getAgentTokens() {
+        return tokenMapper.getAgentTokens();
+    }
 }

+ 11 - 10
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedanceAgentLoadJob.java

@@ -5,6 +5,7 @@ import cn.com.ctop.job.bytedance.data.entity.OauthToken;
 import cn.com.ctop.job.bytedance.data.service.IByteDanceAdvertiserDataService;
 import cn.com.ctop.job.bytedance.data.service.IOauthTokenService;
 import cn.com.ctop.job.bytedance.data.utils.DateUtils;
+import com.alibaba.fastjson.JSONObject;
 import com.xxl.job.core.handler.annotation.XxlJob;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -40,22 +41,22 @@ public class BytedanceAgentLoadJob {
      */
     @XxlJob("bytedanceAgentDailyReport")
     public void bytedanceAdvertiserDailyReport() throws Exception {
-
-        List<Long> agentByTouTiao = tokenService.getAgentByTouTiao();
+        List<JSONObject> tokens = tokenService.getAgentTokens();
         String nowDate = DateUtils.getNowDate("yyyy-MM-dd");
         String yesterda = DateUtils.getAnotherDay("yyyy-MM-dd", nowDate, -1);
         String yesterda2 = DateUtils.getAnotherDay("yyyy-MM-dd", nowDate, -2);
-        for (Long aLong : agentByTouTiao) {
-            String token = tokenService.getByAccountId(Long.valueOf(aLong));
+        String yesterda3 = DateUtils.getAnotherDay("yyyy-MM-dd", nowDate, -3);
+        for (JSONObject obj : tokens) {
             OauthToken oauthToken = new OauthToken();
-            oauthToken.setAccessToken(token);
-            oauthToken.setAccountId(aLong);
+            oauthToken.setAccessToken(obj.getString("access_token"));
+            oauthToken.setAccountId(obj.getLong("agent_id"));
             dailyExecutorService.submit(
                     new Runnable() {
                         @Override
                         public void run() {
-                            advertiserDataService.getAgentReport(oauthToken, yesterda, yesterda, CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
-                            advertiserDataService.getAgentReport(oauthToken, yesterda2, yesterda2, CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
+                            advertiserDataService.getAgentReport(oauthToken, yesterda, yesterda);
+                            advertiserDataService.getAgentReport(oauthToken, yesterda2, yesterda2);
+                            advertiserDataService.getAgentReport(oauthToken, yesterda3, yesterda3);
                         }
                     });
         }
@@ -84,8 +85,8 @@ public class BytedanceAgentLoadJob {
                     new Runnable() {
                         @Override
                         public void run() {
-                            advertiserDataService.getAgentReport(oauthToken, nowDate, nowDate, CtopAdConstant.BYTEDANCE_REPORT_TYPE_HOURLY);
-                            advertiserDataService.getAgentReport(oauthToken, yesterda, yesterda, CtopAdConstant.BYTEDANCE_REPORT_TYPE_HOURLY);
+//                            advertiserDataService.getAgentReport(oauthToken, nowDate, nowDate, CtopAdConstant.BYTEDANCE_REPORT_TYPE_HOURLY);
+//                            advertiserDataService.getAgentReport(oauthToken, yesterda, yesterda, CtopAdConstant.BYTEDANCE_REPORT_TYPE_HOURLY);
 
                         }
                     });