Преглед изворни кода

Merge remote-tracking branch 'origin/test' into test

syh пре 5 година
родитељ
комит
40d1c3d9ab
15 измењених фајлова са 38 додато и 40 уклоњено
  1. 2 19
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/KuaishouHistoryLoadFileJob.java
  2. 1 1
      jeecg-boot-module-system/src/main/resources/application-prod.yml
  3. 1 1
      jeecg-boot-module-system/src/main/resources/application-test.yml
  4. 6 0
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/entity/KuaiShouCreative.java
  5. 6 1
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/mapper/xml/KuaiShouCreativeMapper.xml
  6. 4 0
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/impl/KuaiShouDailyReportTaskServiceImpl.java
  7. 5 10
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/impl/KuaiShouHistoryReportTaskServiceImpl.java
  8. 6 0
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/impl/KuaishouInterfaceServiceImpl.java
  9. 0 1
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/KuaishouReportDailyCampaignMapper.java
  10. 1 1
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/xml/KuaishouReportDailyAccountMapper.xml
  11. 1 1
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/xml/KuaishouReportDailyCampaignMapper.xml
  12. 1 1
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/xml/KuaishouReportDailyCreativeMapper.xml
  13. 1 1
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/xml/KuaishouReportDailyGroupMapper.xml
  14. 1 2
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/utils/DownloadUtils.java
  15. 2 1
      module-report/src/main/java/cn/com/ctop/bytedance/service/impl/BytedanceCreativeDailyReportServiceImpl.java

+ 2 - 19
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/KuaishouHistoryLoadFileJob.java

@@ -1,8 +1,6 @@
 package org.jeecg.modules.ctop.job;
 
-import cn.com.ctop.common.module.entity.CtopOauthToken;
 import cn.com.ctop.common.module.service.ICtopOauthTokenService;
-import cn.com.ctop.common.module.utils.CtopAdConstant;
 import cn.com.ctop.kuaishou.modules.batch.service.IKuaiShouHistoryReportTaskService;
 import lombok.extern.slf4j.Slf4j;
 import org.quartz.Job;
@@ -10,9 +8,7 @@ import org.quartz.JobExecutionContext;
 import org.quartz.JobExecutionException;
 import org.springframework.beans.factory.annotation.Autowired;
 
-import java.util.List;
 import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
 
 /**
  * 获取文件状态并下载入库
@@ -32,22 +28,9 @@ public class KuaishouHistoryLoadFileJob implements Job {
         Thread thread = new Thread() {
             @Override
             public void run() {
-                //  查询快手token
-                List<CtopOauthToken> tokens = tokenService.getTokenListByType(CtopAdConstant.PLATFORM_TYPE_KUAISHOU);
-                if (null == tokens || tokens.size() <= 0) {
-                    log.info("定时获取快手数据异常:未获取到可用的token");
-                    return;
-                }
-                executorService = Executors.newFixedThreadPool(3);
-                for (CtopOauthToken token : tokens) {
-                    executorService.submit(new Runnable() {
-                        @Override
-                        public void run() {
-                            historyReportTaskService.getTaskList();
-                        }
-                    });
-                }
+                historyReportTaskService.getTaskList();
             }
+
         };
         thread.start();
 

+ 1 - 1
jeecg-boot-module-system/src/main/resources/application-prod.yml

@@ -100,7 +100,7 @@ spring:
         poolPreparedStatements: true
         maxPoolPreparedStatementPerConnectionSize: 20
         # 配置监控统计拦截的filters,去掉后监控界面sql无法统计,'wall'用于防火墙
-        filters: stat,wall,slf4j
+        filters: stat,slf4j
         # 通过connectProperties属性来打开mergeSql功能;慢SQL记录
         connectionProperties: druid.stat.mergeSql\=true;druid.stat.slowSqlMillis\=5000
       datasource:

+ 1 - 1
jeecg-boot-module-system/src/main/resources/application-test.yml

@@ -98,7 +98,7 @@ spring:
         poolPreparedStatements: true
         maxPoolPreparedStatementPerConnectionSize: 20
         # 配置监控统计拦截的filters,去掉后监控界面sql无法统计,'wall'用于防火墙
-        filters: stat,wall,slf4j
+        filters: stat,slf4j
         # 通过connectProperties属性来打开mergeSql功能;慢SQL记录
         connectionProperties: druid.stat.mergeSql\=true;druid.stat.slowSqlMillis\=5000
       datasource:

+ 6 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/entity/KuaiShouCreative.java

@@ -206,4 +206,10 @@ public class KuaiShouCreative {
     @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss")
     @ApiModelProperty(value = "修改时间")
     private Date updateTime;
+
+    private Integer firstFrameType;
+
+    private Date creativeCreateTime;
+
+    private Date creativeUpdateTime;
 }

+ 6 - 1
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/mapper/xml/KuaiShouCreativeMapper.xml

@@ -29,7 +29,9 @@
         ad_photo_played_t3s_url,
         description,
         action_bar_text,
-
+        first_frame_type,
+        creative_create_time,
+        creative_update_time,
         create_time,
         update_time)
         values
@@ -61,6 +63,9 @@
             #{creative.adPhotoPlayedT3sUrl},
             #{creative.description},
             #{creative.actionBarText},
+            #{creative.firstFrameType},
+            #{creative.creativeCreateTime},
+            #{creative.creativeUpdateTime},
             #{creative.createTime},
             #{creative.updateTime})
         </foreach>

+ 4 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/impl/KuaiShouDailyReportTaskServiceImpl.java

@@ -20,6 +20,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
+import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -93,6 +94,7 @@ public class KuaiShouDailyReportTaskServiceImpl extends ServiceImpl<KuaiShouDail
                                             loadFile(token, task);
                                         } else if (taskStatus == 3) {
                                             task.setTaskStatus(3);
+                                            task.setUpdateTime(new Date());
                                             this.updateById(task);
                                         }
                                     }
@@ -129,11 +131,13 @@ public class KuaiShouDailyReportTaskServiceImpl extends ServiceImpl<KuaiShouDail
                     reportDailyCreativeMapper.loadCreativeDailyReport(task.getAccountId(), localPath);
                 }
             }
+
             task.setTaskStatus(4);
         } catch (Exception e) {
             e.printStackTrace();
             task.setTaskStatus(5);
         }
+        task.setUpdateTime(new Date());
         this.updateById(task);
     }
 

+ 5 - 10
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/impl/KuaiShouHistoryReportTaskServiceImpl.java

@@ -26,8 +26,6 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
 
 /**
  * 快手历史数据任务记录
@@ -43,8 +41,6 @@ public class KuaiShouHistoryReportTaskServiceImpl extends ServiceImpl<KuaiShouHi
     private String downloadPath;
     static List<Integer> historyTypeList;
     static List<Integer> dailyTypeList;
-    static ExecutorService executorService = Executors.newFixedThreadPool(3);
-
     @Autowired
     private ICtopOauthTokenService ctopOauthTokenService;
     @Autowired
@@ -85,7 +81,6 @@ public class KuaiShouHistoryReportTaskServiceImpl extends ServiceImpl<KuaiShouHi
                 for (int i = 0; i < dailyTypeList.size(); i++) {
                     Integer type = dailyTypeList.get(i);
                     String taskName = accountId + "_" + type + "_" + System.currentTimeMillis();
-                    log.info("任务名称:{}", taskName);
                     param.put("task_name", taskName);
                     taskParams.put("view_type", type);
                     param.put("task_params", taskParams);
@@ -115,13 +110,12 @@ public class KuaiShouHistoryReportTaskServiceImpl extends ServiceImpl<KuaiShouHi
                 for (int i = 0; i < historyTypeList.size(); i++) {
                     Integer type = historyTypeList.get(i);
                     String taskName = accountId + "_" + type + "_" + System.currentTimeMillis();
-                    log.info("任务名称:{}", taskName);
                     param.put("task_name", taskName);
                     taskParams.put("view_type", type);
                     param.put("task_params", taskParams);
                     String result = HttpUtils.kuaiShouhttpPostRequest(url, param.toJSONString(), headers);
                     JSONObject resultJson = JSONObject.parseObject(result);
-                    System.err.println(resultJson);
+                    log.info("请求状态返回数据:{}", resultJson);
                     if (!Check.isNull(resultJson)) {
                         Integer code = resultJson.getInteger("code");
                         if (code == 0) {
@@ -152,6 +146,7 @@ public class KuaiShouHistoryReportTaskServiceImpl extends ServiceImpl<KuaiShouHi
      */
     @Override
     public void getTaskList() {
+        log.info("开始历史数据文件检查下载");
         QueryWrapper<KuaiShouHistoryReportTask> taskQueryWrapper = new QueryWrapper<>();
         taskQueryWrapper.eq("task_status", 0);
         taskQueryWrapper.orderByDesc("create_time");
@@ -171,7 +166,7 @@ public class KuaiShouHistoryReportTaskServiceImpl extends ServiceImpl<KuaiShouHi
                     param.put("task_ids", taskIds);
                     String result = HttpUtils.kuaiShouhttpPostRequest(url, param.toJSONString(), headers);
                     JSONObject resultJson = JSONObject.parseObject(result);
-                    System.err.println(resultJson);
+                    log.info("请求状态返回数据:{}", resultJson);
                     if (!Check.isNull(resultJson)) {
                         Integer code = resultJson.getInteger("code");
                         if (code == 0) {
@@ -246,8 +241,8 @@ public class KuaiShouHistoryReportTaskServiceImpl extends ServiceImpl<KuaiShouHi
 
     private void getDailyTypeList() {
         dailyTypeList = new ArrayList<>();
-        dailyTypeList.add(2);
-        dailyTypeList.add(3);
+     /*   dailyTypeList.add(2);
+        dailyTypeList.add(3);*/
         dailyTypeList.add(4);
     }
 

+ 6 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/impl/KuaishouInterfaceServiceImpl.java

@@ -1802,6 +1802,7 @@ public class KuaishouInterfaceServiceImpl implements IKuaishouInterfaceService {
                         creative.setDescription(displayInfoJson.getString("description"));
                         creative.setActionBarText(displayInfoJson.getString("action_bar_text"));
                     }
+
                     creative.setCreateTime(new Date());
                     creative.setUpdateTime(new Date());
                     creatives.add(creative);
@@ -1971,6 +1972,11 @@ public class KuaishouInterfaceServiceImpl implements IKuaishouInterfaceService {
                         creative.setDescription(displayInfoJson.getString("description"));
                         creative.setActionBarText(displayInfoJson.getString("action_bar_text"));
                     }
+
+                    creative.setFirstFrameType(detailJson.getInteger("first_frame_type"));
+                    creative.setCreativeCreateTime(detailJson.get("create_time")==null?null:detailJson.getDate("create_time"));
+                    creative.setCreativeUpdateTime(detailJson.get("update_time")==null?null:detailJson.getDate("update_time"));
+
                     creative.setCreateTime(new Date());
                     creative.setUpdateTime(new Date());
                     creatives.add(creative);

+ 0 - 1
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/KuaishouReportDailyCampaignMapper.java

@@ -1,6 +1,5 @@
 package cn.com.ctop.kuaishou.modules.report.mapper;
 
-import cn.com.ctop.kuaishou.modules.report.entity.KuaishouReportDailyAccount;
 import cn.com.ctop.kuaishou.modules.report.entity.KuaishouReportDailyCampaign;
 import com.baomidou.mybatisplus.core.mapper.BaseMapper;
 import org.apache.ibatis.annotations.Param;

+ 1 - 1
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/xml/KuaishouReportDailyAccountMapper.xml

@@ -3,7 +3,7 @@
 <mapper namespace="cn.com.ctop.kuaishou.modules.report.mapper.KuaishouReportDailyAccountMapper">
     <insert id="loadAccountDailyReport">
 		LOAD DATA local INFILE  #{localPath}
-		INTO TABLE ctop_kuaishou_report_daily_account_new CHARACTER SET utf8
+		REPLACE INTO TABLE ctop_kuaishou_report_daily_account_new CHARACTER SET utf8
 		FIELDS TERMINATED BY ',' enclosed by '"' LINES TERMINATED BY '\n'
 		IGNORE 1 LINES
 		(@ad_scene,

+ 1 - 1
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/xml/KuaishouReportDailyCampaignMapper.xml

@@ -5,7 +5,7 @@
 
     <insert id="loadCampaignDailyReport">
 		LOAD DATA local INFILE  #{localPath}
-		INTO TABLE ctop_kuaishou_report_daily_campaign_new CHARACTER SET utf8
+	    REPLACE INTO TABLE ctop_kuaishou_report_daily_campaign_new CHARACTER SET utf8
 		FIELDS TERMINATED BY ',' enclosed by '"' LINES TERMINATED BY '\n'
 		IGNORE 1 LINES
 		(campaign_id,

+ 1 - 1
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/xml/KuaishouReportDailyCreativeMapper.xml

@@ -5,7 +5,7 @@
 
     <insert id="loadCreativeDailyReport">
 		LOAD DATA local INFILE  #{localPath}
-		INTO TABLE ctop_kuaishou_report_daily_creative_new CHARACTER SET utf8
+		REPLACE INTO TABLE ctop_kuaishou_report_daily_creative_new CHARACTER SET utf8
 		FIELDS TERMINATED BY ',' enclosed by '"' LINES TERMINATED BY '\n'
 		IGNORE 1 LINES
 		(

+ 1 - 1
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/xml/KuaishouReportDailyGroupMapper.xml

@@ -4,7 +4,7 @@
 
     <insert id="loadGroupDailyReport">
 		LOAD DATA local INFILE  #{localPath}
-		INTO TABLE ctop_kuaishou_report_daily_group_new CHARACTER SET utf8
+		REPLACE INTO TABLE ctop_kuaishou_report_daily_group_new CHARACTER SET utf8
 		FIELDS TERMINATED BY ',' enclosed by '"' LINES TERMINATED BY '\n'
 		IGNORE 1 LINES
 		(

+ 1 - 2
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/utils/DownloadUtils.java

@@ -32,7 +32,7 @@ public class DownloadUtils {
             URL url = new URL(urlStr + "?" + postBody);
             HttpURLConnection conn = (HttpURLConnection) url.openConnection();
             //设置超时间为3秒
-            conn.setConnectTimeout(10 * 1000);
+            conn.setConnectTimeout(120 * 1000);
             //防止屏蔽程序抓取而返回403错误
             conn.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)");
             conn.setRequestProperty("Access-Token", token);
@@ -45,7 +45,6 @@ public class DownloadUtils {
             byte[] getData = readInputStream(inputStream);
 
             //文件保存位置
-            System.err.println(downloadPath);
             File saveDir = new File(downloadPath);
             if (!saveDir.exists()) {
                 saveDir.mkdirs();

+ 2 - 1
module-report/src/main/java/cn/com/ctop/bytedance/service/impl/BytedanceCreativeDailyReportServiceImpl.java

@@ -51,7 +51,8 @@ public class BytedanceCreativeDailyReportServiceImpl extends ServiceImpl<Bytedan
         if (Check.isNull(accountIds)) {
             return null;
         }
-        paramsMap.put("accountIds", accountIds);
+        List<String> list = JSONObject.parseArray(accountIds.toJSONString(), String.class);
+        paramsMap.put("accountIds", list);
         String nowDate = json.getString("startDate");
         String endDate = json.getString("endDate");
         paramsMap.put("startDate", nowDate);