yumeng il y a 5 ans
Parent
commit
582f38bc5d

+ 14 - 7
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/TestController.java

@@ -139,12 +139,9 @@ public class TestController {
 
         try {
 
-
             Thread thread = new Thread() {
                 @Override
                 public void run() {
-
-
                     QueryWrapper<CtopOauthToken> tokenQueryWrapper = new QueryWrapper<>();
                     tokenQueryWrapper.eq("media_id", 2);
                     List<CtopOauthToken> list = tokenService.list(tokenQueryWrapper);
@@ -155,10 +152,20 @@ public class TestController {
                             @Override
                             public void run() {
                                 try {
-                                    //获取全量视频素材数据
-                                    kuaishouInterfaceService.getVideoList(token, null, null);
-                                    // 获取图片信息数据
-                                    //   kuaishouInterfaceService.getImageList(token, startDate, endDate);
+                                    kuaishouInterfaceService.getSuZaoList(token.getAccessToken(), token.getAccountId(), 1);
+                                    String nowDate = DateUtils.getNowDate("yyyy-MM-dd");
+                                    String videoEndDate = DateUtils.addDay(nowDate, -180);
+                                    SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
+                                    Date end = simpleDateFormat.parse(nowDate);
+                                    Date start = simpleDateFormat.parse(videoEndDate);
+                                    List<Date> dates = DateUtils.findDates(start, end);
+                                    for (int i = 0; i < dates.size(); i++) {
+                                        String formatDate = simpleDateFormat.format(dates.get(i));
+                                        kuaishouInterfaceService.getVideoList(token, formatDate);
+                                        kuaishouInterfaceService.getImageList(token, formatDate);
+                                        Thread.sleep(500);
+                                    }
+
                                 } catch (Exception e) {
                                     e.printStackTrace();
                                 } finally {

+ 2 - 3
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/KuaishouMaterialsLoadJob.java

@@ -2,7 +2,6 @@ package org.jeecg.modules.ctop.job;
 
 import cn.com.ctop.common.module.entity.CtopOauthToken;
 import cn.com.ctop.common.module.service.ICtopOauthTokenService;
-import cn.com.ctop.common.module.utils.CtopAdConstant;
 import cn.com.ctop.kuaishou.modules.batch.service.IKuaishouInterfaceService;
 import lombok.extern.slf4j.Slf4j;
 import org.jeecg.common.util.DateUtils;
@@ -43,9 +42,9 @@ public class KuaishouMaterialsLoadJob implements Job {
                             try {
 
                                 //获取全量视频素材数据
-                                kuaishouInterfaceService.getVideoList(token, startDate, endDate);
+                                kuaishouInterfaceService.getVideoList(token, DateUtils.getNowDate("yyyy-MM-dd"));
                                 // 获取图片信息数据
-                                kuaishouInterfaceService.getImageList(token, startDate, endDate);
+                                kuaishouInterfaceService.getImageList(token, DateUtils.getNowDate("yyyy-MM-dd"));
                             } catch (Exception e) {
                                 e.printStackTrace();
                             } finally {

+ 3 - 0
module-ctop/src/main/java/cn/com/ctop/manage/modules/material/service/impl/MaterialUploadImageServiceImpl.java

@@ -13,6 +13,7 @@ import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.http.ParseException;
+import org.jeecg.common.util.DateUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.http.HttpEntity;
@@ -98,6 +99,8 @@ public class MaterialUploadImageServiceImpl implements IMaterialUploadImageServi
                                             if (!Check.isNull(type)) {
                                                 imageGet.setMaterialType(type);
                                             }
+                                            String nowDate = DateUtils.getNowDate("yyyy-MM-dd");
+                                            imageGet.setStatDate(DateUtils.parseDate(nowDate, "yyyy-MM-dd"));
                                             imageGet.setSignature(signature);
                                             imageGet.setImageToken(image_token);
                                             imageGetService.saveOrUpdate(imageGet);

+ 3 - 0
module-ctop/src/main/java/cn/com/ctop/manage/modules/material/service/impl/MaterialUploadServiceImpl.java

@@ -21,6 +21,7 @@ import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import lombok.extern.slf4j.Slf4j;
+import org.jeecg.common.util.DateUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.core.io.FileSystemResource;
@@ -221,6 +222,8 @@ public class MaterialUploadServiceImpl implements IMaterialUploadService {
                                             }
                                             videoGet.setPhotoId(photoId);
                                             videoGet.setSignature(signature);
+                                            String nowDate = DateUtils.getNowDate("yyyy-MM-dd");
+                                            videoGet.setStatDate(DateUtils.parseDate(nowDate, "yyyy-MM-dd"));
                                             kuaiShouVideoGetService.saveOrUpdate(videoGet);
                                             materialUploadImageService.uploadImage(videoGet.getSignature(), accountId, ctopOauthToken.getAccessToken());
 

+ 2 - 2
module-job-kuaishou/src/main/java/cn/com/ctop/job/kuaishou/handler/KuaishouMaterialsLoadJob.java

@@ -34,9 +34,9 @@ public class KuaishouMaterialsLoadJob {
         executorService = Executors.newFixedThreadPool(8);
         tokens.forEach(token -> executorService.submit(() -> {
             //获取全量视频素材数据
-            kuaishouInterfaceService.getVideoList(token, startDate, endDate);
+            kuaishouInterfaceService.getVideoList(token, DateUtils.getNowDate("yyyy-MM-dd"));
             // 获取图片信息数据
-            kuaishouInterfaceService.getImageList(token, startDate, endDate);
+            kuaishouInterfaceService.getImageList(token, DateUtils.getNowDate("yyyy-MM-dd"));
         }));
         XxlJobLogger.log("快手物料数据同步完成");
         return ReturnT.SUCCESS;

+ 9 - 8
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/controller/BatchController.java

@@ -24,6 +24,7 @@ import io.swagger.annotations.Api;
 import lombok.extern.slf4j.Slf4j;
 import org.jeecg.common.api.vo.Result;
 import org.jeecg.common.system.query.QueryGenerator;
+import org.jeecg.common.util.DateUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.*;
 
@@ -133,7 +134,7 @@ public class BatchController {
             Thread thread2 = new Thread() {
                 @Override
                 public void run() {
-                    iKuaishouInterfaceService.getVideoList(oauthToken, null, null);
+                    iKuaishouInterfaceService.getVideoList(oauthToken, DateUtils.getNowDate("yyyy-mm-dd"));
                 }
             };
             thread2.start();
@@ -1049,7 +1050,7 @@ public class BatchController {
         Result<IPage<KuaiShouImageGet>> result = new Result<>();
         try {
             QueryWrapper<KuaiShouImageGet> queryWrapper = QueryGenerator.initQueryWrapper(kuaiShouImageGet, req.getParameterMap());
-            queryWrapper.orderByDesc("create_time");
+            queryWrapper.orderByDesc("stat_date");
             Page<KuaiShouImageGet> page = new Page<>(pageNo, pageSize);
             IPage<KuaiShouImageGet> pageList = kuaiShouImageGetService.page(page, queryWrapper);
             result.setSuccess(true);
@@ -1071,15 +1072,15 @@ public class BatchController {
     @GetMapping(value = "/getVideoList")
     public Result<IPage<KuaiShouVideoGet>> getVideoList(KuaiShouVideoGet kuaiShouVideoGet, @RequestParam(name = "pageNo", defaultValue = "1") Integer pageNo,
                                                         @RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize,
-                                                        @RequestParam(name = "startDate", defaultValue = "2010-01-01") String startDate,
-                                                        @RequestParam(name = "endDate", defaultValue = "2100-01-01") String endDate, HttpServletRequest req) {
+                                                        @RequestParam(name = "startDate", defaultValue = "") String startDate,
+                                                        @RequestParam(name = "endDate", defaultValue = "") String endDate, HttpServletRequest req) {
         Result<IPage<KuaiShouVideoGet>> result = new Result<>();
         try {
             QueryWrapper<KuaiShouVideoGet> queryWrapper = QueryGenerator.initQueryWrapper(kuaiShouVideoGet, req.getParameterMap());
-            startDate = startDate + " 00:00:00";
-            endDate = endDate + " 23:59:59";
-            queryWrapper.between("create_time", startDate, endDate);
-            queryWrapper.orderByDesc("create_time");
+            if (!Check.isNull(startDate) && !Check.isNull(endDate)) {
+                queryWrapper.between("stat_date", startDate, endDate);
+            }
+            queryWrapper.orderByDesc("stat_date");
             Page<KuaiShouVideoGet> page = new Page<>(pageNo, pageSize);
             IPage<KuaiShouVideoGet> pageList = kuaiShouVideoGetService.page(page, queryWrapper);
             result.setSuccess(true);

+ 1 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/entity/KuaiShouImageGet.java

@@ -82,6 +82,7 @@ public class KuaiShouImageGet {
     @Excel(name = "图片 md5", width = 15)
     @ApiModelProperty(value = "图片 md5")
     private String signature;
+    private Date statDate;
     /**
      * 创建时间
      */

+ 2 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/entity/KuaiShouVideoGet.java

@@ -74,6 +74,8 @@ public class KuaiShouVideoGet {
 
     private String signature;
 
+    private Date statDate;
+
 
     private Integer materialType;
     /**

+ 2 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/mapper/xml/KuaiShouImageGetMapper.xml

@@ -14,6 +14,7 @@
         material_type,
         signature,
         image_token,
+        stat_date,
         create_time,
         update_time
         )
@@ -30,6 +31,7 @@
             #{image.materialType},
             #{image.signature},
             #{image.imageToken},
+            #{image.statDate},
             #{image.createTime},
             #{image.updateTime}
             )

+ 3 - 6
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/IKuaishouInterfaceService.java

@@ -309,10 +309,9 @@ public interface IKuaishouInterfaceService {
      * 获取视频列表
      *
      * @param token
-     * @param startDate
-     * @param endDate
+     * @param statDate
      */
-    void getVideoList(CtopOauthToken token, Date startDate, Date endDate);
+    void getVideoList(CtopOauthToken token, String statDate);
 
 
     /**
@@ -411,10 +410,8 @@ public interface IKuaishouInterfaceService {
      * 获取快手图片列表
      *
      * @param token
-     * @param startDate
-     * @param endDate
      */
-    void getImageList(CtopOauthToken token, Date startDate, Date endDate);
+    void getImageList(CtopOauthToken token, String statDate);
 
 
     /**

+ 33 - 19
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/impl/KuaishouInterfaceServiceImpl.java

@@ -47,6 +47,7 @@ import java.io.File;
 import java.io.IOException;
 import java.lang.reflect.ParameterizedType;
 import java.lang.reflect.Type;
+import java.text.SimpleDateFormat;
 import java.util.*;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -186,11 +187,11 @@ public class KuaishouInterfaceServiceImpl implements IKuaishouInterfaceService {
 
 
     @Override
-    public void getVideoList(CtopOauthToken token, Date startDate, Date endDate) {
-        getVideoListByPage(token, startDate, endDate, 1);
+    public void getVideoList(CtopOauthToken token, String statDate) {
+        getVideoListByPage(token, statDate, 1);
     }
 
-    private void getVideoListByPage(CtopOauthToken token, Date startDate, Date endDate, int page) {
+    private void getVideoListByPage(CtopOauthToken token, String statDate, int page) {
         try {
             log.info("获取视频数据,accountId:{}", token.getAccountId());
             String url = PropertiesUtils.getConfig("kuaishou_api_url") + KuaishouInterfaceConstant.VIDEO_LIST;
@@ -201,9 +202,9 @@ public class KuaishouInterfaceServiceImpl implements IKuaishouInterfaceService {
             param.put("advertiser_id", token.getAccountId());
             param.put("page_size", 500);
             param.put("page", page);
-            if (startDate != null && endDate != null) {
-                param.put("start_date", DateUtils.formatDate(startDate));
-                param.put("end_date", DateUtils.formatDate(endDate));
+            if (!Check.isNull(statDate)) {
+                param.put("start_date", statDate);
+                param.put("end_date", statDate);
             }
             String result = HttpUtils.httpPostRequest(url, param, headers);
             JSONObject resultJson = JSONObject.parseObject(result);
@@ -218,12 +219,12 @@ public class KuaishouInterfaceServiceImpl implements IKuaishouInterfaceService {
                 log.info("快手视频列表信息为空=》accountId:{}", token.getAccountId());
                 return;
             }
-            List<KuaiShouVideoGet> videoGets = new ArrayList<>();
             for (int i = 0; i < details.size(); i++) {
                 var detailJson = details.getJSONObject(i);
                 var kuaiShouVideoGet = JSONObject.toJavaObject(detailJson, KuaiShouVideoGet.class);
                 kuaiShouVideoGet.setId(token.getAccountId() + kuaiShouVideoGet.getPhotoId());
                 kuaiShouVideoGet.setAccountId(token.getAccountId());
+                kuaiShouVideoGet.setStatDate(DateUtils.parseDate(statDate, "yyyy-MM-dd"));
                 kuaiShouVideoGet.setUpdateTime(new Date());
                 Integer type = MaterialEnum.getTypeBySize(kuaiShouVideoGet.getWidth(), kuaiShouVideoGet.getHeight());
                 if (!Check.isNull(type)) {
@@ -231,7 +232,7 @@ public class KuaishouInterfaceServiceImpl implements IKuaishouInterfaceService {
                 }
                 kuaiShouVideoGetService.saveOrUpdate(kuaiShouVideoGet);
             }
-            getVideoListByPage(token, startDate, endDate, page + 1);
+            getVideoListByPage(token, statDate, page + 1);
         } catch (Exception e) {
             e.printStackTrace();
         }
@@ -890,9 +891,9 @@ public class KuaishouInterfaceServiceImpl implements IKuaishouInterfaceService {
         //获取全量创意数据
         getCreativeList(token, null, null);
         //获取全量视频素材数据
-        getVideoList(token, null, null);
+
         //获取图片信息数据
-        getImageList(token, null, null);
+
 
         getAppList(token.getAccountId(), token.getAccessToken());
 
@@ -903,6 +904,18 @@ public class KuaishouInterfaceServiceImpl implements IKuaishouInterfaceService {
 
         historyReportTaskService.createTask(token.getAccountId(), token.getAccessToken(), startDate, endDateStr, CtopAdConstant.KUAISHOU_LOAD_JOB_TYPE_HISTORY);
 
+        String nowDate = DateUtils.getNowDate("yyyy-MM-dd");
+        String videoEndDate = DateUtils.addDay(nowDate, -180);
+        SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
+        Date end = simpleDateFormat.parse(nowDate);
+        Date start = simpleDateFormat.parse(videoEndDate);
+        List<Date> dates = DateUtils.findDates(start, end);
+        for (int i = 0; i < dates.size(); i++) {
+            String formatDate = simpleDateFormat.format(dates.get(i));
+            getVideoList(token, formatDate);
+            getImageList(token, formatDate);
+        }
+
 
     }
 
@@ -2531,15 +2544,15 @@ public class KuaishouInterfaceServiceImpl implements IKuaishouInterfaceService {
      * 查询图片列表
      */
     @Override
-    public void getImageList(CtopOauthToken token, Date startDate, Date endDate) {
-        getImageList(token, startDate, endDate, 1);
+    public void getImageList(CtopOauthToken token, String statDate) {
+        getImageList(token, statDate, 1);
     }
 
 
     /**
      * 查询图片列表
      */
-    private void getImageList(CtopOauthToken token, Date startDate, Date endDate, int page) {
+    private void getImageList(CtopOauthToken token, String statDate, int page) {
         try {
             String url = PropertiesUtils.getConfig("kuaishou_api_url") + KuaishouInterfaceConstant.IMAGE_LIST;
             Map<String, String> headers = new HashMap<String, String>();
@@ -2549,9 +2562,9 @@ public class KuaishouInterfaceServiceImpl implements IKuaishouInterfaceService {
             //传参
             JSONObject param = new JSONObject();
             param.put("advertiser_id", token.getAccountId());
-            if (startDate != null && endDate != null) {
-                param.put("startDate", DateUtils.formatDate(startDate));
-                param.put("endDate", DateUtils.formatDate(endDate));
+            if (!Check.isNull(statDate)) {
+                param.put("startDate", DateUtils.formatDate(statDate));
+                param.put("endDate", DateUtils.formatDate(statDate));
             }
             param.put("page_size", 500);
             param.put("page", page);
@@ -2588,13 +2601,14 @@ public class KuaishouInterfaceServiceImpl implements IKuaishouInterfaceService {
                 if (!Check.isNull(type)) {
                     kuaiShouImageGet.setMaterialType(type);
                 }
+                kuaiShouImageGet.setStatDate(DateUtils.parseDate(statDate, "yyyy-MM-dd"));
 
                 kuaiShouImageGet.setCreateTime(new Date());
                 kuaiShouImageGet.setUpdateTime(new Date());
                 imageGets.add(kuaiShouImageGet);
             }
             kuaiShouImageGetService.replaceBatch(imageGets);
-            getImageList(token, startDate, endDate, page + 1);
+            getImageList(token, statDate, page + 1);
         } catch (Exception e) {
             e.printStackTrace();
 
@@ -2907,9 +2921,9 @@ public class KuaishouInterfaceServiceImpl implements IKuaishouInterfaceService {
         JSONObject dataJson = resultJson.getJSONObject("data");
         if (!Check.isNull(dataJson)) {
             Integer total_count = dataJson.getInteger("total_count");
-            if (total_count > 2000) {
+           /* if (total_count > 2000) {
                 return;
-            }
+            }*/
             JSONArray details = dataJson.getJSONArray("details");
             if (Check.isNull(details)) {
                 return;