yumeng 5 years ago
parent
commit
8cba3b49da

+ 43 - 23
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/TestController.java

@@ -79,20 +79,37 @@ public class TestController {
     @Autowired
     private CrawlerDouyinMusicTaskService crawlerDouyinMusicTaskService;
     @Autowired
+    private IKuaiShouHistoryReportTaskService kuaiShouHistoryReportTaskService;
+    @Autowired
     private MusicInfoService musicInfoService;
 
     static ExecutorService executorService = Executors.newFixedThreadPool(15);
     static ExecutorService videoService = Executors.newFixedThreadPool(5);
     static ExecutorService suzhaoService = Executors.newFixedThreadPool(5);
+
+
+    @GetMapping(value = "/getVideoReport")
+    public void getVideoReport() {
+        kuaiShouHistoryReportTaskService.getTaskList();
+    }
+
+
+    @GetMapping(value = "/getMd5")
+    public void getMd5() {
+        kuaiShouHistoryReportTaskService.getMd5();
+    }
+
+
+
     @GetMapping(value = "/syncDouyinMusicAsc")
-    public String syncDouyinMusicAsc(){
+    public String syncDouyinMusicAsc() {
         QueryWrapper<CrawlerDouyinMusicTask> queryWrapper = new QueryWrapper<>();
-        queryWrapper.eq("status","init").orderByAsc("id").last(" limit 20");
+        queryWrapper.eq("status", "init").orderByAsc("id").last(" limit 20");
 
         List<CrawlerDouyinMusicTask> list = crawlerDouyinMusicTaskService.list(queryWrapper);
 
-        while (list != null){
-            for(CrawlerDouyinMusicTask crawlerDouyinMusicTask : list){
+        while (list != null) {
+            for (CrawlerDouyinMusicTask crawlerDouyinMusicTask : list) {
                 MusicInfo musicInfo = new MusicInfo();
                 musicInfo.setExtId(crawlerDouyinMusicTask.getId());
                 musicInfo = douyinMusicService.getMusicInfo(musicInfo);
@@ -104,19 +121,20 @@ public class TestController {
         }
         return "success";
     }
+
     @GetMapping(value = "/syncDouyinMusic")
-    public String syncDouyinMusic(){
-        for(int i =0;i<10;i++){
+    public String syncDouyinMusic() {
+        for (int i = 0; i < 10; i++) {
             final int j = i;
-            Thread thread = new Thread(){
+            Thread thread = new Thread() {
                 @Override
                 public void run() {
                     QueryWrapper<CrawlerDouyinMusicTask> queryWrapper = new QueryWrapper<>();
-                    queryWrapper.eq("status","init").likeLeft("id",j).orderByDesc("id").last(" limit 20");
+                    queryWrapper.eq("status", "init").likeLeft("id", j).orderByDesc("id").last(" limit 20");
                     List<CrawlerDouyinMusicTask> list = crawlerDouyinMusicTaskService.list(queryWrapper);
-                    while (list != null){
-                        log.info("【同步歌曲】第"+j+"线程正在处理:"+list.size()+"条数据");
-                        for(CrawlerDouyinMusicTask crawlerDouyinMusicTask : list){
+                    while (list != null) {
+                        log.info("【同步歌曲】第" + j + "线程正在处理:" + list.size() + "条数据");
+                        for (CrawlerDouyinMusicTask crawlerDouyinMusicTask : list) {
                             MusicInfo musicInfo = new MusicInfo();
                             musicInfo.setExtId(crawlerDouyinMusicTask.getId());
                             douyinMusicService.getMusicInfo(musicInfo);
@@ -130,23 +148,24 @@ public class TestController {
 
         return "success";
     }
+
     @GetMapping(value = "/parseDouyinMusic")
-    public String parseDouyinMusic(){
-        for(int i =0;i<10;i++){
+    public String parseDouyinMusic() {
+        for (int i = 0; i < 10; i++) {
             final int j = i;
-            Thread thread = new Thread(){
+            Thread thread = new Thread() {
                 @Override
                 public void run() {
                     QueryWrapper<CrawlerDouyinMusicTask> queryWrapper = new QueryWrapper<>();
-                    queryWrapper.eq("status","done").likeLeft("id",j).orderByDesc("id").last(" limit 20");
+                    queryWrapper.eq("status", "done").likeLeft("id", j).orderByDesc("id").last(" limit 20");
                     List<CrawlerDouyinMusicTask> list = crawlerDouyinMusicTaskService.list(queryWrapper);
-                    while (list != null){
-                        log.info("【解析分类】第"+j+"线程正在处理:"+list.size()+"条数据");
-                        for(CrawlerDouyinMusicTask crawlerDouyinMusicTask : list){
+                    while (list != null) {
+                        log.info("【解析分类】第" + j + "线程正在处理:" + list.size() + "条数据");
+                        for (CrawlerDouyinMusicTask crawlerDouyinMusicTask : list) {
                             QueryWrapper<MusicInfo> queryWrapper1 = new QueryWrapper<>();
-                            queryWrapper1.eq("ext_id",crawlerDouyinMusicTask.getId()).last(" limit 1");
+                            queryWrapper1.eq("ext_id", crawlerDouyinMusicTask.getId()).last(" limit 1");
                             MusicInfo musicInfo = musicInfoService.getOne(queryWrapper1);
-                            if(musicInfo != null){
+                            if (musicInfo != null) {
                                 douyinMusicService.parseTag(musicInfo);
                             }
                         }
@@ -159,8 +178,9 @@ public class TestController {
 
         return "success";
     }
+
     @GetMapping(value = "/testdouyinmusic")
-    public String testDouyinMusic(){
+    public String testDouyinMusic() {
         douyinMusicService.getHotMusicList();
         return "success";
     }
@@ -172,7 +192,7 @@ public class TestController {
 
             QueryWrapper<UserAllocation> userAllocationQueryWrapper = new QueryWrapper<>();
             userAllocationQueryWrapper.eq("project_id", projectId);
-           // userAllocationQueryWrapper.eq("account_status", 0);
+            // userAllocationQueryWrapper.eq("account_status", 0);
             List<UserAllocation> list = userAllocationService.list(userAllocationQueryWrapper);
             if (!Check.isNull(list)) {
                 for (UserAllocation allocation : list) {
@@ -205,7 +225,7 @@ public class TestController {
 
             QueryWrapper<UserAllocation> userAllocationQueryWrapper = new QueryWrapper<>();
             userAllocationQueryWrapper.eq("project_id", projectId);
-           // userAllocationQueryWrapper.eq("account_status", 0);
+            // userAllocationQueryWrapper.eq("account_status", 0);
             List<UserAllocation> list = userAllocationService.list(userAllocationQueryWrapper);
             if (!Check.isNull(list)) {
                 for (UserAllocation allocation : list) {

+ 4 - 4
jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java

@@ -73,7 +73,7 @@ public class SampleTest {
 
     @Test
     public void getDepartment() {
-       // wechatCheckinDataService.getCheckinData();
+        // wechatCheckinDataService.getCheckinData();
 
         kuaiShouActionBarTextService.getActionBarText();
 
@@ -89,10 +89,10 @@ public class SampleTest {
         QueryWrapper<CtopOauthToken> queryWrapper = new QueryWrapper<>();
         queryWrapper.eq("media_id", 2);
         List<CtopOauthToken> list = oauthTokenService.list(queryWrapper);
-        // for (CtopOauthToken token : list) {
-        kuaiShouHistoryReportTaskService.createTask(1872430L, "9cd4d43ea697ad9f2aa91fc0f4b921e5", "2020-03-15", "2020-09-15", "history");
+        for (CtopOauthToken token : list) {
+            kuaiShouHistoryReportTaskService.createTask(token.getAccountId(), token.getAccessToken(), "2020-03-17", "2020-09-17", "history");
 
-        // }
+        }
 
     }
 

+ 1 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/mapper/xml/KuaiShouReportDailyMaterialMapper.xml

@@ -117,6 +117,7 @@
         <if test="statDate != null and statDate != '' ">
             and stat_date = #{statDate}
         </if>
+        group by photo_id
     </select>
 
     <select id="getVideoGetByPhotoId" resultType="cn.com.ctop.kuaishou.modules.report.entity.VideoGetvo">

+ 1 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/IKuaiShouHistoryReportTaskService.java

@@ -15,4 +15,5 @@ public interface IKuaiShouHistoryReportTaskService extends IService<KuaiShouHist
 
     void getTaskList();
 
+    void getMd5();
 }

+ 50 - 42
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/impl/BatchServiceImpl.java

@@ -1432,6 +1432,8 @@ public class BatchServiceImpl implements IBatchService {
                 unitJson.put("scene_id", scene_id);
             }
 
+            Integer sceneId = Integer.valueOf(scene_id.get(0).toString());
+
             // 出价
             if (!Check.isNull(requestJson.getLong("bid"))) {
                 unitJson.put("bid", requestJson.getLong("bid"));
@@ -1547,33 +1549,37 @@ public class BatchServiceImpl implements IBatchService {
             if (!Check.isNull(requestJson.getInteger("network"))) {
                 targetJson.put("network", requestJson.getInteger("network"));
             }
+
+
             //设备品牌
             if (!Check.isNull(requestJson.getJSONArray("deviceBrand"))) {
                 targetJson.put("device_brand", requestJson.getJSONArray("deviceBrand"));
             }
-            //设备价格
-            if (!Check.isNull(requestJson.getJSONArray("devicePrice"))) {
-                targetJson.put("device_price", requestJson.getJSONArray("devicePrice"));
-            }
-            //商业兴趣类型
-            if (!Check.isNull(requestJson.getInteger("businessInterestType"))) {
-                targetJson.put("business_interest_type", requestJson.getInteger("businessInterestType"));
-            }
-            // 商业兴趣
-            if (!Check.isNull(requestJson.getJSONArray("businessInterest"))) {
-                targetJson.put("business_interest", requestJson.getJSONArray("businessInterest"));
-            }
-            //网红粉丝
-            if (!Check.isNull(requestJson.getJSONArray("fansStar"))) {
-                targetJson.put("fans_star", requestJson.getJSONArray("fansStar"));
-            }
-            //兴趣视频用户
-            if (!Check.isNull(requestJson.getJSONArray("interestVideo"))) {
-                targetJson.put("interest_video", requestJson.getJSONArray("interestVideo"));
-            }
-            // APP行为-按分类
-            if (!Check.isNull(requestJson.getJSONArray("appInterest"))) {
-                targetJson.put("app_interest", requestJson.getJSONArray("appInterest"));
+            if (sceneId != 5) {
+                //设备价格
+                if (!Check.isNull(requestJson.getJSONArray("devicePrice"))) {
+                    targetJson.put("device_price", requestJson.getJSONArray("devicePrice"));
+                }
+                //商业兴趣类型
+                if (!Check.isNull(requestJson.getInteger("businessInterestType"))) {
+                    targetJson.put("business_interest_type", requestJson.getInteger("businessInterestType"));
+                }
+                // 商业兴趣
+                if (!Check.isNull(requestJson.getJSONArray("businessInterest"))) {
+                    targetJson.put("business_interest", requestJson.getJSONArray("businessInterest"));
+                }
+                //网红粉丝
+                if (!Check.isNull(requestJson.getJSONArray("fansStar"))) {
+                    targetJson.put("fans_star", requestJson.getJSONArray("fansStar"));
+                }
+                //兴趣视频用户
+                if (!Check.isNull(requestJson.getJSONArray("interestVideo"))) {
+                    targetJson.put("interest_video", requestJson.getJSONArray("interestVideo"));
+                }
+                // APP行为-按分类
+                if (!Check.isNull(requestJson.getJSONArray("appInterest"))) {
+                    targetJson.put("app_interest", requestJson.getJSONArray("appInterest"));
+                }
             }
             // APP行为-按APP名称
             if (!Check.isNull(requestJson.getJSONArray("appIds"))) {
@@ -1588,26 +1594,28 @@ public class BatchServiceImpl implements IBatchService {
                 targetJson.put("exclude_population", requestJson.getJSONArray("excludePopulation"));
             }
 
-            JSONObject intelliExtendJson = new JSONObject();
+            if (sceneId != 5) {
+                JSONObject intelliExtendJson = new JSONObject();
 
-            // 开启智能扩量
-            if (!Check.isNull(requestJson.getInteger("isOpen"))) {
-                intelliExtendJson.put("is_open", requestJson.getInteger("isOpen"));
-            }
-            //不可突破年龄
-            if (!Check.isNull(requestJson.getInteger("noAgeBreak"))) {
-                intelliExtendJson.put("no_age_break", requestJson.getInteger("noAgeBreak"));
-            }
-            //不可突破性别
-            if (!Check.isNull(requestJson.getInteger("noGenderBreak"))) {
-                intelliExtendJson.put("no_gender_break", requestJson.getInteger("noGenderBreak"));
-            }
-            // 不可突破地域
-            if (!Check.isNull(requestJson.getInteger("noAreaBreak"))) {
-                intelliExtendJson.put("no_area_break", requestJson.getInteger("noAreaBreak"));
-            }
-            if (!Check.isNull(intelliExtendJson)) {
-                targetJson.put("intelli_extend", intelliExtendJson);
+                // 开启智能扩量
+                if (!Check.isNull(requestJson.getInteger("isOpen"))) {
+                    intelliExtendJson.put("is_open", requestJson.getInteger("isOpen"));
+                }
+                //不可突破年龄
+                if (!Check.isNull(requestJson.getInteger("noAgeBreak"))) {
+                    intelliExtendJson.put("no_age_break", requestJson.getInteger("noAgeBreak"));
+                }
+                //不可突破性别
+                if (!Check.isNull(requestJson.getInteger("noGenderBreak"))) {
+                    intelliExtendJson.put("no_gender_break", requestJson.getInteger("noGenderBreak"));
+                }
+                // 不可突破地域
+                if (!Check.isNull(requestJson.getInteger("noAreaBreak"))) {
+                    intelliExtendJson.put("no_area_break", requestJson.getInteger("noAreaBreak"));
+                }
+                if (!Check.isNull(intelliExtendJson)) {
+                    targetJson.put("intelli_extend", intelliExtendJson);
+                }
             }
 
             unitJson.put("target", targetJson);

+ 55 - 20
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/impl/KuaiShouHistoryReportTaskServiceImpl.java

@@ -30,6 +30,8 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 
 /**
  * 快手历史数据任务记录
@@ -207,6 +209,8 @@ public class KuaiShouHistoryReportTaskServiceImpl extends ServiceImpl<KuaiShouHi
     }
 
 
+    static ExecutorService executorService = Executors.newFixedThreadPool(30);
+
     private void loadFile(String token, KuaiShouHistoryReportTask task) {
         try {
             String url = PropertiesUtils.getConfig("kuaishou_api_url") + KuaishouInterfaceConstant.ASYNC_TASK_DOWNLOAD;
@@ -235,33 +239,62 @@ public class KuaiShouHistoryReportTaskServiceImpl extends ServiceImpl<KuaiShouHi
                 } else if (task.getViewType() == 5) {
                     kuaishouReportDailyMaterialMapper.loadMaterialDailyReport(task.getAccountId(), localPath);
 
-                    List<KuaiShouReportDailyMaterialVo> list = kuaishouReportDailyMaterialMapper.getPhotoList(task.getAccountId(), null);
-                    if (!Check.isNull(list)) {
-                        for (KuaiShouReportDailyMaterialVo materialVo : list) {
-                            Long id = materialVo.getId();
+
+                }
+
+
+            }
+            task.setTaskStatus(4);
+
+        } catch (Exception e) {
+            e.printStackTrace();
+            task.setTaskStatus(5);
+        }
+        this.updateById(task);
+
+    }
+
+
+    @Autowired
+    private ICtopOauthTokenService oauthTokenService;
+
+    @Override
+    public void getMd5() {
+
+        QueryWrapper<CtopOauthToken> queryWrapper = new QueryWrapper<>();
+        queryWrapper.eq("media_id", 2);
+        List<CtopOauthToken> list = oauthTokenService.list(queryWrapper);
+        for (CtopOauthToken token : list) {
+
+
+            List<KuaiShouReportDailyMaterialVo> photoList = kuaishouReportDailyMaterialMapper.getPhotoList(token.getAccountId(), null);
+            if (!Check.isNull(photoList)) {
+                for (KuaiShouReportDailyMaterialVo materialVo : photoList) {
+                    executorService.submit(new Runnable() {
+                        @Override
+                        public void run() {
+                            //        Long id = materialVo.getId();
                             Long photoId = materialVo.getPhotoId();
                             VideoGetvo videoGetvo = kuaishouReportDailyMaterialMapper.getVideoGetByPhotoId(photoId);
                             if (!Check.isNull(videoGetvo)) {
+                                QueryWrapper<KuaiShouReportDailyMaterial> updateQueryWrapper = new QueryWrapper<>();
+                                updateQueryWrapper.eq("photo_id", photoId);
                                 KuaiShouReportDailyMaterial material = new KuaiShouReportDailyMaterial();
-                                material.setId(id);
                                 material.setSignature(videoGetvo.getSignature());
                                 material.setChannelType(videoGetvo.getChannelType());
-                                kuaishouReportDailyMaterialMapper.updateById(material);
+                                kuaishouReportDailyMaterialMapper.update(material, updateQueryWrapper);
+                                log.info("修改成功:photoId:{}",photoId);
+                                // kuaishouReportDailyMaterialMapper.updateById(material);
                             }
                         }
-
-                    }
+                    });
                 }
 
-
             }
-            task.setTaskStatus(4);
 
-        } catch (Exception e) {
-            e.printStackTrace();
-            task.setTaskStatus(5);
+
         }
-        this.updateById(task);
+
 
     }
 
@@ -270,21 +303,23 @@ public class KuaiShouHistoryReportTaskServiceImpl extends ServiceImpl<KuaiShouHi
 
         historyTypeList = new ArrayList<>();
 
-        historyTypeList.add(1);
+    /*    historyTypeList.add(1);
         historyTypeList.add(2);
         historyTypeList.add(3);
         historyTypeList.add(4);
-      /*  historyTypeList.add(5);*/
-        historyTypeList.add(10);
+        historyTypeList.add(10);*/
+        historyTypeList.add(5);
+
     }
 
     private void getDailyTypeList() {
         dailyTypeList = new ArrayList<>();
-        dailyTypeList.add(2);
+        dailyTypeList.add(5);
+        /*dailyTypeList.add(2);
         dailyTypeList.add(3);
         dailyTypeList.add(4);
-       /* dailyTypeList.add(5);*/
-        dailyTypeList.add(10);
+
+        dailyTypeList.add(10);*/
     }
 
 }