xuzuoyun 4 лет назад
Родитель
Сommit
ed53e2788c
18 измененных файлов с 437 добавлено и 3 удалено
  1. 3 1
      jeecg-boot-module-system/src/main/java/org/jeecg/config/ShiroConfig.java
  2. 88 0
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/TestController.java
  3. 22 0
      module-common/src/main/java/cn/com/ctop/common/module/entity/MusicInfo.java
  4. 20 0
      module-common/src/main/java/cn/com/ctop/common/module/entity/MusicType.java
  5. 7 0
      module-common/src/main/java/cn/com/ctop/common/module/mapper/MusicInfoMapper.java
  6. 8 0
      module-common/src/main/java/cn/com/ctop/common/module/mapper/MusicTypeMapper.java
  7. 7 0
      module-common/src/main/java/cn/com/ctop/common/module/service/MusicInfoService.java
  8. 8 0
      module-common/src/main/java/cn/com/ctop/common/module/service/MusicTypeService.java
  9. 11 0
      module-common/src/main/java/cn/com/ctop/common/module/service/impl/MusicInfoServiceImpl.java
  10. 14 0
      module-common/src/main/java/cn/com/ctop/common/module/service/impl/MusicTypeServiceImpl.java
  11. 15 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/core/entity/CrawlerDouyinMusicTask.java
  12. 8 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/core/mapper/CrawlerDouyinMusicTaskMapper.java
  13. 9 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/core/mapper/xml/CrawlerDouyinMusicTaskMapper.xml
  14. 8 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/core/service/CrawlerDouyinMusicTaskService.java
  15. 18 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/core/service/impl/CrawlerDouyinMusicServiceImpl.java
  16. 9 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/douyin/service/DouyinMusicService.java
  17. 175 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/douyin/service/impl/DouyinMusicServiceImpl.java
  18. 7 2
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/taobao/service/impl/TaobaoJhsbybtServiceImpl.java

+ 3 - 1
jeecg-boot-module-system/src/main/java/org/jeecg/config/ShiroConfig.java

@@ -155,7 +155,9 @@ public class ShiroConfig {
         filterChainDefinitionMap.put("/test/getGroupData/**", "anon");
         filterChainDefinitionMap.put("/test/getVideoAndImage/**", "anon");
         filterChainDefinitionMap.put("/test/gerCreative/**", "anon");
-
+        filterChainDefinitionMap.put("/test/testdouyinmusic","anon");
+        filterChainDefinitionMap.put("/test/syncDouyinMusic","anon");
+        filterChainDefinitionMap.put("/test/parseDouyinMusic","anon");
         filterChainDefinitionMap.put("/report/kuaishouReportDailyAgent/get", "anon");
 
         filterChainDefinitionMap.put("/appium/appiumTask/run/login", "anon");

+ 88 - 0
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/TestController.java

@@ -5,6 +5,9 @@ import cn.com.ctop.common.module.mapper.CtopOauthTokenMapper;
 import cn.com.ctop.common.module.service.*;
 import cn.com.ctop.common.module.utils.*;
 import cn.com.ctop.common.module.vo.ResFileDTO;
+import cn.com.ctop.crawler.modules.core.entity.CrawlerDouyinMusicTask;
+import cn.com.ctop.crawler.modules.core.service.CrawlerDouyinMusicTaskService;
+import cn.com.ctop.crawler.modules.douyin.service.DouyinMusicService;
 import cn.com.ctop.kuaishou.modules.batch.service.*;
 import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouWebInterfaceService;
 import cn.com.ctop.toutiao.modules.material.service.IByteDanceAdvertiserDataService;
@@ -71,11 +74,96 @@ public class TestController {
     private IKuaishouInterfaceService2 kuaishouInterfaceService2;
     @Autowired
     private IUserAllocationService userAllocationService;
+    @Autowired
+    private DouyinMusicService douyinMusicService;
+    @Autowired
+    private CrawlerDouyinMusicTaskService crawlerDouyinMusicTaskService;
+    @Autowired
+    private MusicInfoService musicInfoService;
 
     static ExecutorService executorService = Executors.newFixedThreadPool(15);
     static ExecutorService videoService = Executors.newFixedThreadPool(5);
     static ExecutorService suzhaoService = Executors.newFixedThreadPool(5);
+    @GetMapping(value = "/syncDouyinMusicAsc")
+    public String syncDouyinMusicAsc(){
+        QueryWrapper<CrawlerDouyinMusicTask> queryWrapper = new QueryWrapper<>();
+        queryWrapper.eq("status","init").orderByAsc("id").last(" limit 20");
+
+        List<CrawlerDouyinMusicTask> list = crawlerDouyinMusicTaskService.list(queryWrapper);
+
+        while (list != null){
+            for(CrawlerDouyinMusicTask crawlerDouyinMusicTask : list){
+                MusicInfo musicInfo = new MusicInfo();
+                musicInfo.setExtId(crawlerDouyinMusicTask.getId());
+                musicInfo = douyinMusicService.getMusicInfo(musicInfo);
+                musicInfoService.save(musicInfo);
+                crawlerDouyinMusicTask.setStatus("done");
+                crawlerDouyinMusicTaskService.updateById(crawlerDouyinMusicTask);
+            }
+            list = crawlerDouyinMusicTaskService.list(queryWrapper);
+        }
+        return "success";
+    }
+    @GetMapping(value = "/syncDouyinMusic")
+    public String syncDouyinMusic(){
+        for(int i =0;i<10;i++){
+            final int j = i;
+            Thread thread = new Thread(){
+                @Override
+                public void run() {
+                    QueryWrapper<CrawlerDouyinMusicTask> queryWrapper = new QueryWrapper<>();
+                    queryWrapper.eq("status","init").likeLeft("id",j).orderByDesc("id").last(" limit 20");
+                    List<CrawlerDouyinMusicTask> list = crawlerDouyinMusicTaskService.list(queryWrapper);
+                    while (list != null){
+                        log.info("【同步歌曲】第"+j+"线程正在处理:"+list.size()+"条数据");
+                        for(CrawlerDouyinMusicTask crawlerDouyinMusicTask : list){
+                            MusicInfo musicInfo = new MusicInfo();
+                            musicInfo.setExtId(crawlerDouyinMusicTask.getId());
+                            douyinMusicService.getMusicInfo(musicInfo);
+                        }
+                        list = crawlerDouyinMusicTaskService.list(queryWrapper);
+                    }
+                }
+            };
+            thread.start();
+        }
 
+        return "success";
+    }
+    @GetMapping(value = "/parseDouyinMusic")
+    public String parseDouyinMusic(){
+        for(int i =0;i<10;i++){
+            final int j = i;
+            Thread thread = new Thread(){
+                @Override
+                public void run() {
+                    QueryWrapper<CrawlerDouyinMusicTask> queryWrapper = new QueryWrapper<>();
+                    queryWrapper.eq("status","done").likeLeft("id",j).orderByDesc("id").last(" limit 20");
+                    List<CrawlerDouyinMusicTask> list = crawlerDouyinMusicTaskService.list(queryWrapper);
+                    while (list != null){
+                        log.info("【解析分类】第"+j+"线程正在处理:"+list.size()+"条数据");
+                        for(CrawlerDouyinMusicTask crawlerDouyinMusicTask : list){
+                            QueryWrapper<MusicInfo> queryWrapper1 = new QueryWrapper<>();
+                            queryWrapper1.eq("ext_id",crawlerDouyinMusicTask.getId()).last(" limit 1");
+                            MusicInfo musicInfo = musicInfoService.getOne(queryWrapper1);
+                            if(musicInfo != null){
+                                douyinMusicService.parseTag(musicInfo);
+                            }
+                        }
+                        list = crawlerDouyinMusicTaskService.list(queryWrapper);
+                    }
+                }
+            };
+            thread.start();
+        }
+
+        return "success";
+    }
+    @GetMapping(value = "/testdouyinmusic")
+    public String testDouyinMusic(){
+        douyinMusicService.getHotMusicList();
+        return "success";
+    }
 
     @GetMapping(value = "/getDailyDataByProjectId")
     public String getDailyDataByProjectId(Long projectId, String startDate, String endDate) {

+ 22 - 0
module-common/src/main/java/cn/com/ctop/common/module/entity/MusicInfo.java

@@ -0,0 +1,22 @@
+package cn.com.ctop.common.module.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+@Data
+@TableName("ctop_music_info")
+public class MusicInfo {
+    @TableId(value = "id", type = IdType.AUTO)
+    private Long id;
+    private String name;
+    private String author;
+    private Integer usedCount;
+    private String musicUrl;
+    private String coverUrl;
+    private Integer duration;
+    private String source;
+    private Long extId;
+    private String extInfo;
+}

+ 20 - 0
module-common/src/main/java/cn/com/ctop/common/module/entity/MusicType.java

@@ -0,0 +1,20 @@
+package cn.com.ctop.common.module.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+@Data
+@TableName("ctop_music_type")
+public class MusicType {
+    @TableId(value = "id", type = IdType.INPUT)
+    private Long id;
+    private String type;
+    private String typeValue;
+    private String extType;
+    private String extTypeValue;
+    private Long musicId;
+    private String musicName;
+    private String author;
+}

+ 7 - 0
module-common/src/main/java/cn/com/ctop/common/module/mapper/MusicInfoMapper.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.common.module.mapper;
+
+import cn.com.ctop.common.module.entity.MusicInfo;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+public interface MusicInfoMapper extends BaseMapper<MusicInfo> {
+}

+ 8 - 0
module-common/src/main/java/cn/com/ctop/common/module/mapper/MusicTypeMapper.java

@@ -0,0 +1,8 @@
+package cn.com.ctop.common.module.mapper;
+
+import cn.com.ctop.common.module.entity.MusicInfo;
+import cn.com.ctop.common.module.entity.MusicType;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+public interface MusicTypeMapper extends BaseMapper<MusicType> {
+}

+ 7 - 0
module-common/src/main/java/cn/com/ctop/common/module/service/MusicInfoService.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.common.module.service;
+
+import cn.com.ctop.common.module.entity.MusicInfo;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+public interface MusicInfoService extends IService<MusicInfo> {
+}

+ 8 - 0
module-common/src/main/java/cn/com/ctop/common/module/service/MusicTypeService.java

@@ -0,0 +1,8 @@
+package cn.com.ctop.common.module.service;
+
+import cn.com.ctop.common.module.entity.MusicInfo;
+import cn.com.ctop.common.module.entity.MusicType;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+public interface MusicTypeService extends IService<MusicType> {
+}

+ 11 - 0
module-common/src/main/java/cn/com/ctop/common/module/service/impl/MusicInfoServiceImpl.java

@@ -0,0 +1,11 @@
+package cn.com.ctop.common.module.service.impl;
+
+import cn.com.ctop.common.module.entity.MusicInfo;
+import cn.com.ctop.common.module.mapper.MusicInfoMapper;
+import cn.com.ctop.common.module.service.MusicInfoService;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.springframework.stereotype.Service;
+
+@Service
+public class MusicInfoServiceImpl extends ServiceImpl<MusicInfoMapper, MusicInfo> implements MusicInfoService {
+}

+ 14 - 0
module-common/src/main/java/cn/com/ctop/common/module/service/impl/MusicTypeServiceImpl.java

@@ -0,0 +1,14 @@
+package cn.com.ctop.common.module.service.impl;
+
+import cn.com.ctop.common.module.entity.MusicInfo;
+import cn.com.ctop.common.module.entity.MusicType;
+import cn.com.ctop.common.module.mapper.MusicInfoMapper;
+import cn.com.ctop.common.module.mapper.MusicTypeMapper;
+import cn.com.ctop.common.module.service.MusicInfoService;
+import cn.com.ctop.common.module.service.MusicTypeService;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.springframework.stereotype.Service;
+
+@Service
+public class MusicTypeServiceImpl extends ServiceImpl<MusicTypeMapper, MusicType> implements MusicTypeService {
+}

+ 15 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/core/entity/CrawlerDouyinMusicTask.java

@@ -0,0 +1,15 @@
+package cn.com.ctop.crawler.modules.core.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+@Data
+@TableName("ctop_crawler_douyin_music_task")
+public class CrawlerDouyinMusicTask {
+    @TableId(value = "id", type = IdType.INPUT)
+    private Long id;
+    private String status;
+
+}

+ 8 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/core/mapper/CrawlerDouyinMusicTaskMapper.java

@@ -0,0 +1,8 @@
+package cn.com.ctop.crawler.modules.core.mapper;
+
+import cn.com.ctop.crawler.modules.core.entity.CrawlerDouyinMusicTask;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+public interface CrawlerDouyinMusicTaskMapper extends BaseMapper<CrawlerDouyinMusicTask> {
+    public void replaceInto(CrawlerDouyinMusicTask crawlerDouyinMusicTask);
+}

+ 9 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/core/mapper/xml/CrawlerDouyinMusicTaskMapper.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="cn.com.ctop.crawler.modules.core.mapper.CrawlerDouyinMusicTaskMapper">
+
+
+    <insert id="replaceInto" parameterType="cn.com.ctop.crawler.modules.core.entity.CrawlerDouyinMusicTask">
+		replace into ctop_crawler_douyin_music_task (id,status) values(#{id},#{status})
+	</insert>
+</mapper>

+ 8 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/core/service/CrawlerDouyinMusicTaskService.java

@@ -0,0 +1,8 @@
+package cn.com.ctop.crawler.modules.core.service;
+
+import cn.com.ctop.crawler.modules.core.entity.CrawlerDouyinMusicTask;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+public interface CrawlerDouyinMusicTaskService extends IService<CrawlerDouyinMusicTask> {
+    public void replaceInto(CrawlerDouyinMusicTask crawlerDouyinMusicTask);
+}

+ 18 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/core/service/impl/CrawlerDouyinMusicServiceImpl.java

@@ -0,0 +1,18 @@
+package cn.com.ctop.crawler.modules.core.service.impl;
+
+import cn.com.ctop.crawler.modules.core.entity.CrawlerDouyinMusicTask;
+import cn.com.ctop.crawler.modules.core.mapper.CrawlerDouyinMusicTaskMapper;
+import cn.com.ctop.crawler.modules.core.service.CrawlerDouyinMusicTaskService;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+@Service
+public class CrawlerDouyinMusicServiceImpl extends ServiceImpl<CrawlerDouyinMusicTaskMapper, CrawlerDouyinMusicTask> implements CrawlerDouyinMusicTaskService {
+    @Autowired
+    private CrawlerDouyinMusicTaskMapper crawlerDouyinMusicTaskMapper;
+    @Override
+    public void replaceInto(CrawlerDouyinMusicTask crawlerDouyinMusicTask) {
+        crawlerDouyinMusicTaskMapper.replaceInto(crawlerDouyinMusicTask);
+    }
+}

+ 9 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/douyin/service/DouyinMusicService.java

@@ -0,0 +1,9 @@
+package cn.com.ctop.crawler.modules.douyin.service;
+
+import cn.com.ctop.common.module.entity.MusicInfo;
+
+public interface DouyinMusicService {
+    public void getHotMusicList();
+    public MusicInfo getMusicInfo(MusicInfo musicInfo);
+    public void parseTag(MusicInfo musicInfo);
+}

Разница между файлами не показана из-за своего большого размера
+ 175 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/douyin/service/impl/DouyinMusicServiceImpl.java


+ 7 - 2
module-crawler/src/main/java/cn/com/ctop/crawler/modules/taobao/service/impl/TaobaoJhsbybtServiceImpl.java

@@ -58,7 +58,7 @@ public class TaobaoJhsbybtServiceImpl implements TaobaoJhsbybtService {
 
     public static void main(String[] args) {
         try {
-            PrintStream out = new PrintStream("D:/bybt0904.csv");
+            PrintStream out = new PrintStream("D:/bybt0916.csv");
             System.setOut(out);
         } catch (FileNotFoundException e) {
             e.printStackTrace();
@@ -116,10 +116,15 @@ public class TaobaoJhsbybtServiceImpl implements TaobaoJhsbybtService {
                             couponAmount);
                 }
                 i++;
-                Thread.sleep(3000L);
+
             } catch (Exception e) {
                 e.printStackTrace();
             }
+            try {
+                Thread.sleep(3000L);
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
         }
 
     }