|
@@ -5,6 +5,9 @@ import cn.com.ctop.common.module.mapper.CtopOauthTokenMapper;
|
|
|
import cn.com.ctop.common.module.service.*;
|
|
|
import cn.com.ctop.common.module.utils.*;
|
|
|
import cn.com.ctop.common.module.vo.ResFileDTO;
|
|
|
+import cn.com.ctop.crawler.modules.core.entity.CrawlerDouyinMusicTask;
|
|
|
+import cn.com.ctop.crawler.modules.core.service.CrawlerDouyinMusicTaskService;
|
|
|
+import cn.com.ctop.crawler.modules.douyin.service.DouyinMusicService;
|
|
|
import cn.com.ctop.kuaishou.modules.batch.service.*;
|
|
|
import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouWebInterfaceService;
|
|
|
import cn.com.ctop.toutiao.modules.material.service.IByteDanceAdvertiserDataService;
|
|
@@ -71,11 +74,96 @@ public class TestController {
|
|
|
private IKuaishouInterfaceService2 kuaishouInterfaceService2;
|
|
|
@Autowired
|
|
|
private IUserAllocationService userAllocationService;
|
|
|
+ @Autowired
|
|
|
+ private DouyinMusicService douyinMusicService;
|
|
|
+ @Autowired
|
|
|
+ private CrawlerDouyinMusicTaskService crawlerDouyinMusicTaskService;
|
|
|
+ @Autowired
|
|
|
+ private MusicInfoService musicInfoService;
|
|
|
|
|
|
static ExecutorService executorService = Executors.newFixedThreadPool(15);
|
|
|
static ExecutorService videoService = Executors.newFixedThreadPool(5);
|
|
|
static ExecutorService suzhaoService = Executors.newFixedThreadPool(5);
|
|
|
+ @GetMapping(value = "/syncDouyinMusicAsc")
|
|
|
+ public String syncDouyinMusicAsc(){
|
|
|
+ QueryWrapper<CrawlerDouyinMusicTask> queryWrapper = new QueryWrapper<>();
|
|
|
+ queryWrapper.eq("status","init").orderByAsc("id").last(" limit 20");
|
|
|
+
|
|
|
+ List<CrawlerDouyinMusicTask> list = crawlerDouyinMusicTaskService.list(queryWrapper);
|
|
|
+
|
|
|
+ while (list != null){
|
|
|
+ for(CrawlerDouyinMusicTask crawlerDouyinMusicTask : list){
|
|
|
+ MusicInfo musicInfo = new MusicInfo();
|
|
|
+ musicInfo.setExtId(crawlerDouyinMusicTask.getId());
|
|
|
+ musicInfo = douyinMusicService.getMusicInfo(musicInfo);
|
|
|
+ musicInfoService.save(musicInfo);
|
|
|
+ crawlerDouyinMusicTask.setStatus("done");
|
|
|
+ crawlerDouyinMusicTaskService.updateById(crawlerDouyinMusicTask);
|
|
|
+ }
|
|
|
+ list = crawlerDouyinMusicTaskService.list(queryWrapper);
|
|
|
+ }
|
|
|
+ return "success";
|
|
|
+ }
|
|
|
+ @GetMapping(value = "/syncDouyinMusic")
|
|
|
+ public String syncDouyinMusic(){
|
|
|
+ for(int i =0;i<10;i++){
|
|
|
+ final int j = i;
|
|
|
+ Thread thread = new Thread(){
|
|
|
+ @Override
|
|
|
+ public void run() {
|
|
|
+ QueryWrapper<CrawlerDouyinMusicTask> queryWrapper = new QueryWrapper<>();
|
|
|
+ queryWrapper.eq("status","init").likeLeft("id",j).orderByDesc("id").last(" limit 20");
|
|
|
+ List<CrawlerDouyinMusicTask> list = crawlerDouyinMusicTaskService.list(queryWrapper);
|
|
|
+ while (list != null){
|
|
|
+ log.info("【同步歌曲】第"+j+"线程正在处理:"+list.size()+"条数据");
|
|
|
+ for(CrawlerDouyinMusicTask crawlerDouyinMusicTask : list){
|
|
|
+ MusicInfo musicInfo = new MusicInfo();
|
|
|
+ musicInfo.setExtId(crawlerDouyinMusicTask.getId());
|
|
|
+ douyinMusicService.getMusicInfo(musicInfo);
|
|
|
+ }
|
|
|
+ list = crawlerDouyinMusicTaskService.list(queryWrapper);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ };
|
|
|
+ thread.start();
|
|
|
+ }
|
|
|
|
|
|
+ return "success";
|
|
|
+ }
|
|
|
+ @GetMapping(value = "/parseDouyinMusic")
|
|
|
+ public String parseDouyinMusic(){
|
|
|
+ for(int i =0;i<10;i++){
|
|
|
+ final int j = i;
|
|
|
+ Thread thread = new Thread(){
|
|
|
+ @Override
|
|
|
+ public void run() {
|
|
|
+ QueryWrapper<CrawlerDouyinMusicTask> queryWrapper = new QueryWrapper<>();
|
|
|
+ queryWrapper.eq("status","done").likeLeft("id",j).orderByDesc("id").last(" limit 20");
|
|
|
+ List<CrawlerDouyinMusicTask> list = crawlerDouyinMusicTaskService.list(queryWrapper);
|
|
|
+ while (list != null){
|
|
|
+ log.info("【解析分类】第"+j+"线程正在处理:"+list.size()+"条数据");
|
|
|
+ for(CrawlerDouyinMusicTask crawlerDouyinMusicTask : list){
|
|
|
+ QueryWrapper<MusicInfo> queryWrapper1 = new QueryWrapper<>();
|
|
|
+ queryWrapper1.eq("ext_id",crawlerDouyinMusicTask.getId()).last(" limit 1");
|
|
|
+ MusicInfo musicInfo = musicInfoService.getOne(queryWrapper1);
|
|
|
+ if(musicInfo != null){
|
|
|
+ douyinMusicService.parseTag(musicInfo);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ list = crawlerDouyinMusicTaskService.list(queryWrapper);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ };
|
|
|
+ thread.start();
|
|
|
+ }
|
|
|
+
|
|
|
+ return "success";
|
|
|
+ }
|
|
|
+ @GetMapping(value = "/testdouyinmusic")
|
|
|
+ public String testDouyinMusic(){
|
|
|
+ douyinMusicService.getHotMusicList();
|
|
|
+ return "success";
|
|
|
+ }
|
|
|
|
|
|
@GetMapping(value = "/getDailyDataByProjectId")
|
|
|
public String getDailyDataByProjectId(Long projectId, String startDate, String endDate) {
|