Selaa lähdekoodia

修改抖音音乐爬虫逻辑

xuzuoyun 4 vuotta sitten
vanhempi
commit
c1f3a887d8

+ 3 - 3
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/TestController.java

@@ -133,7 +133,7 @@ public class TestController {
 
         List<CrawlerDouyinMusicTask> list = crawlerDouyinMusicTaskService.list(queryWrapper);
 
-        while (list != null) {
+        while (list != null && list.size() > 0) {
             for (CrawlerDouyinMusicTask crawlerDouyinMusicTask : list) {
                 MusicInfo musicInfo = new MusicInfo();
                 musicInfo.setExtId(crawlerDouyinMusicTask.getId());
@@ -157,7 +157,7 @@ public class TestController {
                     QueryWrapper<CrawlerDouyinMusicTask> queryWrapper = new QueryWrapper<>();
                     queryWrapper.eq("status", "init").likeLeft("id", j).orderByDesc("id").last(" limit 20");
                     List<CrawlerDouyinMusicTask> list = crawlerDouyinMusicTaskService.list(queryWrapper);
-                    while (list != null) {
+                    while (list != null && list.size() > 0) {
                         log.info("【同步歌曲】第" + j + "线程正在处理:" + list.size() + "条数据");
                         for (CrawlerDouyinMusicTask crawlerDouyinMusicTask : list) {
                             MusicInfo musicInfo = new MusicInfo();
@@ -184,7 +184,7 @@ public class TestController {
                     QueryWrapper<CrawlerDouyinMusicTask> queryWrapper = new QueryWrapper<>();
                     queryWrapper.eq("status", "done").likeLeft("id", j).orderByDesc("id").last(" limit 20");
                     List<CrawlerDouyinMusicTask> list = crawlerDouyinMusicTaskService.list(queryWrapper);
-                    while (list != null) {
+                    while (list != null && list.size() > 0) {
                         log.info("【解析分类】第" + j + "线程正在处理:" + list.size() + "条数据");
                         for (CrawlerDouyinMusicTask crawlerDouyinMusicTask : list) {
                             QueryWrapper<MusicInfo> queryWrapper1 = new QueryWrapper<>();

+ 10 - 2
module-crawler/src/main/java/cn/com/ctop/crawler/modules/douyin/service/impl/DouyinMusicServiceImpl.java

@@ -112,6 +112,10 @@ public class DouyinMusicServiceImpl implements DouyinMusicService {
             crawlerDouyinMusicTask.setStatus("parsed");
             crawlerDouyinMusicTaskService.updateById(crawlerDouyinMusicTask);
         }catch (Exception e){
+            CrawlerDouyinMusicTask crawlerDouyinMusicTask = new CrawlerDouyinMusicTask();
+            crawlerDouyinMusicTask.setId(musicInfo.getExtId());
+            crawlerDouyinMusicTask.setStatus("error");
+            crawlerDouyinMusicTaskService.updateById(crawlerDouyinMusicTask);
             e.printStackTrace();
         }
     }
@@ -153,7 +157,7 @@ public class DouyinMusicServiceImpl implements DouyinMusicService {
                     for (int i = 0; i < size; i++){
                         CrawlerDouyinMusicTask crawlerDouyinMusicTask = crawlerDouyinMusicTaskService.getById(dataNode.get(i).asLong());
                         if(crawlerDouyinMusicTask == null){
-                            crawlerDouyinMusicTask = new CrawlerDouyinMusicTask(); 
+                            crawlerDouyinMusicTask = new CrawlerDouyinMusicTask();
                             crawlerDouyinMusicTask.setId(dataNode.get(i).asLong());
                             crawlerDouyinMusicTask.setStatus("init");
                             crawlerDouyinMusicTaskService.replaceInto(crawlerDouyinMusicTask);
@@ -162,6 +166,10 @@ public class DouyinMusicServiceImpl implements DouyinMusicService {
                 }
             }
         } catch (IOException e) {
+            CrawlerDouyinMusicTask crawlerDouyinMusic = new CrawlerDouyinMusicTask();
+            crawlerDouyinMusic.setId(musicInfo.getExtId());
+            crawlerDouyinMusic.setStatus("error");
+            crawlerDouyinMusicTaskService.updateById(crawlerDouyinMusic);
             e.printStackTrace();
         }
         return musicInfo;
@@ -169,7 +177,7 @@ public class DouyinMusicServiceImpl implements DouyinMusicService {
     public static void main(String[] args){
         DouyinMusicServiceImpl douyinMusicService = new DouyinMusicServiceImpl();
         MusicInfo musicInfo = new MusicInfo();
-        musicInfo.setExtId(6705193173741733889L);
+        musicInfo.setExtId(12646L);
         douyinMusicService.getMusicInfo(musicInfo);
     }
 }

+ 1 - 1
module-crawler/src/main/java/cn/com/ctop/crawler/modules/taobao/service/impl/TaobaoJhsbybtServiceImpl.java

@@ -58,7 +58,7 @@ public class TaobaoJhsbybtServiceImpl implements TaobaoJhsbybtService {
 
     public static void main(String[] args) {
         try {
-            PrintStream out = new PrintStream("D:/bybt0916.csv");
+            PrintStream out = new PrintStream("D:/bybt0929.csv");
             System.setOut(out);
         } catch (FileNotFoundException e) {
             e.printStackTrace();