Просмотр исходного кода

修改巨量引擎爬虫逻辑

songyinghao 5 лет назад
Родитель
Сommit
0381846039

+ 20 - 14
module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/service/impl/OceanEngineServiceImpl.java

@@ -29,6 +29,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
+import java.io.FileInputStream;
 import java.net.HttpURLConnection;
 import java.net.URL;
 import java.util.*;
@@ -83,12 +84,14 @@ public class OceanEngineServiceImpl implements IOceanEngineService {
                             if(material.getVideoUrl() == null){
                                 material.setVideoUrl(getVideoUrl(material.getVid()));
                             }
-//                            material.setVideoUrl(getVideoUrl(material.getVid()));
-//                            material.setCoverImage(FfmpegUtils.getVideoFirstImage(material.getVideoUrl()));
-//                            if(material.getCoverImage() == null){
-//                                Thread.sleep(1000);
-//                                material.setCoverImage(FfmpegUtils.getVideoFirstImage(material.getVideoUrl()));
-//                            }
+                            String base64Code = FfmpegUtils.getVideoFirstImage(material.getVideoUrl());
+                            if(base64Code == null){
+                                Thread.sleep(1000);
+                                base64Code = FfmpegUtils.getVideoFirstImage(material.getVideoUrl());
+                            }
+                            EffectCase.base64ToFile("/mnt/crawler/hotmat/",base64Code,material.getVid()+".jpg");
+                            String coverUrl = OSSUtils.uploadFile2OSS(new FileInputStream("/mnt/crawler/hotmat/"+material.getVid()+".jpg"),"/ocean-engine/hot-material/",material.getVid()+".jpg");
+                            material.setCoverUrl(coverUrl);
                             hotMaterialService.saveOrUpdate(material);
                         }
                         hotMaterialHandler(page+1,appCode,appName);
@@ -151,11 +154,14 @@ public class OceanEngineServiceImpl implements IOceanEngineService {
                     if (effectCase.getVideoUrl() == null){
                         effectCase.setVideoUrl(getVideoUrl(effectCase.getVid()));
                     }
-//                    effectCase.setCoverImage(FfmpegUtils.getVideoFirstImage(effectCase.getVideoUrl()));
-//                    if (effectCase.getCoverImage() == null){
-//                        Thread.sleep(1000);
-//                        effectCase.setCoverImage(FfmpegUtils.getVideoFirstImage(effectCase.getVideoUrl()));
-//                    }
+                    String base64Code = FfmpegUtils.getVideoFirstImage(effectCase.getVideoUrl());
+                    if(base64Code == null){
+                        Thread.sleep(1000);
+                        base64Code = FfmpegUtils.getVideoFirstImage(effectCase.getVideoUrl());
+                    }
+                    EffectCase.base64ToFile("/mnt/crawler/effectcase/",base64Code,effectCase.getVid()+".jpg");
+                    String coverUrl = OSSUtils.uploadFile2OSS(new FileInputStream("/mnt/crawler/effectcase/"+effectCase.getVid()+".jpg"),"/ocean-engine/effect-case/",effectCase.getVid()+".jpg");
+                    effectCase.setCoverUrl(coverUrl);
                     effectCase.setCreativeLabel(getCase.getString("creative_label"));
                 }
                 effectCaseService.saveOrUpdate(effectCase);
@@ -193,9 +199,9 @@ public class OceanEngineServiceImpl implements IOceanEngineService {
                                 Thread.sleep(1000);
                                 base64Code = FfmpegUtils.getVideoFirstImage(douyinHot.getUrl());
                             }
-//                            EffectCase.base64ToFile(,);
-                            EffectCase.base64ToFile("",base64Code,douyinHot.getMid()+".jpg");
-//                            OSSUtils.uploadFile2OSS()
+                            EffectCase.base64ToFile("/mnt/crawler/douyinhot/",base64Code,douyinHot.getMid()+".jpg");
+                            String coverUrl = OSSUtils.uploadFile2OSS(new FileInputStream("/mnt/crawler/douyinhot/"+douyinHot.getMid()+".jpg"),"/ocean-engine/doyin-hot/",douyinHot.getMid()+".jpg");
+                            douyinHot.setCoverUrl(coverUrl);
                             douyinHotService.saveOrUpdate(douyinHot);
                         }
                         douyinHotHandler(page+1,listType);