|
@@ -2,8 +2,10 @@ package cn.com.ctop.crawler.modules.oceanengine.service.impl;
|
|
|
|
|
|
import cn.com.ctop.common.module.utils.Base64Utils;
|
|
import cn.com.ctop.common.module.utils.Base64Utils;
|
|
import cn.com.ctop.common.module.utils.HttpUtils2;
|
|
import cn.com.ctop.common.module.utils.HttpUtils2;
|
|
|
|
+import cn.com.ctop.common.module.utils.OSSUtils;
|
|
import cn.com.ctop.crawler.modules.core.util.FateadmHttpUtil;
|
|
import cn.com.ctop.crawler.modules.core.util.FateadmHttpUtil;
|
|
import cn.com.ctop.crawler.modules.core.util.FateadmUtil;
|
|
import cn.com.ctop.crawler.modules.core.util.FateadmUtil;
|
|
|
|
+import cn.com.ctop.crawler.modules.core.util.FfmpegUtils;
|
|
import cn.com.ctop.crawler.modules.oceanengine.entity.DouyinHot;
|
|
import cn.com.ctop.crawler.modules.oceanengine.entity.DouyinHot;
|
|
import cn.com.ctop.crawler.modules.oceanengine.entity.EffectCase;
|
|
import cn.com.ctop.crawler.modules.oceanengine.entity.EffectCase;
|
|
import cn.com.ctop.crawler.modules.oceanengine.entity.HotMaterial;
|
|
import cn.com.ctop.crawler.modules.oceanengine.entity.HotMaterial;
|
|
@@ -27,6 +29,8 @@ import org.springframework.beans.factory.annotation.Autowired;
|
|
import org.springframework.beans.factory.annotation.Value;
|
|
import org.springframework.beans.factory.annotation.Value;
|
|
import org.springframework.stereotype.Service;
|
|
import org.springframework.stereotype.Service;
|
|
|
|
|
|
|
|
+import java.io.FileInputStream;
|
|
|
|
+import java.io.InputStream;
|
|
import java.net.HttpURLConnection;
|
|
import java.net.HttpURLConnection;
|
|
import java.net.URL;
|
|
import java.net.URL;
|
|
import java.util.*;
|
|
import java.util.*;
|
|
@@ -63,7 +67,6 @@ public class OceanEngineServiceImpl implements IOceanEngineService {
|
|
@Override
|
|
@Override
|
|
public void hotMaterialHandler(int page, int appCode, String appName){
|
|
public void hotMaterialHandler(int page, int appCode, String appName){
|
|
String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_material/list?material_type=3&order_by=click_show_rate&period_type=30&aggr_app_code="+appCode+"&page="+page+"&limit=20");
|
|
String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_material/list?material_type=3&order_by=click_show_rate&period_type=30&aggr_app_code="+appCode+"&page="+page+"&limit=20");
|
|
- System.out.println(result);
|
|
|
|
ObjectMapper mapper = new ObjectMapper();
|
|
ObjectMapper mapper = new ObjectMapper();
|
|
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
|
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
|
try {
|
|
try {
|
|
@@ -81,12 +84,18 @@ public class OceanEngineServiceImpl implements IOceanEngineService {
|
|
if(material.getVideoUrl() == null){
|
|
if(material.getVideoUrl() == null){
|
|
material.setVideoUrl(getVideoUrl(material.getVid()));
|
|
material.setVideoUrl(getVideoUrl(material.getVid()));
|
|
}
|
|
}
|
|
-// material.setVideoUrl(getVideoUrl(material.getVid()));
|
|
|
|
-// material.setCoverImage(FfmpegUtils.getVideoFirstImage(material.getVideoUrl()));
|
|
|
|
-// if(material.getCoverImage() == null){
|
|
|
|
-// Thread.sleep(1000);
|
|
|
|
-// material.setCoverImage(FfmpegUtils.getVideoFirstImage(material.getVideoUrl()));
|
|
|
|
-// }
|
|
|
|
|
|
+
|
|
|
|
+ String imageCode = FfmpegUtils.getVideoFirstImage(material.getVideoUrl(),20);
|
|
|
|
+ if(null == imageCode){
|
|
|
|
+ Thread.sleep(1000L);
|
|
|
|
+ imageCode = FfmpegUtils.getVideoFirstImage(material.getVideoUrl(),20);
|
|
|
|
+ }
|
|
|
|
+ String destPath = "D:\\data\\effectCase\\hot-material\\";
|
|
|
|
+ String fileName = material.getVid()+".jpg";
|
|
|
|
+ EffectCase.base64ToFile(destPath,imageCode,fileName);
|
|
|
|
+ InputStream inputStream = new FileInputStream(destPath+fileName);
|
|
|
|
+ String finalUrl = OSSUtils.uploadFile2OSS(inputStream, "ocean-engine/hot-material/", fileName);
|
|
|
|
+ material.setCoverUrl(finalUrl);
|
|
hotMaterialService.saveOrUpdate(material);
|
|
hotMaterialService.saveOrUpdate(material);
|
|
}
|
|
}
|
|
hotMaterialHandler(page+1,appCode,appName);
|
|
hotMaterialHandler(page+1,appCode,appName);
|
|
@@ -149,11 +158,17 @@ public class OceanEngineServiceImpl implements IOceanEngineService {
|
|
if (effectCase.getVideoUrl() == null){
|
|
if (effectCase.getVideoUrl() == null){
|
|
effectCase.setVideoUrl(getVideoUrl(effectCase.getVid()));
|
|
effectCase.setVideoUrl(getVideoUrl(effectCase.getVid()));
|
|
}
|
|
}
|
|
-// effectCase.setCoverImage(FfmpegUtils.getVideoFirstImage(effectCase.getVideoUrl()));
|
|
|
|
-// if (effectCase.getCoverImage() == null){
|
|
|
|
-// Thread.sleep(1000);
|
|
|
|
-// effectCase.setCoverImage(FfmpegUtils.getVideoFirstImage(effectCase.getVideoUrl()));
|
|
|
|
-// }
|
|
|
|
|
|
+ String imageCode = FfmpegUtils.getVideoFirstImage(effectCase.getVideoUrl(),20);
|
|
|
|
+ if(null == imageCode){
|
|
|
|
+ Thread.sleep(1000L);
|
|
|
|
+ imageCode = FfmpegUtils.getVideoFirstImage(effectCase.getVideoUrl(),20);
|
|
|
|
+ }
|
|
|
|
+ String destPath = "D:\\data\\effectCase\\effect-case\\";
|
|
|
|
+ String fileName = effectCase.getVid()+".jpg";
|
|
|
|
+ EffectCase.base64ToFile(destPath,imageCode,fileName);
|
|
|
|
+ InputStream inputStream = new FileInputStream(destPath+fileName);
|
|
|
|
+ String finalUrl = OSSUtils.uploadFile2OSS(inputStream, "ocean-engine/effect-case/", fileName);
|
|
|
|
+ effectCase.setCoverUrl(finalUrl);
|
|
effectCase.setCreativeLabel(getCase.getString("creative_label"));
|
|
effectCase.setCreativeLabel(getCase.getString("creative_label"));
|
|
}
|
|
}
|
|
effectCaseService.saveOrUpdate(effectCase);
|
|
effectCaseService.saveOrUpdate(effectCase);
|
|
@@ -180,17 +195,19 @@ public class OceanEngineServiceImpl implements IOceanEngineService {
|
|
if(node.get("data")!= null){
|
|
if(node.get("data")!= null){
|
|
String dataJson = node.get("data").get("items").toString();
|
|
String dataJson = node.get("data").get("items").toString();
|
|
List<DouyinHot> list = mapper.readValue(dataJson,new TypeReference<List<DouyinHot>>() {});
|
|
List<DouyinHot> list = mapper.readValue(dataJson,new TypeReference<List<DouyinHot>>() {});
|
|
- if (list != null && list.size() > 0){
|
|
|
|
|
|
+ if (list != null && !list.isEmpty()){
|
|
for (DouyinHot douyinHot : list){
|
|
for (DouyinHot douyinHot : list){
|
|
douyinHot.setUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+douyinHot.getVideoId()+"&line=0"));
|
|
douyinHot.setUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+douyinHot.getVideoId()+"&line=0"));
|
|
if(douyinHot.getUrl() == null){
|
|
if(douyinHot.getUrl() == null){
|
|
douyinHot.setUrl(getVideoUrl(douyinHot.getVideoId()));
|
|
douyinHot.setUrl(getVideoUrl(douyinHot.getVideoId()));
|
|
}
|
|
}
|
|
-// douyinHot.setCoverImage(FfmpegUtils.getVideoFirstImage(douyinHot.getUrl()));
|
|
|
|
-// if(douyinHot.getCoverImage() == null){
|
|
|
|
-// Thread.sleep(1000);
|
|
|
|
-// douyinHot.setCoverImage(FfmpegUtils.getVideoFirstImage(douyinHot.getUrl()));
|
|
|
|
-// }
|
|
|
|
|
|
+ String imageCode = FfmpegUtils.getVideoFirstImage(douyinHot.getUrl(),20);
|
|
|
|
+ String destPath = "D:\\data\\effectCase\\douyin-hot\\";
|
|
|
|
+ String fileName = douyinHot.getId()+".jpg";
|
|
|
|
+ EffectCase.base64ToFile(destPath,imageCode,fileName);
|
|
|
|
+ InputStream inputStream = new FileInputStream(destPath+fileName);
|
|
|
|
+ String finalUrl = OSSUtils.uploadFile2OSS(inputStream, "ocean-engine/douyin-hot/", fileName);
|
|
|
|
+ douyinHot.setCoverUrl(finalUrl);
|
|
douyinHotService.saveOrUpdate(douyinHot);
|
|
douyinHotService.saveOrUpdate(douyinHot);
|
|
}
|
|
}
|
|
douyinHotHandler(page+1,listType);
|
|
douyinHotHandler(page+1,listType);
|