|
@@ -18,6 +18,10 @@ import com.fasterxml.jackson.databind.JsonNode;
|
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
import javafx.scene.effect.Effect;
|
|
import javafx.scene.effect.Effect;
|
|
import org.apache.http.impl.cookie.BasicClientCookie;
|
|
import org.apache.http.impl.cookie.BasicClientCookie;
|
|
|
|
+import org.apache.poi.hslf.blip.Bitmap;
|
|
|
|
+import org.bytedeco.javacpp.opencv_core;
|
|
|
|
+import org.bytedeco.javacv.FFmpegFrameGrabber;
|
|
|
|
+import org.bytedeco.javacv.Frame;
|
|
import org.openqa.selenium.Cookie;
|
|
import org.openqa.selenium.Cookie;
|
|
import org.openqa.selenium.WebDriver;
|
|
import org.openqa.selenium.WebDriver;
|
|
import org.openqa.selenium.chrome.ChromeDriver;
|
|
import org.openqa.selenium.chrome.ChromeDriver;
|
|
@@ -26,10 +30,17 @@ import org.quartz.Job;
|
|
import org.quartz.JobExecutionContext;
|
|
import org.quartz.JobExecutionContext;
|
|
import org.quartz.JobExecutionException;
|
|
import org.quartz.JobExecutionException;
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
|
|
+import sun.misc.BASE64Encoder;
|
|
|
|
|
|
|
|
+import javax.imageio.ImageIO;
|
|
|
|
+import java.awt.*;
|
|
|
|
+import java.awt.image.BufferedImage;
|
|
|
|
+import java.io.ByteArrayOutputStream;
|
|
|
|
+import java.io.File;
|
|
import java.net.HttpURLConnection;
|
|
import java.net.HttpURLConnection;
|
|
import java.net.URL;
|
|
import java.net.URL;
|
|
import java.util.*;
|
|
import java.util.*;
|
|
|
|
+import java.util.List;
|
|
|
|
|
|
public class OceanengineJob implements Job {
|
|
public class OceanengineJob implements Job {
|
|
@Autowired
|
|
@Autowired
|
|
@@ -38,7 +49,11 @@ public class OceanengineJob implements Job {
|
|
private IEffectCaseService effectCaseService;
|
|
private IEffectCaseService effectCaseService;
|
|
@Autowired
|
|
@Autowired
|
|
private IHotMaterialService hotMaterialService;
|
|
private IHotMaterialService hotMaterialService;
|
|
|
|
+ public static void main(String[] args){
|
|
|
|
+ OceanengineJob oceanengineJob = new OceanengineJob();
|
|
|
|
+ oceanengineJob.getVideoFirstImage("http://v5-dy.ixigua.com/4651275f4a249f78f7c20bda0a1aa7bb/5e2524c0/video/m/2202d662c8f1cc54dce8a3670f57aa7f1f91154c11b000109273b87f177/?a=1128&br=2754&bt=1377&cr=0&cs=0&dr=0&ds=3&er=&l=20200120105447010012034216116D5EDE&lr=aweme&qs=0&rc=O2dpZGU8ZGg4ZDM0aThnZ0ApNmg2OWllaGU3N2g0ZjZoPGcuYmEtYi8wXzYxYjAzNTYxczMvY18tNS5eMjUyLjAvXjY6Yw%3D%3D");
|
|
|
|
|
|
|
|
+ }
|
|
private String getRealVideoUrl(String url){
|
|
private String getRealVideoUrl(String url){
|
|
try {
|
|
try {
|
|
URL serverUrl = new URL(url);
|
|
URL serverUrl = new URL(url);
|
|
@@ -59,7 +74,6 @@ public class OceanengineJob implements Job {
|
|
}
|
|
}
|
|
private void hotMaterialHandler(int page,int appCode,String appName){
|
|
private void hotMaterialHandler(int page,int appCode,String appName){
|
|
String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_material/list?material_type=3&order_by=convert_show_rate&period_type=3&aggr_app_code="+appCode+"&page="+page+"&limit=20");
|
|
String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_material/list?material_type=3&order_by=convert_show_rate&period_type=3&aggr_app_code="+appCode+"&page="+page+"&limit=20");
|
|
- System.out.println(result);
|
|
|
|
ObjectMapper mapper = new ObjectMapper();
|
|
ObjectMapper mapper = new ObjectMapper();
|
|
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
|
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
|
try {
|
|
try {
|
|
@@ -77,6 +91,12 @@ public class OceanengineJob implements Job {
|
|
if(material.getVideoUrl() == null){
|
|
if(material.getVideoUrl() == null){
|
|
material.setVideoUrl(getVideoUrl(material.getVid()));
|
|
material.setVideoUrl(getVideoUrl(material.getVid()));
|
|
}
|
|
}
|
|
|
|
+// material.setVideoUrl(getVideoUrl(material.getVid()));
|
|
|
|
+ material.setCoverImage(getVideoFirstImage(material.getVideoUrl()));
|
|
|
|
+ if(material.getCoverImage() == null){
|
|
|
|
+ Thread.sleep(1000);
|
|
|
|
+ material.setCoverImage(getVideoFirstImage(material.getVideoUrl()));
|
|
|
|
+ }
|
|
hotMaterialService.saveOrUpdate(material);
|
|
hotMaterialService.saveOrUpdate(material);
|
|
}
|
|
}
|
|
hotMaterialHandler(page+1,appCode,appName);
|
|
hotMaterialHandler(page+1,appCode,appName);
|
|
@@ -100,7 +120,6 @@ public class OceanengineJob implements Job {
|
|
}
|
|
}
|
|
private void effectCaseHandler(int page){
|
|
private void effectCaseHandler(int page){
|
|
String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/case/effect_case_list?industry=0&style=0&mode=0&page="+page+"&limit=20");
|
|
String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/case/effect_case_list?industry=0&style=0&mode=0&page="+page+"&limit=20");
|
|
- System.out.println(result);
|
|
|
|
ObjectMapper mapper = new ObjectMapper();
|
|
ObjectMapper mapper = new ObjectMapper();
|
|
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
|
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
|
try {
|
|
try {
|
|
@@ -129,6 +148,12 @@ public class OceanengineJob implements Job {
|
|
if (effectCase.getVideoUrl() == null){
|
|
if (effectCase.getVideoUrl() == null){
|
|
effectCase.setVideoUrl(getVideoUrl(effectCase.getVid()));
|
|
effectCase.setVideoUrl(getVideoUrl(effectCase.getVid()));
|
|
}
|
|
}
|
|
|
|
+// effectCase.setVideoUrl(getVideoUrl(effectCase.getVid()));
|
|
|
|
+ effectCase.setCoverImage(getVideoFirstImage(effectCase.getVideoUrl()));
|
|
|
|
+ if (effectCase.getCoverImage() == null){
|
|
|
|
+ Thread.sleep(1000);
|
|
|
|
+ effectCase.setCoverImage(getVideoFirstImage(effectCase.getVideoUrl()));
|
|
|
|
+ }
|
|
effectCase.setCreativeLabel(detailNode.get("data").get("effect_case").get("creative_label").toString());
|
|
effectCase.setCreativeLabel(detailNode.get("data").get("effect_case").get("creative_label").toString());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
@@ -143,9 +168,46 @@ public class OceanengineJob implements Job {
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
|
|
+ private String getVideoFirstImage(String url){
|
|
|
|
+ try {
|
|
|
|
+ FFmpegFrameGrabber ff = new FFmpegFrameGrabber(url);
|
|
|
|
+ ff.start();
|
|
|
|
+ int lenght = ff.getLengthInFrames();
|
|
|
|
+ int i = 0;
|
|
|
|
+ Frame f = null;
|
|
|
|
+ while (i < lenght) {
|
|
|
|
+// 过滤前5帧,避免出现全黑的图片,依自己情况而定
|
|
|
|
+ f = ff.grabFrame();
|
|
|
|
+ if ((i > 5) && (f.image != null)) {
|
|
|
|
+ break;
|
|
|
|
+ }
|
|
|
|
+ i++;
|
|
|
|
+ }
|
|
|
|
+ opencv_core.IplImage img = f.image;
|
|
|
|
+ int owidth = img.width();
|
|
|
|
+ int oheight = img.height();
|
|
|
|
+// 对截取的帧进行等比例缩放
|
|
|
|
+ int width = 800;
|
|
|
|
+ int height = (int) (((double) width / owidth) * oheight);
|
|
|
|
+ BufferedImage bi = new BufferedImage(width, height, BufferedImage.TYPE_3BYTE_BGR);
|
|
|
|
+ bi.getGraphics().drawImage(f.image.getBufferedImage().getScaledInstance(width, height, Image.SCALE_SMOOTH),
|
|
|
|
+ 0, 0, null);
|
|
|
|
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();//io流
|
|
|
|
+ ImageIO.write(bi, "jpg", baos);
|
|
|
|
+ byte[] bytes = baos.toByteArray();//转换成字节
|
|
|
|
+ BASE64Encoder encoder = new BASE64Encoder();
|
|
|
|
+ String png_base64 = encoder.encodeBuffer(bytes).trim();//转换成base64串
|
|
|
|
+ png_base64 = png_base64.replaceAll("\n", "").replaceAll("\r", "");//删除 \r\n
|
|
|
|
+ // ImageIO.write(bufferedImage, "png", new File("D:/qrcode1.png"));
|
|
|
|
+ return "data:image/jpg;base64,"+png_base64;
|
|
|
|
+ }catch (Exception e){
|
|
|
|
+ e.printStackTrace();
|
|
|
|
+ return null;
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ }
|
|
private void douyinHotHandler(int page,int listType){
|
|
private void douyinHotHandler(int page,int listType){
|
|
String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_video/user_side/list?page="+page+"&limit=20&list_type="+listType+"&aggr_duration_type=-1");
|
|
String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_video/user_side/list?page="+page+"&limit=20&list_type="+listType+"&aggr_duration_type=-1");
|
|
- System.out.println(result);
|
|
|
|
ObjectMapper mapper = new ObjectMapper();
|
|
ObjectMapper mapper = new ObjectMapper();
|
|
try {
|
|
try {
|
|
JsonNode node = mapper.readTree(result);
|
|
JsonNode node = mapper.readTree(result);
|
|
@@ -155,12 +217,17 @@ public class OceanengineJob implements Job {
|
|
String dataJson = node.get("data").get("items").toString();
|
|
String dataJson = node.get("data").get("items").toString();
|
|
List<DouyinHot> list = mapper.readValue(dataJson,new TypeReference<List<DouyinHot>>() {});
|
|
List<DouyinHot> list = mapper.readValue(dataJson,new TypeReference<List<DouyinHot>>() {});
|
|
if (list != null && list.size() > 0){
|
|
if (list != null && list.size() > 0){
|
|
-// douyinHotService.replaceBatch(list);
|
|
|
|
for (DouyinHot douyinHot : list){
|
|
for (DouyinHot douyinHot : list){
|
|
douyinHot.setUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+douyinHot.getVideoId()+"&line=0"));
|
|
douyinHot.setUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+douyinHot.getVideoId()+"&line=0"));
|
|
if(douyinHot.getUrl() == null){
|
|
if(douyinHot.getUrl() == null){
|
|
douyinHot.setUrl(getVideoUrl(douyinHot.getVideoId()));
|
|
douyinHot.setUrl(getVideoUrl(douyinHot.getVideoId()));
|
|
}
|
|
}
|
|
|
|
+// douyinHot.setUrl(getVideoUrl(douyinHot.getVideoId()));
|
|
|
|
+ douyinHot.setCoverImage(getVideoFirstImage(douyinHot.getUrl()));
|
|
|
|
+ if(douyinHot.getCoverImage() == null){
|
|
|
|
+ Thread.sleep(1000);
|
|
|
|
+ douyinHot.setCoverImage(getVideoFirstImage(douyinHot.getUrl()));
|
|
|
|
+ }
|
|
douyinHotService.saveOrUpdate(douyinHot);
|
|
douyinHotService.saveOrUpdate(douyinHot);
|
|
}
|
|
}
|
|
douyinHotHandler(page+1,listType);
|
|
douyinHotHandler(page+1,listType);
|
|
@@ -188,6 +255,7 @@ public class OceanengineJob implements Job {
|
|
hotMaterialHandler(1,4,"抖音");
|
|
hotMaterialHandler(1,4,"抖音");
|
|
hotMaterialHandler(1,8,"头条");
|
|
hotMaterialHandler(1,8,"头条");
|
|
hotMaterialHandler(1,9,"穿山甲");
|
|
hotMaterialHandler(1,9,"穿山甲");
|
|
|
|
+ System.out.println("巨量创意抓取完成");
|
|
}
|
|
}
|
|
|
|
|
|
private void login(String account,String password){
|
|
private void login(String account,String password){
|