Quellcode durchsuchen

Merge remote-tracking branch 'origin/test' into test_hot

# Conflicts:
#	module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/entity/DouyinHot.java
#	module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/entity/EffectCase.java
#	module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/entity/HotMaterial.java
syh vor 5 Jahren
Ursprung
Commit
4f4e75d54c

+ 124 - 15
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/OceanengineJob.java

@@ -1,5 +1,6 @@
 package org.jeecg.modules.ctop.job;
 
+import cn.com.ctop.common.module.utils.Base64Utils;
 import cn.com.ctop.common.module.utils.HttpUtils2;
 import cn.com.ctop.crawler.modules.core.util.FateadmHttpUtil;
 import cn.com.ctop.crawler.modules.core.util.FateadmUtil;
@@ -14,6 +15,10 @@ import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.http.impl.cookie.BasicClientCookie;
+import org.apache.poi.hslf.blip.Bitmap;
+import org.bytedeco.javacpp.opencv_core;
+import org.bytedeco.javacv.FFmpegFrameGrabber;
+import org.bytedeco.javacv.Frame;
 import org.openqa.selenium.Cookie;
 import org.openqa.selenium.WebDriver;
 import org.openqa.selenium.chrome.ChromeDriver;
@@ -22,8 +27,17 @@ import org.quartz.Job;
 import org.quartz.JobExecutionContext;
 import org.quartz.JobExecutionException;
 import org.springframework.beans.factory.annotation.Autowired;
+import sun.misc.BASE64Encoder;
 
+import javax.imageio.ImageIO;
+import java.awt.*;
+import java.awt.image.BufferedImage;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.net.HttpURLConnection;
+import java.net.URL;
 import java.util.*;
+import java.util.List;
 
 public class OceanengineJob implements Job {
     @Autowired
@@ -32,10 +46,31 @@ public class OceanengineJob implements Job {
     private IEffectCaseService effectCaseService;
     @Autowired
     private IHotMaterialService hotMaterialService;
+    public static void main(String[] args){
+        OceanengineJob oceanengineJob = new OceanengineJob();
+        oceanengineJob.getVideoFirstImage("http://v5-dy.ixigua.com/4651275f4a249f78f7c20bda0a1aa7bb/5e2524c0/video/m/2202d662c8f1cc54dce8a3670f57aa7f1f91154c11b000109273b87f177/?a=1128&br=2754&bt=1377&cr=0&cs=0&dr=0&ds=3&er=&l=20200120105447010012034216116D5EDE&lr=aweme&qs=0&rc=O2dpZGU8ZGg4ZDM0aThnZ0ApNmg2OWllaGU3N2g0ZjZoPGcuYmEtYi8wXzYxYjAzNTYxczMvY18tNS5eMjUyLjAvXjY6Yw%3D%3D");
 
+    }
+    private String getRealVideoUrl(String url){
+        try {
+            URL serverUrl = new URL(url);
+            HttpURLConnection conn = (HttpURLConnection) serverUrl
+                    .openConnection();
+            conn.setRequestMethod("GET");
+            // 必须设置false,否则会自动redirect到Location的地址
+            conn.setInstanceFollowRedirects(false);
+            conn.addRequestProperty("User-Agent",
+                    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.2.8) Firefox/3.6.8");
+            conn.connect();
+            String location = conn.getHeaderField("Location");
+            return location;
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+        return null;
+    }
     private void hotMaterialHandler(int page,int appCode,String appName){
-        String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_material/list?material_type=3&order_by=convert_show_rate&period_type=3&aggr_app_code="+appCode+"&page="+page+"&limit=100");
-        System.out.println(result);
+        String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_material/list?material_type=3&order_by=convert_show_rate&period_type=3&aggr_app_code="+appCode+"&page="+page+"&limit=20");
         ObjectMapper mapper = new ObjectMapper();
         mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
         try {
@@ -49,7 +84,16 @@ public class OceanengineJob implements Job {
                         for (HotMaterial material : list){
                             material.setAppCode(appCode);
                             material.setAppName(appName);
-                            material.setVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+material.getVid()+"&line=0");
+                            material.setVideoUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+material.getVid()+"&line=0"));
+                            if(material.getVideoUrl() == null){
+                                material.setVideoUrl(getVideoUrl(material.getVid()));
+                            }
+//                            material.setVideoUrl(getVideoUrl(material.getVid()));
+                            material.setCoverImage(getVideoFirstImage(material.getVideoUrl()));
+                            if(material.getCoverImage() == null){
+                                Thread.sleep(1000);
+                                material.setCoverImage(getVideoFirstImage(material.getVideoUrl()));
+                            }
                             hotMaterialService.saveOrUpdate(material);
                         }
                         hotMaterialHandler(page+1,appCode,appName);
@@ -60,9 +104,19 @@ public class OceanengineJob implements Job {
             e.printStackTrace();
         }
     }
+    private String getVideoUrl(String vid) throws Exception{
+        String url = "http://i.snssdk.com/video/urls/1/toutiao/mp4/"+vid;
+        String result = HttpUtils2.httpGetRequest(url);
+        ObjectMapper mapper = new ObjectMapper();
+        if(result != null){
+            JsonNode node = mapper.readTree(result);
+            String mainUrl = node.get("data").get("video_list").get("video_1").get("main_url").asText();
+            return Base64Utils.decode(mainUrl);
+        }
+        return null;
+    }
     private void effectCaseHandler(int page){
-        String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/case/effect_case_list?industry=0&style=0&mode=0&page="+page+"&limit=100");
-        System.out.println(result);
+        String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/case/effect_case_list?industry=0&style=0&mode=0&page="+page+"&limit=20");
         ObjectMapper mapper = new ObjectMapper();
         mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
         try {
@@ -87,7 +141,16 @@ public class OceanengineJob implements Job {
                                     effectCase.setCreativeScore(detailNode.get("data").get("effect_case").get("creative_score").asText());
                                     effectCase.setEffect(detailNode.get("data").get("effect_case").get("effect").toString());
                                     effectCase.setMethods(detailNode.get("data").get("effect_case").get("methods").toString());
-                                    effectCase.setVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+effectCase.getVid()+"&line=0");
+                                    effectCase.setVideoUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+effectCase.getVid()+"&line=0"));
+                                    if (effectCase.getVideoUrl() == null){
+                                        effectCase.setVideoUrl(getVideoUrl(effectCase.getVid()));
+                                    }
+//                                    effectCase.setVideoUrl(getVideoUrl(effectCase.getVid()));
+                                    effectCase.setCoverImage(getVideoFirstImage(effectCase.getVideoUrl()));
+                                    if (effectCase.getCoverImage() == null){
+                                        Thread.sleep(1000);
+                                        effectCase.setCoverImage(getVideoFirstImage(effectCase.getVideoUrl()));
+                                    }
                                     effectCase.setCreativeLabel(detailNode.get("data").get("effect_case").get("creative_label").toString());
                                 }
                             }
@@ -102,9 +165,46 @@ public class OceanengineJob implements Job {
         }
     }
 
+    private String getVideoFirstImage(String url){
+        try {
+            FFmpegFrameGrabber ff = new FFmpegFrameGrabber(url);
+            ff.start();
+            int lenght = ff.getLengthInFrames();
+            int i = 0;
+            Frame f = null;
+            while (i < lenght) {
+// 过滤前5帧,避免出现全黑的图片,依自己情况而定
+                f = ff.grabFrame();
+                if ((i > 5) && (f.image != null)) {
+                    break;
+                }
+                i++;
+            }
+            opencv_core.IplImage img = f.image;
+            int owidth = img.width();
+            int oheight = img.height();
+// 对截取的帧进行等比例缩放
+            int width = 800;
+            int height = (int) (((double) width / owidth) * oheight);
+            BufferedImage bi = new BufferedImage(width, height, BufferedImage.TYPE_3BYTE_BGR);
+            bi.getGraphics().drawImage(f.image.getBufferedImage().getScaledInstance(width, height, Image.SCALE_SMOOTH),
+                    0, 0, null);
+            ByteArrayOutputStream baos = new ByteArrayOutputStream();//io流
+            ImageIO.write(bi, "jpg", baos);
+            byte[] bytes = baos.toByteArray();//转换成字节
+            BASE64Encoder encoder = new BASE64Encoder();
+            String png_base64 = encoder.encodeBuffer(bytes).trim();//转换成base64串
+            png_base64 = png_base64.replaceAll("\n", "").replaceAll("\r", "");//删除 \r\n
+            //        ImageIO.write(bufferedImage, "png", new File("D:/qrcode1.png"));
+            return "data:image/jpg;base64,"+png_base64;
+        }catch (Exception e){
+            e.printStackTrace();
+            return null;
+        }
+
+    }
     private void douyinHotHandler(int page,int listType){
-        String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_video/user_side/list?page="+page+"&limit=100&list_type="+listType+"&aggr_duration_type=-1");
-        System.out.println(result);
+        String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_video/user_side/list?page="+page+"&limit=20&list_type="+listType+"&aggr_duration_type=-1");
         ObjectMapper mapper = new ObjectMapper();
         try {
             JsonNode node = mapper.readTree(result);
@@ -114,9 +214,17 @@ public class OceanengineJob implements Job {
                     String dataJson = node.get("data").get("items").toString();
                     List<DouyinHot> list = mapper.readValue(dataJson,new TypeReference<List<DouyinHot>>() {});
                     if (list != null && list.size() > 0){
-//                        douyinHotService.replaceBatch(list);
                         for (DouyinHot douyinHot : list){
-                            douyinHot.setUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+douyinHot.getVideoId()+"&line=0");
+                            douyinHot.setUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+douyinHot.getVideoId()+"&line=0"));
+                            if(douyinHot.getUrl() == null){
+                                douyinHot.setUrl(getVideoUrl(douyinHot.getVideoId()));
+                            }
+//                            douyinHot.setUrl(getVideoUrl(douyinHot.getVideoId()));
+                            douyinHot.setCoverImage(getVideoFirstImage(douyinHot.getUrl()));
+                            if(douyinHot.getCoverImage() == null){
+                                Thread.sleep(1000);
+                                douyinHot.setCoverImage(getVideoFirstImage(douyinHot.getUrl()));
+                            }
                             douyinHotService.saveOrUpdate(douyinHot);
                         }
                         douyinHotHandler(page+1,listType);
@@ -134,23 +242,24 @@ public class OceanengineJob implements Job {
     }
     @Override
     public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
-        String account="1728681417@qq.com";
-        String password = "Yy-704127411";
+        String account="3248395570@qq.com";
+        String password = "Ydxq-704127411";
         login(account,password);
-//        douyinHotHandler(1,1);
-//        effectCaseHandler(1);
+        douyinHotHandler(1,1);
+        effectCaseHandler(1);
         hotMaterialHandler(1,1,"西瓜");
         hotMaterialHandler(1,3,"火山");
         hotMaterialHandler(1,4,"抖音");
         hotMaterialHandler(1,8,"头条");
         hotMaterialHandler(1,9,"穿山甲");
+        System.out.println("巨量创意抓取完成");
     }
 
     private void login(String account,String password){
         String url = "https://cc.oceanengine.com/login";
         System.getProperties().setProperty("webdriver.chrome.driver", "D:/chromedriver.exe");
         ChromeOptions chromeOptions = new ChromeOptions();
-//        chromeOptions.addArguments("--headless");
+        chromeOptions.addArguments("--headless");
         chromeOptions.addArguments("--no-sandbox");
         chromeOptions.addArguments("--window-size=1920,1080");
         chromeOptions.addArguments("--user-agent=" + HttpUtils2.USER_AGENT);

+ 20 - 0
module-common/src/main/java/cn/com/ctop/common/module/utils/HttpUtils2.java

@@ -260,6 +260,26 @@ public class HttpUtils2 {
         return strReturn;
     }
 
+    public static String getRedirectUrl(String url){
+        HttpClient httpClient = createSSLClientDefault();
+        HttpResponse response = null;
+        HttpGet httpGet = new HttpGet(url);
+        String result = null;
+        try {
+//            httpGet.setHeader("User-Agent", USER_AGENT);
+            response = httpClient.execute(httpGet);
+            int statusCode = response.getStatusLine().getStatusCode();
+            if (statusCode == HttpStatus.SC_MOVED_PERMANENTLY || statusCode == HttpStatus.SC_MOVED_TEMPORARILY) {
+                String newUrl = response.getFirstHeader("Location").getValue();
+                return newUrl;
+            }
+
+        }catch (Exception e){
+            e.printStackTrace();
+        }
+        return null;
+    }
+
     public static String httpGetRequest(String url) {
         HttpClient httpClient = createSSLClientDefault();
         HttpResponse response = null;

+ 0 - 1
module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/entity/EffectCase.java

@@ -6,7 +6,6 @@ import com.baomidou.mybatisplus.annotation.TableId;
 import com.baomidou.mybatisplus.annotation.TableName;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import lombok.Data;
-
 import java.util.Date;
 
 @Data

+ 0 - 1
module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/entity/HotMaterial.java

@@ -5,7 +5,6 @@ import com.baomidou.mybatisplus.annotation.TableId;
 import com.baomidou.mybatisplus.annotation.TableName;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import lombok.Data;
-
 import java.util.Date;
 
 @Data

+ 5 - 0
pom.xml

@@ -383,6 +383,11 @@
             <artifactId>aliyun-java-sdk-mts</artifactId>
             <version>2.5.2</version>
         </dependency>
+        <dependency>
+            <groupId>org.bytedeco</groupId>
+            <artifactId>javacv</artifactId>
+            <version>0.8</version>
+        </dependency>
     </dependencies>
 
     <dependencyManagement>