Selaa lähdekoodia

修改巨量引擎爬虫封面数据查询方式

syh 5 vuotta sitten
vanhempi
commit
67cf76fabd

+ 23 - 0
jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java

@@ -3,6 +3,8 @@ package org.jeecg;
 import cn.com.ctop.common.module.entity.BindAccountLogin;
 import cn.com.ctop.common.module.service.IBindAccountLoginService;
 import cn.com.ctop.common.module.utils.CtopAdConstant;
+import cn.com.ctop.crawler.modules.oceanengine.entity.HotMaterial;
+import cn.com.ctop.crawler.modules.oceanengine.service.IHotMaterialService;
 import cn.com.ctop.crawler.modules.oceanengine.service.IOceanEngineService;
 import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouWebInterfaceService;
 import lombok.extern.slf4j.Slf4j;
@@ -24,6 +26,21 @@ import java.util.concurrent.Executors;
 public class SampleTest {
     @Autowired
     private IOceanEngineService oceanEngineService;
+    @Autowired
+    private IHotMaterialService hotMaterialService;
+    @Test
+    public void initImageUrl(){
+        List<HotMaterial> hotMaterials =  hotMaterialService.getId();
+        if(null!=hotMaterials&&!hotMaterials.isEmpty()){
+            hotMaterials.forEach(douyinHot -> {
+                HotMaterial getHot = hotMaterialService.getById(douyinHot.getId());
+//                String image = getHot.getCoverImage().replace("data:image/jpg;base64,", "");
+                String destPath = "D:\\data\\effectCase\\";
+                String fileName = getHot.getId()+".jpg";
+//                EffectCase.base64ToFile(destPath,image,fileName);
+            });
+        }
+    }
     @Test
     public void testOceanEngineJob(){
         String account="3248395570@qq.com";
@@ -88,6 +105,10 @@ public class SampleTest {
         Long end = System.currentTimeMillis();
         log.info("总用时:{}毫秒",end-start);
     }
+    @Test
+    public void testBase64(){
+
+    }
 }
 //刷新token:885936毫秒
 
@@ -106,3 +127,5 @@ public class SampleTest {
 
 
 
+
+

+ 54 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/core/util/FfmpegUtils.java

@@ -0,0 +1,54 @@
+package cn.com.ctop.crawler.modules.core.util;
+
+import org.bytedeco.javacv.FFmpegFrameGrabber;
+import org.bytedeco.javacv.Frame;
+import org.bytedeco.javacv.Java2DFrameConverter;
+
+import javax.imageio.ImageIO;
+import java.awt.*;
+import java.awt.image.BufferedImage;
+import java.io.ByteArrayOutputStream;
+import java.util.Base64;
+
+public class FfmpegUtils {
+    public static String getVideoFirstImage(String url){
+        try {
+            FFmpegFrameGrabber ff = new FFmpegFrameGrabber(url);
+            ff.start();
+            int lenght = ff.getLengthInFrames();
+            int i = 0;
+            Frame f = null;
+            while (i < lenght) {
+// 过滤前5帧,避免出现全黑的图片,依自己情况而定
+                f = ff.grabFrame();
+                if ((i > 5) && (f.image != null)) {
+                    break;
+                }
+                i++;
+            }
+//            opencv_core.IplImage img = f.image;
+            int owidth = f.imageWidth;
+            int oheight = f.imageHeight;
+            Java2DFrameConverter converter =new Java2DFrameConverter();
+            BufferedImage fecthedImage =converter.getBufferedImage(f);
+// 对截取的帧进行等比例缩放
+            int width = 800;
+            int height = (int) (((double) width / owidth) * oheight);
+            BufferedImage bi = new BufferedImage(width, height, BufferedImage.TYPE_3BYTE_BGR);
+            bi.getGraphics().drawImage(fecthedImage.getScaledInstance(width, height, Image.SCALE_SMOOTH),
+                    0, 0, null);
+            ByteArrayOutputStream baos = new ByteArrayOutputStream();//io流
+            ImageIO.write(bi, "jpg", baos);
+            byte[] bytes = baos.toByteArray();//转换成字节
+            String png_base64 = Base64.getEncoder().encodeToString(bytes).trim();//转换成base64串
+            png_base64 = png_base64.replaceAll("\n", "").replaceAll("\r", "");//删除 \r\n
+            //        ImageIO.write(bufferedImage, "png", new File("D:/qrcode1.png"));
+            baos.close();
+            ff.close();
+            return "data:image/jpg;base64,"+png_base64;
+        }catch (Exception e){
+            e.printStackTrace();
+            return null;
+        }
+    }
+}

+ 1 - 1
module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/entity/DouyinHot.java

@@ -20,7 +20,7 @@ public class DouyinHot {
     private Long userId;
     @JsonProperty("core_user_name")
     private String userName;
-    private String coverImage;
+    private String coverUrl;
     private String title;
     @JsonProperty("head_image_uri")
     private String headImageUri;

+ 42 - 10
module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/entity/EffectCase.java

@@ -7,6 +7,10 @@ import com.baomidou.mybatisplus.annotation.TableName;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import lombok.Data;
 
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.util.Base64;
 import java.util.Date;
 
 /**
@@ -17,6 +21,7 @@ import java.util.Date;
 public class EffectCase {
     @TableId(value = "id", type = IdType.INPUT)
     private String id;
+    private String coverUrl;
     private String vid;
     @JsonProperty("creative_label_1")
     @TableField(value = "creative_label_1")
@@ -35,7 +40,7 @@ public class EffectCase {
     private String effect;
     @JsonProperty("conversion_type")
     private String conversionType;
-    private String coverImage;
+//    private String coverImage;
     private String creativeLabel;
     private String tag;
     @JsonProperty("creative_title")
@@ -136,15 +141,6 @@ public class EffectCase {
     public void setConversionType(String conversionType) {
         this.conversionType = conversionType;
     }
-
-    public String getCoverImage() {
-        return coverImage;
-    }
-
-    public void setCoverImage(String coverImage) {
-        this.coverImage = coverImage;
-    }
-
     public String getCreativeLabel() {
         return creativeLabel;
     }
@@ -208,4 +204,40 @@ public class EffectCase {
     public void setUpdateTime(Date updateTime) {
         this.updateTime = updateTime;
     }
+
+    public static void base64ToFile(String destPath,String base64, String fileName) {
+        File file = null;
+        //创建文件目录
+        String filePath=destPath;
+        File  dir=new File(filePath);
+        if (!dir.exists() && !dir.isDirectory()) {
+            dir.mkdirs();
+        }
+        BufferedOutputStream bos = null;
+        java.io.FileOutputStream fos = null;
+        try {
+            byte[] bytes = Base64.getMimeDecoder().decode(base64);
+            file=new File(filePath+"/"+fileName);
+            fos = new java.io.FileOutputStream(file);
+            bos = new BufferedOutputStream(fos);
+            bos.write(bytes);
+        } catch (Exception e) {
+            e.printStackTrace();
+        } finally {
+            if (bos != null) {
+                try {
+                    bos.close();
+                } catch (IOException e) {
+                    e.printStackTrace();
+                }
+            }
+            if (fos != null) {
+                try {
+                    fos.close();
+                } catch (IOException e) {
+                    e.printStackTrace();
+                }
+            }
+        }
+    }
 }

+ 1 - 1
module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/entity/HotMaterial.java

@@ -20,7 +20,7 @@ public class HotMaterial {
     private String videoUrl;
     private String title;
     private Integer appCode;
-    private String coverImage;
+    private String coverUrl;
     private String appName;
     private Date createTime;
     private Date updateTime;

+ 3 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/service/IDouyinHotService.java

@@ -10,4 +10,7 @@ public interface IDouyinHotService extends IService<DouyinHot> {
     public void replaceBatch(List<DouyinHot> list);
 
     IPage<DouyinHot> queryHotFeeds(String userName, String start, String end, String orderByColumn, String orderByType, Integer pageNo, Integer pageSize);
+
+    List<DouyinHot> getId();
+
 }

+ 5 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/service/IHotMaterialService.java

@@ -4,6 +4,11 @@ import cn.com.ctop.crawler.modules.oceanengine.entity.HotMaterial;
 import com.baomidou.mybatisplus.core.metadata.IPage;
 import com.baomidou.mybatisplus.extension.service.IService;
 
+import java.util.List;
+
 public interface IHotMaterialService extends IService<HotMaterial> {
     IPage<HotMaterial> queryHotFeeds(String userName, String start, String end, String orderByColumn, String orderByType, Integer pageNo, Integer pageSize);
+
+    List<HotMaterial> getId();
+
 }

+ 8 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/service/impl/DouyinHotServiceImpl.java

@@ -39,4 +39,12 @@ public class DouyinHotServiceImpl extends ServiceImpl<DouyinHotMapper, DouyinHot
         Page<DouyinHot> page = new Page<>(pageNo, pageSize);
         return this.page(page, queryWrapper);
     }
+
+    @Override
+    public List<DouyinHot> getId() {
+        QueryWrapper<DouyinHot> queryWrapper = new QueryWrapper<>();
+        queryWrapper.select("id").orderByDesc("create_time");
+        return this.list(queryWrapper);
+
+    }
 }

+ 10 - 1
module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/service/impl/HotMaterialServiceImpl.java

@@ -9,13 +9,15 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import org.springframework.stereotype.Service;
 
+import java.util.List;
+
 @Service
 public class HotMaterialServiceImpl extends ServiceImpl<HotMaterialMapper, HotMaterial> implements IHotMaterialService {
     @Override
     public IPage<HotMaterial> queryHotFeeds(String userName, String start, String end, String orderByColumn, String orderByType, Integer pageNo, Integer pageSize) {
         QueryWrapper<HotMaterial> queryWrapper = new QueryWrapper<>();
         if (null != userName && !"".equals(userName)) {
-            queryWrapper.like("author", "%" + userName + "%");
+            queryWrapper.like("author", userName);
         }
         if (null != start && !"".equals(start) && null != end && !"".equals(end)) {
             queryWrapper.between("create_time", start, end);
@@ -28,4 +30,11 @@ public class HotMaterialServiceImpl extends ServiceImpl<HotMaterialMapper, HotMa
         Page<HotMaterial> page = new Page<>(pageNo, pageSize);
         return this.page(page, queryWrapper);
     }
+
+    @Override
+    public List<HotMaterial> getId() {
+        QueryWrapper<HotMaterial> queryWrapper = new QueryWrapper<>();
+        queryWrapper.select("id").orderByDesc("create_time");
+        return this.list(queryWrapper);
+    }
 }

+ 15 - 67
module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/service/impl/OceanEngineServiceImpl.java

@@ -19,9 +19,6 @@ import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.http.impl.cookie.BasicClientCookie;
-import org.bytedeco.javacv.FFmpegFrameGrabber;
-import org.bytedeco.javacv.Frame;
-import org.bytedeco.javacv.Java2DFrameConverter;
 import org.openqa.selenium.Cookie;
 import org.openqa.selenium.WebDriver;
 import org.openqa.selenium.chrome.ChromeDriver;
@@ -30,13 +27,8 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
-import javax.imageio.ImageIO;
-import java.awt.*;
-import java.awt.image.BufferedImage;
-import java.io.ByteArrayOutputStream;
 import java.net.HttpURLConnection;
 import java.net.URL;
-import java.util.List;
 import java.util.*;
 
 @Service
@@ -90,11 +82,11 @@ public class OceanEngineServiceImpl implements IOceanEngineService {
                                 material.setVideoUrl(getVideoUrl(material.getVid()));
                             }
 //                            material.setVideoUrl(getVideoUrl(material.getVid()));
-                            material.setCoverImage(getVideoFirstImage(material.getVideoUrl()));
-                            if(material.getCoverImage() == null){
-                                Thread.sleep(1000);
-                                material.setCoverImage(getVideoFirstImage(material.getVideoUrl()));
-                            }
+//                            material.setCoverImage(FfmpegUtils.getVideoFirstImage(material.getVideoUrl()));
+//                            if(material.getCoverImage() == null){
+//                                Thread.sleep(1000);
+//                                material.setCoverImage(FfmpegUtils.getVideoFirstImage(material.getVideoUrl()));
+//                            }
                             hotMaterialService.saveOrUpdate(material);
                         }
                         hotMaterialHandler(page+1,appCode,appName);
@@ -124,9 +116,7 @@ public class OceanEngineServiceImpl implements IOceanEngineService {
     @Override
     public void effectCaseHandler(int page){
         String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/case/effect_case_list?industry=0&style=0&mode=0&page="+page+"&limit=20");
-        System.out.println(result);
         try {
-            ObjectMapper mapper = new ObjectMapper();
             JSONObject node = JSONObject.parseObject(result);
             Integer code = node.getInteger("code");
             String message = node.getString("msg");
@@ -159,11 +149,11 @@ public class OceanEngineServiceImpl implements IOceanEngineService {
                     if (effectCase.getVideoUrl() == null){
                         effectCase.setVideoUrl(getVideoUrl(effectCase.getVid()));
                     }
-                    effectCase.setCoverImage(getVideoFirstImage(effectCase.getVideoUrl()));
-                    if (effectCase.getCoverImage() == null){
-                        Thread.sleep(1000);
-                        effectCase.setCoverImage(getVideoFirstImage(effectCase.getVideoUrl()));
-                    }
+//                    effectCase.setCoverImage(FfmpegUtils.getVideoFirstImage(effectCase.getVideoUrl()));
+//                    if (effectCase.getCoverImage() == null){
+//                        Thread.sleep(1000);
+//                        effectCase.setCoverImage(FfmpegUtils.getVideoFirstImage(effectCase.getVideoUrl()));
+//                    }
                     effectCase.setCreativeLabel(getCase.getString("creative_label"));
                 }
                 effectCaseService.saveOrUpdate(effectCase);
@@ -177,48 +167,6 @@ public class OceanEngineServiceImpl implements IOceanEngineService {
     @Autowired
     private IEffectCaseService effectCaseService;
 
-    public String getVideoFirstImage(String url){
-        try {
-            FFmpegFrameGrabber ff = new FFmpegFrameGrabber(url);
-            ff.start();
-            int lenght = ff.getLengthInFrames();
-            int i = 0;
-            Frame f = null;
-            while (i < lenght) {
-// 过滤前5帧,避免出现全黑的图片,依自己情况而定
-                f = ff.grabFrame();
-                if ((i > 5) && (f.image != null)) {
-                    break;
-                }
-                i++;
-            }
-//            opencv_core.IplImage img = f.image;
-            int owidth = f.imageWidth;
-            int oheight = f.imageHeight;
-            Java2DFrameConverter converter =new Java2DFrameConverter();
-            BufferedImage fecthedImage =converter.getBufferedImage(f);
-// 对截取的帧进行等比例缩放
-            int width = 800;
-            int height = (int) (((double) width / owidth) * oheight);
-            BufferedImage bi = new BufferedImage(width, height, BufferedImage.TYPE_3BYTE_BGR);
-            bi.getGraphics().drawImage(fecthedImage.getScaledInstance(width, height, Image.SCALE_SMOOTH),
-                    0, 0, null);
-            ByteArrayOutputStream baos = new ByteArrayOutputStream();//io流
-            ImageIO.write(bi, "jpg", baos);
-            byte[] bytes = baos.toByteArray();//转换成字节
-            String png_base64 = Base64.getEncoder().encodeToString(bytes).trim();//转换成base64串
-            png_base64 = png_base64.replaceAll("\n", "").replaceAll("\r", "");//删除 \r\n
-            //        ImageIO.write(bufferedImage, "png", new File("D:/qrcode1.png"));
-            baos.close();
-            ff.close();
-            return "data:image/jpg;base64,"+png_base64;
-        }catch (Exception e){
-            e.printStackTrace();
-            return null;
-        }
-
-    }
-
     @Autowired
     private IDouyinHotService douyinHotService;
     @Override
@@ -238,11 +186,11 @@ public class OceanEngineServiceImpl implements IOceanEngineService {
                             if(douyinHot.getUrl() == null){
                                 douyinHot.setUrl(getVideoUrl(douyinHot.getVideoId()));
                             }
-                            douyinHot.setCoverImage(getVideoFirstImage(douyinHot.getUrl()));
-                            if(douyinHot.getCoverImage() == null){
-                                Thread.sleep(1000);
-                                douyinHot.setCoverImage(getVideoFirstImage(douyinHot.getUrl()));
-                            }
+//                            douyinHot.setCoverImage(FfmpegUtils.getVideoFirstImage(douyinHot.getUrl()));
+//                            if(douyinHot.getCoverImage() == null){
+//                                Thread.sleep(1000);
+//                                douyinHot.setCoverImage(FfmpegUtils.getVideoFirstImage(douyinHot.getUrl()));
+//                            }
                             douyinHotService.saveOrUpdate(douyinHot);
                         }
                         douyinHotHandler(page+1,listType);