syh 5 rokov pred
rodič
commit
c26366eb34
16 zmenil súbory, kde vykonal 754 pridanie a 283 odobranie
  1. 0 26
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/KuaishouAutoCommentLoginJob.java
  2. 26 26
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/KuaishouCommentAutoDelete.java
  3. 47 0
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/KuaishouCommentAutoLoginJob.java
  4. 2 5
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/PangolinLoginJob.java
  5. 144 0
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/opencv/utils/ExtractionVideoKeyFrames.java
  6. 1 1
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/system/controller/SysDictController.java
  7. 1 1
      jeecg-boot-module-system/src/main/resources/application-test.yml
  8. 64 3
      jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java
  9. 6 0
      module-common/src/main/java/cn/com/ctop/common/module/service/IBindAccountLoginService.java
  10. 41 0
      module-common/src/main/java/cn/com/ctop/common/module/service/impl/BindAccountLoginServiceImpl.java
  11. 51 1
      module-common/src/main/java/cn/com/ctop/common/module/utils/HttpUtils2.java
  12. 163 2
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/entity/EffectCase.java
  13. 45 40
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/service/impl/OceanEngineServiceImpl.java
  14. 4 13
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinCrawlerServiceImpl.java
  15. 1 0
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/graphql/service/IKuaishouWebInterfaceService.java
  16. 158 165
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/graphql/service/impl/KuaishouWebInterfaceServiceImpl.java

+ 0 - 26
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/KuaishouAutoCommentLoginJob.java

@@ -1,26 +0,0 @@
-package org.jeecg.modules.ctop.job;
-
-import cn.com.ctop.common.module.entity.BindAccountLogin;
-import cn.com.ctop.common.module.service.IBindAccountLoginService;
-import cn.com.ctop.common.module.utils.CtopAdConstant;
-import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouWebInterfaceService;
-import org.quartz.Job;
-import org.quartz.JobExecutionContext;
-import org.quartz.JobExecutionException;
-import org.springframework.beans.factory.annotation.Autowired;
-
-import java.util.List;
-
-public class KuaishouAutoCommentLoginJob implements Job {
-    @Autowired
-    private IBindAccountLoginService bindAccountLoginService;
-    @Autowired
-    private IKuaishouWebInterfaceService kuaishouWebInterfaceService;
-    @Override
-    public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
-        List<BindAccountLogin> loginList = bindAccountLoginService.getListByParams(CtopAdConstant.PLATFORM_TYPE_KUAISHOU_PY,1);
-        if(null!=loginList&&!loginList.isEmpty()){
-            loginList.forEach(login-> kuaishouWebInterfaceService.getkuaishouWebLoginCookie(login));
-        }
-    }
-}

+ 26 - 26
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/KuaishouCommentAutoDelete.java

@@ -13,6 +13,9 @@ import org.springframework.beans.factory.annotation.Value;
 
 import java.util.HashMap;
 import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 
 @Slf4j
 public class KuaishouCommentAutoDelete implements Job {
@@ -22,36 +25,33 @@ public class KuaishouCommentAutoDelete implements Job {
     private IKuaishouWebInterfaceService kuaishouWebInterfaceService;
     @Autowired
     private IBindAccountLoginService bindAccountLoginService;
+    static ExecutorService executorService = null;
+    //线程计数器
+    static CountDownLatch countDownLatch = null;
 
     @Override
     public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
-        Thread thread = new Thread() {
-            @Override
-            public void run() {
-                List<BindAccountLogin>loginList = bindAccountLoginService.getListByParams(CtopAdConstant.PLATFORM_TYPE_KUAISHOU_PY,1);
-                if (loginList != null && !loginList.isEmpty()) {
-                    for (BindAccountLogin account : loginList) {
-                        try {
-                            log.info("正在删除评论:{}", account.getAccountName());
-                            kuaishouWebInterfaceService.adkuaishouWebLogin(account.getAccountName(), account.getPassword());
-                            kuaishouWebInterfaceService.deleteAllComment(new HashMap<>());
-                            Runtime runtime = Runtime.getRuntime();
-                            Process pro = runtime.exec(cleanUrl);
-                            int status = pro.waitFor();
-                            if (status == 0) {
-                                log.info("success,删除评论调用shell脚本执行成功");
-                            } else {
-                                log.info("error,删除评论调用shell脚本执行失败");
-                            }
-                            Thread.sleep(5 * 1000L);
-                        } catch (Exception e) {
-                            e.printStackTrace();
-                        }
-                    }
+        Long start = System.currentTimeMillis();
+        List<BindAccountLogin>loginList = bindAccountLoginService.getListByParams(CtopAdConstant.PLATFORM_TYPE_KUAISHOU_PY,1);
+        if (loginList != null && !loginList.isEmpty()) {
+            executorService = Executors.newFixedThreadPool(5);
+            countDownLatch = new CountDownLatch(loginList.size());
+            loginList.forEach(login -> executorService.submit(()->{
+                try {
+                    kuaishouWebInterfaceService.deleteAllComment(new HashMap<>(),login);
+                }catch (Exception e){
+                    log.error(e.getMessage(),e);
+                }finally {
+                    countDownLatch.countDown();
                 }
+            }));
+            try {
+                countDownLatch.await();
+            } catch (InterruptedException e) {
+                e.printStackTrace();
             }
-        };
-        thread.start();
-
+            Long end = System.currentTimeMillis();
+            log.info("快手删评论所用时长:{}毫秒",end-start);
+        }
     }
 }

+ 47 - 0
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/KuaishouCommentAutoLoginJob.java

@@ -0,0 +1,47 @@
+package org.jeecg.modules.ctop.job;
+
+import cn.com.ctop.common.module.entity.BindAccountLogin;
+import cn.com.ctop.common.module.service.IBindAccountLoginService;
+import cn.com.ctop.common.module.utils.CtopAdConstant;
+import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouWebInterfaceService;
+import lombok.extern.slf4j.Slf4j;
+import org.quartz.Job;
+import org.quartz.JobExecutionContext;
+import org.quartz.JobExecutionException;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+
+import java.util.List;
+
+@Slf4j
+public class KuaishouCommentAutoLoginJob implements Job {
+    @Value("${chrome.script.clean}")
+    private String cleanUrl;
+    @Autowired
+    private IBindAccountLoginService bindAccountLoginService;
+    @Autowired
+    private IKuaishouWebInterfaceService kuaishouWebInterfaceService;
+    @Override
+    public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
+        List<BindAccountLogin> list = bindAccountLoginService.getListByParams(CtopAdConstant.PLATFORM_TYPE_KUAISHOU_PY,1);
+        if(list!=null&&!list.isEmpty()){
+            for (BindAccountLogin login:list) {
+                if(null==login.getCookie()||"".equals(login.getCookie().trim())){
+                    kuaishouWebInterfaceService.getkuaishouWebLoginCookie(login);
+                    Runtime runtime = Runtime.getRuntime();
+                    try {
+                        Process pro = runtime.exec(cleanUrl);
+                        int status = pro.waitFor();
+                        if (status == 0) {
+                            log.info("success,删除评论调用shell脚本执行成功");
+                        } else {
+                            log.info("error,删除评论调用shell脚本执行失败");
+                        }
+                    } catch (Exception e) {
+                        log.error(e.getMessage(),e);
+                    }
+                }
+            }
+        }
+    }
+}

+ 2 - 5
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/PangolinLoginJob.java

@@ -2,8 +2,8 @@ package org.jeecg.modules.ctop.job;
 
 import cn.com.ctop.common.module.entity.BindAccountLogin;
 import cn.com.ctop.common.module.service.IBindAccountLoginService;
+import cn.com.ctop.common.module.utils.CtopAdConstant;
 import cn.com.ctop.crawler.modules.pangolin.service.PangolinLoginService;
-import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import org.quartz.Job;
 import org.quartz.JobExecutionContext;
 import org.quartz.JobExecutionException;
@@ -18,10 +18,7 @@ public class PangolinLoginJob implements Job {
     private PangolinLoginService pangolinLoginService;
     @Override
     public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
-        QueryWrapper<BindAccountLogin> queryWrapper = new QueryWrapper<>();
-        queryWrapper.eq("login_type","pangolin");
-        queryWrapper.eq("status",1);
-        List<BindAccountLogin> list = bindAccountLoginService.list(queryWrapper);
+        List<BindAccountLogin> list = bindAccountLoginService.getListByParams(CtopAdConstant.PLATFORM_TYPE_PANGOLIN_PY,1);
         if(list!=null&&!list.isEmpty()){
             for (BindAccountLogin login:list) {
                 if(null==login.getCookie()||"".equals(login.getCookie().trim())){

+ 144 - 0
jeecg-boot-module-system/src/main/java/org/jeecg/modules/opencv/utils/ExtractionVideoKeyFrames.java

@@ -0,0 +1,144 @@
+package org.jeecg.modules.opencv.utils;
+
+import org.opencv.core.*;
+import org.opencv.imgcodecs.Imgcodecs;
+import org.opencv.imgproc.Imgproc;
+import org.opencv.videoio.VideoCapture;
+import org.opencv.videoio.Videoio;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
+public class ExtractionVideoKeyFrames {
+    Mat computeColorHist3D(Mat im, Size size)
+    {
+        Mat dst = new Mat(), hist = new Mat();
+        Imgproc.resize(im, dst, size, 0, 0, Imgproc.INTER_NEAREST);
+        Mat hsv_src = new Mat();
+        Imgproc.cvtColor(im, hsv_src, Imgproc.COLOR_BGR2HSV);
+        List<Mat> images=new ArrayList<Mat>(3);
+        images.add(hsv_src);
+
+        MatOfInt channels= new MatOfInt(0);
+        MatOfInt hSize = new MatOfInt(256);
+        MatOfFloat ranges= new MatOfFloat(0,255);
+        //获取图像直方图
+        Imgproc.calcHist(images, channels, new Mat(), hist, hSize, ranges);
+        return hist;
+    }
+
+    Mat computeColorHist(Mat im, Size size)
+    {
+        Mat dst = new Mat(), hist = new Mat();
+        Imgproc.resize(im, dst, size, 0, 0, Imgproc.INTER_NEAREST);
+        Mat gray = new Mat(im.size(),CvType.CV_8UC1);
+        Imgproc.cvtColor(im,gray,Imgproc.COLOR_RGB2GRAY);
+
+        List<Mat> images = new ArrayList<Mat>();
+        images.add(gray);
+
+        MatOfInt channels= new MatOfInt(0);
+        MatOfInt hSize = new MatOfInt(128);
+        MatOfFloat ranges= new MatOfFloat(0,128);
+        Imgproc.calcHist(images, channels, new Mat(), hist, hSize, ranges);
+        return hist;
+    }
+
+    double computeASE(Mat a, Mat b) {
+        return  1 - Imgproc.compareHist(a, b, 0);
+    }
+
+    int ExtractKeyframes(String videoPath, double rate, String outImgPath){
+        VideoCapture cap = new VideoCapture();
+        System.out.println(cap.isOpened());
+        if (!cap.open(videoPath)) {
+            System.out.println("Open video file error!");
+            return -1;
+        }
+
+        File file = new File(outImgPath);
+        if(!file.exists()){
+            file.mkdirs();
+        }
+        //  Videoio.CAP_PROP_FRAME_COUNT
+        int width = (int)cap.get(Videoio.CAP_PROP_FRAME_WIDTH);
+        int height = (int)cap.get(Videoio.CAP_PROP_FRAME_HEIGHT);
+        float fps = (float)cap.get(Videoio.CAP_PROP_FPS);
+        int nframes = (int)cap.get(Videoio.CAP_PROP_FRAME_COUNT);
+        System.out.println("W:"+width+";H:"+height+";FPS:"+fps+";N:"+nframes);
+
+        int nscale = 1;
+        if (width%4==0 && height%4==0){
+            nscale = 4;
+        }else if (width%2==0 && height%2==0){
+            nscale = 2;
+        }
+        Size size = new Size(width/nscale, height/nscale);
+
+        Mat frame = new Mat();
+        Mat lastFrame = new Mat();
+        Mat histA = new Mat();
+        Mat histB = new Mat();
+        long nCount = 0;
+        int frameID = 1;
+        int lastFrameID = 1;
+        double tic = (double)Core.getTickCount();
+        while (cap.read(frame)) {
+            if (frame.empty()){
+                break;
+            }
+
+            if (lastFrameID==1) {
+                frameID = (int)cap.get(Videoio.CAP_PROP_POS_FRAMES);
+                lastFrameID = frameID;
+                frame.copyTo(lastFrame);
+                continue;
+            }
+            if (++nCount % 15 != 0){
+                continue;	// Processing one frame in each second
+            }
+
+            frameID = (int)cap.get(Videoio.CAP_PROP_POS_FRAMES);
+            histA = computeColorHist3D(frame, size);
+
+            double aseV = rate;
+            if (histB.empty()){
+                System.out.println(aseV);
+            }
+            else{
+                aseV = computeASE(histA, histB);
+                System.out.println(aseV);
+            }
+
+            if (aseV >= rate) {
+                String imagePath = outImgPath+"/"+lastFrameID+".jpg";
+                if (!Imgcodecs.imwrite(imagePath, lastFrame)){
+                    return -2;
+                }
+                lastFrameID = frameID;
+                frame.copyTo(lastFrame);
+            }
+
+            histA.copyTo(histB);
+        }
+
+        if (!lastFrame.empty()) {
+            String imagePath=outImgPath+"/"+ lastFrameID+".jpg";
+            if (!Imgcodecs.imwrite(imagePath, lastFrame)){
+                return -2;
+            }
+        }
+        System.out.println("NFRAMES(in reality)="+nCount);
+
+        double toc = (double)Core.getTickCount();
+        System.out.println("ELAPSED TIME="+(toc-tic)/Core.getTickFrequency());
+        return 0;
+    }
+
+    public static void main(String[] args) {
+        System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
+        ExtractionVideoKeyFrames evf = new ExtractionVideoKeyFrames();
+        evf.ExtractKeyframes("D:\\data\\fill.mp4",0.5,"D:\\data\\");
+    }
+}

+ 1 - 1
jeecg-boot-module-system/src/main/java/org/jeecg/modules/system/controller/SysDictController.java

@@ -405,7 +405,7 @@ public class SysDictController {
 			@RequestParam(name="text") String text,
 			@RequestParam(name="code") String code,
 			@RequestParam(name="hasChildField") String hasChildField) {
-		Result<List<TreeSelectModel>> result = new Result<List<TreeSelectModel>>();
+		Result<List<TreeSelectModel>> result = new Result<>();
 		List<TreeSelectModel> ls = sysDictService.queryTreeList(tbname, text, code, pidField, pid,hasChildField);
 		result.setSuccess(true);
 		result.setResult(ls);

+ 1 - 1
jeecg-boot-module-system/src/main/resources/application-test.yml

@@ -146,7 +146,7 @@ jeecg:
     webapp: /data/webapp
     video-upload: /data/upload/video/
     image-upload: /data/upload/image/
-    chrome-driver: /usr/bin/chromedriver
+    chrome-driver: D://chromedriver.exe
     csv-upload: /data/upload/csv/
     report-history: /data/report/history/
     report-daily: /data/report/daily/

+ 64 - 3
jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java

@@ -1,6 +1,10 @@
 package org.jeecg;
 
+import cn.com.ctop.common.module.entity.BindAccountLogin;
+import cn.com.ctop.common.module.service.IBindAccountLoginService;
+import cn.com.ctop.common.module.utils.CtopAdConstant;
 import cn.com.ctop.crawler.modules.oceanengine.service.IOceanEngineService;
+import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouWebInterfaceService;
 import lombok.extern.slf4j.Slf4j;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -8,6 +12,12 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.test.context.junit4.SpringRunner;
 
+import java.util.HashMap;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
 @RunWith(SpringRunner.class)
 @SpringBootTest
 @Slf4j
@@ -19,16 +29,67 @@ public class SampleTest {
         String account="3248395570@qq.com";
         String password = "Ydxq-704127411";
         oceanEngineService.login(account,password);
-//        oceanEngineService.douyinHotHandler(1,1);
+        oceanEngineService.douyinHotHandler(1,1);
         oceanEngineService.effectCaseHandler(1);
-        oceanEngineService.hotMaterialHandler(1,1,"西瓜");
-        oceanEngineService.hotMaterialHandler(1,3,"火山");
         oceanEngineService.hotMaterialHandler(1,4,"抖音");
         oceanEngineService.hotMaterialHandler(1,8,"头条");
+        oceanEngineService.hotMaterialHandler(1,1,"西瓜");
+        oceanEngineService.hotMaterialHandler(1,3,"火山");
         oceanEngineService.hotMaterialHandler(1,9,"穿山甲");
         log.info("巨量创意抓取完成");
     }
+
+    @Autowired
+    private IBindAccountLoginService bindAccountLoginService;
+    @Autowired
+    private IKuaishouWebInterfaceService kuaishouWebInterfaceService;
+    @Test
+    public void kuaishouLogin(){
+        Long start =System.currentTimeMillis();
+        List<BindAccountLogin> list = bindAccountLoginService.getListByParams(CtopAdConstant.PLATFORM_TYPE_KUAISHOU_PY,1);
+        if(list!=null&&!list.isEmpty()){
+            for (BindAccountLogin login:list) {
+                if(null==login.getCookie()||"".equals(login.getCookie().trim())){
+                    kuaishouWebInterfaceService.getkuaishouWebLoginCookie(login);
+                }
+            }
+        }
+        Long end = System.currentTimeMillis();
+        log.info("总用时:{}毫秒",end-start);
+    }
+    static ExecutorService executorService = null;
+    static CountDownLatch countDownLatch = null;
+    @Test
+    public void kuaishouCommentDelete(){
+        Long start =System.currentTimeMillis();
+        List<BindAccountLogin> list = bindAccountLoginService.getListByParams(CtopAdConstant.PLATFORM_TYPE_KUAISHOU_PY,1);
+        if(list!=null&&!list.isEmpty()){
+            executorService = Executors.newFixedThreadPool(10);
+            countDownLatch = new CountDownLatch(list.size());
+            list.forEach(login -> {
+                executorService.submit(()->{
+                    try {
+                        if(null!=login.getCookie()&&!"".equals(login.getCookie().trim())){
+                            kuaishouWebInterfaceService.deleteAllComment(new HashMap<>(),login);
+                        }
+                    }catch (Exception e){
+
+                    }finally {
+                        countDownLatch.countDown();
+                    }
+                });
+            });
+        }
+        try {
+            countDownLatch.await();
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+        }
+        Long end = System.currentTimeMillis();
+        log.info("总用时:{}毫秒",end-start);
+    }
 }
+//刷新token:885936毫秒
 
 
 

+ 6 - 0
module-common/src/main/java/cn/com/ctop/common/module/service/IBindAccountLoginService.java

@@ -2,6 +2,8 @@ package cn.com.ctop.common.module.service;
 
 import cn.com.ctop.common.module.entity.BindAccountLogin;
 import com.baomidou.mybatisplus.extension.service.IService;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import org.apache.http.client.CookieStore;
 
 import java.util.List;
 import java.util.Map;
@@ -13,6 +15,8 @@ import java.util.Map;
  * @Version: V1.0
  */
 public interface IBindAccountLoginService extends IService<BindAccountLogin> {
+    boolean checkLogin(BindAccountLogin bindAccountLogin, String response);
+
     /**
      * 查询
      *
@@ -24,4 +28,6 @@ public interface IBindAccountLoginService extends IService<BindAccountLogin> {
     Map<String, Object> getListByUserId(String userId);
 
     List<BindAccountLogin> getListByParams(String loginType,Integer status);
+
+    CookieStore setCookie(BindAccountLogin bindAccountLogin,String domain) throws JsonProcessingException;
 }

+ 41 - 0
module-common/src/main/java/cn/com/ctop/common/module/service/impl/BindAccountLoginServiceImpl.java

@@ -8,7 +8,13 @@ import cn.com.ctop.common.module.utils.ResultMapUtils;
 import cn.com.ctop.common.module.utils.StatusCode;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.http.client.CookieStore;
+import org.apache.http.impl.client.BasicCookieStore;
+import org.apache.http.impl.cookie.BasicClientCookie;
 import org.springframework.stereotype.Service;
 
 import java.util.ArrayList;
@@ -26,6 +32,15 @@ import java.util.Map;
 @Service
 public class BindAccountLoginServiceImpl extends ServiceImpl<BindAccountLoginMapper, BindAccountLogin> implements IBindAccountLoginService {
     @Override
+    public boolean checkLogin(BindAccountLogin bindAccountLogin, String response) {
+        if (response.contains("<html>")) {
+            bindAccountLogin.setCookie("");
+            this.updateById(bindAccountLogin);
+            return false;
+        }
+        return true;
+    }
+    @Override
     public BindAccountLogin selectByAccountId(String accountId) {
         QueryWrapper<BindAccountLogin> queryWrapper = new QueryWrapper<>();
         queryWrapper.eq("account_id", accountId).orderByDesc("id").last("limit 1");
@@ -60,4 +75,30 @@ public class BindAccountLoginServiceImpl extends ServiceImpl<BindAccountLoginMap
         }
         return this.list(queryWrapper);
     }
+
+    @Override
+    public CookieStore setCookie(BindAccountLogin bindAccountLogin,String domain) throws JsonProcessingException {
+        ObjectMapper mapper = new ObjectMapper();
+        List<String> cookieList = mapper.readValue(bindAccountLogin.getCookie(), new TypeReference<List<String>>() {});
+        if(cookieList == null||cookieList.isEmpty()){
+            return null;
+        }
+        CookieStore cookieStore = new BasicCookieStore();
+        for (String cookieStr : cookieList) {
+            String[] cookieArray = cookieStr.split(";");
+            Map<String, String> cookieMap = new HashMap<>();
+            for (String cookie : cookieArray) {
+                String[] kv = cookie.split("=");
+                if (kv.length > 1) {
+                    cookieMap.put(kv[0].trim(), kv[1].trim());
+                }
+            }
+            BasicClientCookie clientCookie = new BasicClientCookie(cookieArray[0].split("=")[0],
+                    cookieArray[0].split("=")[1]);
+            clientCookie.setDomain(domain);
+            clientCookie.setPath(cookieMap.get("path"));
+            cookieStore.addCookie(clientCookie);
+        }
+        return cookieStore;
+    }
 }

+ 51 - 1
module-common/src/main/java/cn/com/ctop/common/module/utils/HttpUtils2.java

@@ -45,7 +45,7 @@ public class HttpUtils2 {
     /**
      * setProxy(new HttpHost("106.125.239.179", 4245))
      */
-    public static Map<String, Map<String, String>> fontsMap = new HashMap<String, Map<String, String>>();
+    public static Map<String, Map<String, String>> fontsMap = new HashMap<>();
     public static String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36";
 
     public static CloseableHttpClient createSSLClientDefault() {
@@ -66,6 +66,24 @@ public class HttpUtils2 {
         return HttpClients.createDefault();
     }
 
+    public static CloseableHttpClient createSSLClientDefault(CookieStore setCookieStore) {
+        try {
+            SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {
+                //信任所有证书
+                @Override
+                public boolean isTrusted(X509Certificate[] chain, String authType) throws CertificateException {
+                    return true;
+                }
+            }).build();
+            RequestConfig globalConfig = RequestConfig.custom().setConnectTimeout(30000).setSocketTimeout(30000).setCookieSpec(CookieSpecs.STANDARD).build();
+            SSLConnectionSocketFactory sslFactory = new SSLConnectionSocketFactory(sslContext);
+            return HttpClients.custom().setDefaultCookieStore(setCookieStore).setDefaultRequestConfig(globalConfig).setConnectionReuseStrategy((response, context) -> false).setSSLSocketFactory(sslFactory).build();
+        } catch (Exception e) {
+            log.error("处理Https证书异常", e);
+        }
+        return HttpClients.createDefault();
+    }
+
     public static String httpPostParamRequest(String url, Map<String, Object> param, Map<String, String> headers) {
         HttpClient httpClient = createSSLClientDefault();
         String strReturn = "";
@@ -569,4 +587,36 @@ public class HttpUtils2 {
     }
 
 
+    public static String httpPostRequest(String url, Map<String, Object> param, Map<String, String> headers, CookieStore cookieStore) {
+        HttpClient httpClient = createSSLClientDefault(cookieStore);
+        String strReturn = "";
+        try {
+            HttpPost httppost = new HttpPost(url);
+            httppost.setHeader("User-Agent", USER_AGENT);
+            httppost.addHeader("Content-Type", "application/json");
+            if (headers != null) {
+                for (String key : headers.keySet()) {
+                    httppost.setHeader(key, headers.get(key));
+                }
+            }
+
+            httppost.setEntity(new StringEntity(new Gson().toJson(param), Charset.forName("UTF-8")));
+            HttpEntity respentity;
+
+            HttpResponse response = httpClient.execute(httppost);
+            int statusCode = response.getStatusLine().getStatusCode();
+            if (statusCode == HttpStatus.SC_MOVED_PERMANENTLY || statusCode == HttpStatus.SC_MOVED_TEMPORARILY) {
+                String newUrl = response.getFirstHeader("Location").getValue();
+                return httpPostRequest(newUrl, param, headers);
+            } else if (statusCode == HttpStatus.SC_OK) {
+                respentity = response.getEntity();
+                strReturn = EntityUtils.toString(respentity);
+                return strReturn;
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+            log.error(e.getMessage());
+        }
+        return strReturn;
+    }
 }

+ 163 - 2
module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/entity/EffectCase.java

@@ -6,13 +6,14 @@ import com.baomidou.mybatisplus.annotation.TableId;
 import com.baomidou.mybatisplus.annotation.TableName;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import lombok.Data;
+
 import java.util.Date;
 
-@Data
-@TableName("ctop_oceanengine_effect_case")
 /**
  * 效果案例
  */
+@Data
+@TableName("ctop_oceanengine_effect_case")
 public class EffectCase {
     @TableId(value = "id", type = IdType.INPUT)
     private String id;
@@ -47,4 +48,164 @@ public class EffectCase {
     private String industryTag;
     private Date createTime;
     private Date updateTime;
+
+    public String getId() {
+        return id;
+    }
+
+    public void setId(String id) {
+        this.id = id;
+    }
+
+    public String getVid() {
+        return vid;
+    }
+
+    public void setVid(String vid) {
+        this.vid = vid;
+    }
+
+    public String getCreativeLabel1() {
+        return creativeLabel1;
+    }
+
+    public void setCreativeLabel1(String creativeLabel1) {
+        this.creativeLabel1 = creativeLabel1;
+    }
+
+    public Integer getCreativeLabel2() {
+        return creativeLabel2;
+    }
+
+    public void setCreativeLabel2(Integer creativeLabel2) {
+        this.creativeLabel2 = creativeLabel2;
+    }
+
+    public String getCreateDate() {
+        return createDate;
+    }
+
+    public void setCreateDate(String createDate) {
+        this.createDate = createDate;
+    }
+
+    public Long getCreativeId() {
+        return creativeId;
+    }
+
+    public void setCreativeId(Long creativeId) {
+        this.creativeId = creativeId;
+    }
+
+    public String getMethods() {
+        return methods;
+    }
+
+    public void setMethods(String methods) {
+        this.methods = methods;
+    }
+
+    public String getVideoUrl() {
+        return videoUrl;
+    }
+
+    public void setVideoUrl(String videoUrl) {
+        this.videoUrl = videoUrl;
+    }
+
+    public String getAuthor() {
+        return author;
+    }
+
+    public void setAuthor(String author) {
+        this.author = author;
+    }
+
+    public String getEffect() {
+        return effect;
+    }
+
+    public void setEffect(String effect) {
+        this.effect = effect;
+    }
+
+    public String getConversionType() {
+        return conversionType;
+    }
+
+    public void setConversionType(String conversionType) {
+        this.conversionType = conversionType;
+    }
+
+    public String getCoverImage() {
+        return coverImage;
+    }
+
+    public void setCoverImage(String coverImage) {
+        this.coverImage = coverImage;
+    }
+
+    public String getCreativeLabel() {
+        return creativeLabel;
+    }
+
+    public void setCreativeLabel(String creativeLabel) {
+        this.creativeLabel = creativeLabel;
+    }
+
+    public String getTag() {
+        return tag;
+    }
+
+    public void setTag(String tag) {
+        this.tag = tag;
+    }
+
+    public String getCreativeTitle() {
+        return creativeTitle;
+    }
+
+    public void setCreativeTitle(String creativeTitle) {
+        this.creativeTitle = creativeTitle;
+    }
+
+    public String getCreativeInfo() {
+        return creativeInfo;
+    }
+
+    public void setCreativeInfo(String creativeInfo) {
+        this.creativeInfo = creativeInfo;
+    }
+
+    public String getCreativeScore() {
+        return creativeScore;
+    }
+
+    public void setCreativeScore(String creativeScore) {
+        this.creativeScore = creativeScore;
+    }
+
+    public String getIndustryTag() {
+        return industryTag;
+    }
+
+    public void setIndustryTag(String industryTag) {
+        this.industryTag = industryTag;
+    }
+
+    public Date getCreateTime() {
+        return createTime;
+    }
+
+    public void setCreateTime(Date createTime) {
+        this.createTime = createTime;
+    }
+
+    public Date getUpdateTime() {
+        return updateTime;
+    }
+
+    public void setUpdateTime(Date updateTime) {
+        this.updateTime = updateTime;
+    }
 }

+ 45 - 40
module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/service/impl/OceanEngineServiceImpl.java

@@ -11,10 +11,13 @@ import cn.com.ctop.crawler.modules.oceanengine.service.IDouyinHotService;
 import cn.com.ctop.crawler.modules.oceanengine.service.IEffectCaseService;
 import cn.com.ctop.crawler.modules.oceanengine.service.IHotMaterialService;
 import cn.com.ctop.crawler.modules.oceanengine.service.IOceanEngineService;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.http.impl.cookie.BasicClientCookie;
 import org.bytedeco.javacv.FFmpegFrameGrabber;
 import org.bytedeco.javacv.Frame;
@@ -37,6 +40,7 @@ import java.util.List;
 import java.util.*;
 
 @Service
+@Slf4j
 public class OceanEngineServiceImpl implements IOceanEngineService {
     @Value("${jeecg.path.chrome-driver}")
     private String chromeDriver;
@@ -66,7 +70,7 @@ public class OceanEngineServiceImpl implements IOceanEngineService {
 
     @Override
     public void hotMaterialHandler(int page, int appCode, String appName){
-        String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_material/list?material_type=3&order_by=convert_show_rate&period_type=3&aggr_app_code="+appCode+"&page="+page+"&limit=20");
+        String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_material/list?material_type=3&order_by=click_show_rate&period_type=30&aggr_app_code="+appCode+"&page="+page+"&limit=20");
         System.out.println(result);
         ObjectMapper mapper = new ObjectMapper();
         mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
@@ -121,49 +125,50 @@ public class OceanEngineServiceImpl implements IOceanEngineService {
     public void effectCaseHandler(int page){
         String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/case/effect_case_list?industry=0&style=0&mode=0&page="+page+"&limit=20");
         System.out.println(result);
-        ObjectMapper mapper = new ObjectMapper();
-        mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
         try {
-            JsonNode node = mapper.readTree(result);
-            int code = node.get("code").asInt();
-            if(code == 0) {
-                if (node.get("data") != null) {
-                    String dataJson = node.get("data").get("effect_cases").toString();
-                    List<EffectCase> list = mapper.readValue(dataJson,new TypeReference<List<EffectCase>>() {});
-                    if (list != null && list.size() > 0) {
-                        for (EffectCase effectCase:list){
-                            String resultDetail = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/case/effect_case/"+effectCase.getId());
-                            JsonNode detailNode = mapper.readTree(resultDetail);
-                            int detailCode = detailNode.get("code").asInt();
-                            if(detailCode == 0){
-                                if(detailNode.get("data") != null){
-//                                    String detailJson = node.get("data").get("effect_cases").toString();
-//                                    EffectCase effectCase1 = mapper.readValue(detailJson,EffectCase.class);
-                                    effectCase.setAuthor(detailNode.get("data").get("effect_case").get("author").asText());
-                                    effectCase.setCreateDate(detailNode.get("data").get("effect_case").get("create_date").asText());
-                                    effectCase.setCreativeId(detailNode.get("data").get("effect_case").get("creative_id").asLong());
-                                    effectCase.setCreativeScore(detailNode.get("data").get("effect_case").get("creative_score").asText());
-                                    effectCase.setEffect(detailNode.get("data").get("effect_case").get("effect").toString());
-                                    effectCase.setMethods(detailNode.get("data").get("effect_case").get("methods").toString());
-                                    effectCase.setVideoUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+effectCase.getVid()+"&line=0"));
-                                    if (effectCase.getVideoUrl() == null){
-                                        effectCase.setVideoUrl(getVideoUrl(effectCase.getVid()));
-                                    }
-//                                    effectCase.setVideoUrl(getVideoUrl(effectCase.getVid()));
-                                    effectCase.setCoverImage(getVideoFirstImage(effectCase.getVideoUrl()));
-                                    if (effectCase.getCoverImage() == null){
-                                        Thread.sleep(1000);
-                                        effectCase.setCoverImage(getVideoFirstImage(effectCase.getVideoUrl()));
-                                    }
-                                    effectCase.setCreativeLabel(detailNode.get("data").get("effect_case").get("creative_label").toString());
-                                }
-                            }
-                            effectCaseService.saveOrUpdate(effectCase);
-                        }
-                        effectCaseHandler(page+1);
+            ObjectMapper mapper = new ObjectMapper();
+            JSONObject node = JSONObject.parseObject(result);
+            Integer code = node.getInteger("code");
+            String message = node.getString("msg");
+            if(null==code||code != 0) {
+                log.error("巨量创意获取效果案例异常:{}",message);
+                return;
+            }
+            JSONArray effectCaseList = node.getJSONObject("data").getJSONArray("effect_cases");
+            if (effectCaseList == null || effectCaseList.isEmpty()) {
+                log.error("巨量创意获取效果案例数据为空:{}",message);
+                return;
+            }
+            List<EffectCase> effectCases = effectCaseList.toJavaList(EffectCase.class);
+            for (EffectCase effectCase:effectCases){
+                String resultDetail = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/case/effect_case/"+effectCase.getId());
+                JSONObject detailNode = JSONObject.parseObject(resultDetail);
+                Integer detailCode = detailNode.getInteger("code");
+                if(null ==detailCode|| detailCode != 0){
+                    continue;
+                }
+                JSONObject getCase = detailNode.getJSONObject("data").getJSONObject("effect_case");
+                if(getCase != null){
+                    effectCase.setAuthor(getCase.getString("author"));
+                    effectCase.setCreateDate(getCase.getString("create_date"));
+                    effectCase.setCreativeId(getCase.getLong("creative_id"));
+                    effectCase.setCreativeScore(getCase.getString("creative_score"));
+                    effectCase.setEffect(getCase.getString("effect"));
+                    effectCase.setMethods(getCase.getString("methods"));
+                    effectCase.setVideoUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+effectCase.getVid()+"&line=0"));
+                    if (effectCase.getVideoUrl() == null){
+                        effectCase.setVideoUrl(getVideoUrl(effectCase.getVid()));
+                    }
+                    effectCase.setCoverImage(getVideoFirstImage(effectCase.getVideoUrl()));
+                    if (effectCase.getCoverImage() == null){
+                        Thread.sleep(1000);
+                        effectCase.setCoverImage(getVideoFirstImage(effectCase.getVideoUrl()));
                     }
+                    effectCase.setCreativeLabel(getCase.getString("creative_label"));
                 }
+                effectCaseService.saveOrUpdate(effectCase);
             }
+            effectCaseHandler(page+1);
         }catch (Exception e){
             e.printStackTrace();
         }

+ 4 - 13
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinCrawlerServiceImpl.java

@@ -58,7 +58,7 @@ public class PangolinCrawlerServiceImpl implements PangolinCrawlerService {
                 }
             }
             String result = HttpUtils2.httpPostRequest(url, null, null);
-            if (checkLogin(bindAccountLogin, result)) {
+            if (bindAccountLoginService.checkLogin(bindAccountLogin, result)) {
                 JsonNode resultNode = mapper.readTree(result);
                 Iterator<JsonNode> appIterator = resultNode.get("data").get("List").elements();
                 while (appIterator.hasNext()) {
@@ -122,7 +122,7 @@ public class PangolinCrawlerServiceImpl implements PangolinCrawlerService {
                 }
             }
             String result = HttpUtils2.httpGetRequest(url);
-            if (checkLogin(bindAccountLogin, result)) {
+            if (bindAccountLoginService.checkLogin(bindAccountLogin, result)) {
                 JsonNode resultNode = mapper.readTree(result);
                 Iterator<JsonNode> appIterator = resultNode.get("data").get("AppNameList").elements();
                 while (appIterator.hasNext()) {
@@ -188,7 +188,7 @@ public class PangolinCrawlerServiceImpl implements PangolinCrawlerService {
             paramMap.put("search", searchMap);
             paramMap.put("page", pageMap);
             String result = HttpUtils2.httpPostRequest(url, paramMap, null);
-            if (checkLogin(bindAccountLogin, result)) {
+            if (bindAccountLoginService.checkLogin(bindAccountLogin, result)) {
                 JsonNode resultNode = mapper.readTree(result);
                 Iterator<JsonNode> listIterator = resultNode.get("data").get("list").elements();
                 while (listIterator.hasNext()) {
@@ -259,7 +259,7 @@ public class PangolinCrawlerServiceImpl implements PangolinCrawlerService {
             paramMap.put("channelName", "");
             paramMap.put("appId", appIds);
             String result = HttpUtils2.httpPostRequest(url, paramMap, null);
-            if (checkLogin(bindAccountLogin, result)) {
+            if (bindAccountLoginService.checkLogin(bindAccountLogin, result)) {
                 JsonNode resultNode = mapper.readTree(result);
                 Iterator<String> clickIterator = resultNode.get("data").get("ChartData").get("click_num").fieldNames();
                 while (clickIterator.hasNext()) {
@@ -341,13 +341,4 @@ public class PangolinCrawlerServiceImpl implements PangolinCrawlerService {
             e.printStackTrace();
         }
     }
-
-    public boolean checkLogin(BindAccountLogin bindAccountLogin, String response) {
-        if (response.contains("<html>")) {
-            bindAccountLogin.setCookie("");
-            bindAccountLoginService.updateById(bindAccountLogin);
-            return false;
-        }
-        return true;
-    }
 }

+ 1 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/graphql/service/IKuaishouWebInterfaceService.java

@@ -13,6 +13,7 @@ public interface IKuaishouWebInterfaceService {
     void adkuaishouWebLogin(String phone, String password) throws IOException;
 
     void deleteAllComment(Map<String, String> pcursorMap);
+    void deleteAllComment(Map<String,String>pcursorMap,BindAccountLogin login);
 
     String qrLogin(String taskId) throws Exception;
 

Rozdielové dáta súboru neboli zobrazené, pretože súbor je príliš veľký
+ 158 - 165
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/graphql/service/impl/KuaishouWebInterfaceServiceImpl.java