Procházet zdrojové kódy

Merge branch 'master' into master_batch_upload

# Conflicts:
#	jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java
yumeng před 5 roky
rodič
revize
386de814ee
28 změnil soubory, kde provedl 1286 přidání a 1005 odebrání
  1. 1 0
      jeecg-boot-module-system/src/main/java/org/jeecg/config/ShiroConfig.java
  2. 20 8
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/PangolinController.java
  3. 27 30
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/KuaishouCommentAutoDelete.java
  4. 47 0
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/KuaishouCommentAutoLoginJob.java
  5. 10 342
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/OceanengineJob.java
  6. 2 5
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/PangolinLoginJob.java
  7. 144 0
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/opencv/utils/ExtractionVideoKeyFrames.java
  8. 1 1
      jeecg-boot-module-system/src/main/resources/application-test.yml
  9. 70 78
      jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java
  10. 8 0
      module-common/src/main/java/cn/com/ctop/common/module/entity/BindAccountLogin.java
  11. 13 0
      module-common/src/main/java/cn/com/ctop/common/module/service/IBindAccountLoginService.java
  12. 78 1
      module-common/src/main/java/cn/com/ctop/common/module/service/impl/BindAccountLoginServiceImpl.java
  13. 51 1
      module-common/src/main/java/cn/com/ctop/common/module/utils/HttpUtils2.java
  14. 54 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/core/util/FfmpegUtils.java
  15. 1 1
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/entity/DouyinHot.java
  16. 196 3
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/entity/EffectCase.java
  17. 1 1
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/entity/HotMaterial.java
  18. 3 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/service/IDouyinHotService.java
  19. 5 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/service/IHotMaterialService.java
  20. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/service/IOceanEngineService.java
  21. 8 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/service/impl/DouyinHotServiceImpl.java
  22. 10 1
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/service/impl/HotMaterialServiceImpl.java
  23. 299 340
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/service/impl/OceanEngineServiceImpl.java
  24. 4 13
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinCrawlerServiceImpl.java
  25. 1 2
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinLoginServiceImpl.java
  26. 4 2
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/controller/BatchController.java
  27. 18 13
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/graphql/service/IKuaishouWebInterfaceService.java
  28. 203 163
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/graphql/service/impl/KuaishouWebInterfaceServiceImpl.java

+ 1 - 0
jeecg-boot-module-system/src/main/java/org/jeecg/config/ShiroConfig.java

@@ -147,6 +147,7 @@ public class ShiroConfig {
 		filterChainDefinitionMap.put("/ks/web/test", "anon");
         filterChainDefinitionMap.put("/ureport/**", "anon");
         filterChainDefinitionMap.put("/ctop/advertiser/**", "anon");
+		filterChainDefinitionMap.put("/pangolin/**", "anon");
 
 		// 添加自己的过滤器并且取名为jwt
 		Map<String, Filter> filterMap = new HashMap<>(1);

+ 20 - 8
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/PangolinController.java

@@ -37,6 +37,7 @@ public class PangolinController {
     private ISysRoleService sysRoleService;
     @Autowired
     private IBindAccountLoginService bindAccountLoginService;
+
     @Autowired
     private IUserAllocationService userAllocationService;
     @PostMapping("api/user/list")
@@ -66,9 +67,9 @@ public class PangolinController {
 
     @PostMapping("api/channel/list")
     public Map<String, Object> channelList(@RequestBody JSONObject data) {
-//        LoginUser user = (LoginUser) SecurityUtils.getSubject().getPrincipal();
-//        String userId = user.getId();
-        String userId = "6b4c821adc414dc8b7718ab63ccbfcaf";
+        LoginUser user = (LoginUser) SecurityUtils.getSubject().getPrincipal();
+        String userId = user.getId();
+//        String userId = "6b4c821adc414dc8b7718ab63ccbfcaf";
         String roleCode = sysRoleService.getRoleCodeByUserId(userId);
         if(null!=roleCode&&(roleCode.equals("admin"))){
             userId = null;
@@ -76,12 +77,23 @@ public class PangolinController {
         return pangolinChannelService.getListByUserId(userId);
     }
 
+//    @PostMapping("api/user/list")
+//    public Map<String, Object> userList() {
+//        LoginUser user = (LoginUser) SecurityUtils.getSubject().getPrincipal();
+//        String userId = user.getId();
+////        String userId = "6b4c821adc414dc8b7718ab63ccbfcaf";
+//        String roleCode = sysRoleService.getRoleCodeByUserId(userId);
+//        if(null!=roleCode&&(roleCode.equals("admin"))){
+//            userId = null;
+//        }
+//        return bindAccountLoginService.getListByUserId(userId);
+//    }
 
     @PostMapping("api/app/list")
     public Map<String, Object> appList(@RequestBody JSONObject data) {
-//        LoginUser user = (LoginUser) SecurityUtils.getSubject().getPrincipal();
-//        String userId = user.getId();
-        String userId = "6b4c821adc414dc8b7718ab63ccbfcaf";
+        LoginUser user = (LoginUser) SecurityUtils.getSubject().getPrincipal();
+        String userId = user.getId();
+//        String userId = "6b4c821adc414dc8b7718ab63ccbfcaf";
         String roleCode = sysRoleService.getRoleCodeByUserId(userId);
         if(null!=roleCode&&(roleCode.equals("admin"))){
             userId = null;
@@ -111,8 +123,8 @@ public class PangolinController {
     @PostMapping("api/activation/list")
     public Map<String, Object> activationList(@RequestBody JSONObject data) {
         LoginUser user = (LoginUser) SecurityUtils.getSubject().getPrincipal();
-        String userId = user.getId();
-//        String userId = "6b4c821adc414dc8b7718ab63ccbfcaf";
+//        String userId = user.getId();
+        String userId = "6b4c821adc414dc8b7718ab63ccbfcaf";
         String roleCode = sysRoleService.getRoleCodeByUserId(userId);
         if(null!=roleCode&&(roleCode.equals("admin"))){
             userId = null;

+ 27 - 30
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/KuaishouCommentAutoDelete.java

@@ -2,8 +2,8 @@ package org.jeecg.modules.ctop.job;
 
 import cn.com.ctop.common.module.entity.BindAccountLogin;
 import cn.com.ctop.common.module.service.IBindAccountLoginService;
+import cn.com.ctop.common.module.utils.CtopAdConstant;
 import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouWebInterfaceService;
-import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import lombok.extern.slf4j.Slf4j;
 import org.quartz.Job;
 import org.quartz.JobExecutionContext;
@@ -13,6 +13,9 @@ import org.springframework.beans.factory.annotation.Value;
 
 import java.util.HashMap;
 import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 
 @Slf4j
 public class KuaishouCommentAutoDelete implements Job {
@@ -22,39 +25,33 @@ public class KuaishouCommentAutoDelete implements Job {
     private IKuaishouWebInterfaceService kuaishouWebInterfaceService;
     @Autowired
     private IBindAccountLoginService bindAccountLoginService;
+    static ExecutorService executorService = null;
+    //线程计数器
+    static CountDownLatch countDownLatch = null;
 
     @Override
     public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
-        Thread thread = new Thread() {
-            @Override
-            public void run() {
-                QueryWrapper<BindAccountLogin> queryWrapper = new QueryWrapper<>();
-                queryWrapper.eq("login_type", "kuaishou");
-                queryWrapper.eq("status", 1);
-                List<BindAccountLogin> list = bindAccountLoginService.list(queryWrapper);
-                if (list != null && list.size() > 0) {
-                    for (BindAccountLogin account : list) {
-                        try {
-                            log.info("正在删除评论:{}", account.getAccountName());
-                            kuaishouWebInterfaceService.adkuaishouWebLogin(account.getAccountName(), account.getPassword());
-                            kuaishouWebInterfaceService.deleteAllComment(new HashMap<>());
-                            Runtime runtime = Runtime.getRuntime();
-                            Process pro = runtime.exec(cleanUrl);
-                            int status = pro.waitFor();
-                            if (status == 0) {
-                                log.info("success,删除评论调用shell脚本执行成功");
-                            } else {
-                                log.info("error,删除评论调用shell脚本执行失败");
-                            }
-                            Thread.sleep(5 * 1000L);
-                        } catch (Exception e) {
-                            e.printStackTrace();
-                        }
-                    }
+        Long start = System.currentTimeMillis();
+        List<BindAccountLogin>loginList = bindAccountLoginService.getListByParams(CtopAdConstant.PLATFORM_TYPE_KUAISHOU_PY,1);
+        if (loginList != null && !loginList.isEmpty()) {
+            executorService = Executors.newFixedThreadPool(5);
+            countDownLatch = new CountDownLatch(loginList.size());
+            loginList.forEach(login -> executorService.submit(()->{
+                try {
+                    kuaishouWebInterfaceService.deleteAllComment(new HashMap<>(),login);
+                }catch (Exception e){
+                    log.error(e.getMessage(),e);
+                }finally {
+                    countDownLatch.countDown();
                 }
+            }));
+            try {
+                countDownLatch.await();
+            } catch (InterruptedException e) {
+                e.printStackTrace();
             }
-        };
-        thread.start();
-
+            Long end = System.currentTimeMillis();
+            log.info("快手删评论所用时长:{}毫秒",end-start);
+        }
     }
 }

+ 47 - 0
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/KuaishouCommentAutoLoginJob.java

@@ -0,0 +1,47 @@
+package org.jeecg.modules.ctop.job;
+
+import cn.com.ctop.common.module.entity.BindAccountLogin;
+import cn.com.ctop.common.module.service.IBindAccountLoginService;
+import cn.com.ctop.common.module.utils.CtopAdConstant;
+import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouWebInterfaceService;
+import lombok.extern.slf4j.Slf4j;
+import org.quartz.Job;
+import org.quartz.JobExecutionContext;
+import org.quartz.JobExecutionException;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+
+import java.util.List;
+
+@Slf4j
+public class KuaishouCommentAutoLoginJob implements Job {
+    @Value("${chrome.script.clean}")
+    private String cleanUrl;
+    @Autowired
+    private IBindAccountLoginService bindAccountLoginService;
+    @Autowired
+    private IKuaishouWebInterfaceService kuaishouWebInterfaceService;
+    @Override
+    public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
+        List<BindAccountLogin> list = bindAccountLoginService.getListByParams(CtopAdConstant.PLATFORM_TYPE_KUAISHOU_PY,1);
+        if(list!=null&&!list.isEmpty()){
+            for (BindAccountLogin login:list) {
+                if(null==login.getCookie()||"".equals(login.getCookie().trim())){
+                    kuaishouWebInterfaceService.getkuaishouWebLoginCookie(login);
+                    Runtime runtime = Runtime.getRuntime();
+                    try {
+                        Process pro = runtime.exec(cleanUrl);
+                        int status = pro.waitFor();
+                        if (status == 0) {
+                            log.info("success,删除评论调用shell脚本执行成功");
+                        } else {
+                            log.info("error,删除评论调用shell脚本执行失败");
+                        }
+                    } catch (Exception e) {
+                        log.error(e.getMessage(),e);
+                    }
+                }
+            }
+        }
+    }
+}

+ 10 - 342
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/OceanengineJob.java

@@ -1,361 +1,29 @@
 package org.jeecg.modules.ctop.job;
 
-import cn.com.ctop.common.module.utils.Base64Utils;
-import cn.com.ctop.common.module.utils.HttpUtils2;
-import cn.com.ctop.crawler.modules.core.util.FateadmHttpUtil;
-import cn.com.ctop.crawler.modules.core.util.FateadmUtil;
-import cn.com.ctop.crawler.modules.oceanengine.entity.DouyinHot;
-import cn.com.ctop.crawler.modules.oceanengine.entity.EffectCase;
-import cn.com.ctop.crawler.modules.oceanengine.entity.HotMaterial;
-import cn.com.ctop.crawler.modules.oceanengine.service.IDouyinHotService;
-import cn.com.ctop.crawler.modules.oceanengine.service.IEffectCaseService;
-import cn.com.ctop.crawler.modules.oceanengine.service.IHotMaterialService;
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
+import cn.com.ctop.crawler.modules.oceanengine.service.IOceanEngineService;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.http.impl.cookie.BasicClientCookie;
-import org.bytedeco.javacv.FFmpegFrameGrabber;
-import org.bytedeco.javacv.Frame;
-import org.bytedeco.javacv.Java2DFrameConverter;
-import org.openqa.selenium.Cookie;
-import org.openqa.selenium.WebDriver;
-import org.openqa.selenium.chrome.ChromeDriver;
-import org.openqa.selenium.chrome.ChromeOptions;
 import org.quartz.Job;
 import org.quartz.JobExecutionContext;
 import org.quartz.JobExecutionException;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
-
-import javax.imageio.ImageIO;
-import java.awt.*;
-import java.awt.image.BufferedImage;
-import java.io.ByteArrayOutputStream;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.util.List;
-import java.util.*;
 
 @Slf4j
 public class OceanengineJob implements Job {
-    @Value("${jeecg.path.chrome-driver}")
-    private String chromeDriver;
-    @Autowired
-    private IDouyinHotService douyinHotService;
-    @Autowired
-    private IEffectCaseService effectCaseService;
     @Autowired
-    private IHotMaterialService hotMaterialService;
-
-    public static String getVideoUrlByVid(String vid) {
-        String url = "https://aweme.snssdk.com/aweme/v1/playwm/?video_id=" + vid + "&line=0";
-        return getRealVideoUrl(url);
-    }
+    private IOceanEngineService oceanEngineService;
 
-    public static String getRealVideoUrl(String url) {
-        try {
-            URL serverUrl = new URL(url);
-            HttpURLConnection conn = (HttpURLConnection) serverUrl
-                    .openConnection();
-            conn.setRequestMethod("GET");
-            // 必须设置false,否则会自动redirect到Location的地址
-            conn.setInstanceFollowRedirects(false);
-            conn.addRequestProperty("User-Agent",
-                    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.2.8) Firefox/3.6.8");
-            conn.connect();
-            String location = conn.getHeaderField("Location");
-            return location;
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-        return null;
-    }
-    private void hotMaterialHandler(int page,int appCode,String appName){
-        String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_material/list?material_type=3&order_by=convert_show_rate&period_type=3&aggr_app_code="+appCode+"&page="+page+"&limit=20");
-        ObjectMapper mapper = new ObjectMapper();
-        mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-        try {
-            JsonNode node = mapper.readTree(result);
-            int code = node.get("code").asInt();
-            if (code == 0) {
-                if (node.get("data") != null&&node.get("data").get("materials")!=null) {
-                    String dataJson = node.get("data").get("materials").toString();
-                    List<HotMaterial> list = mapper.readValue(dataJson,new TypeReference<List<HotMaterial>>() {});
-                    if(list != null && list.size() > 0){
-                        for (HotMaterial material : list){
-                            material.setAppCode(appCode);
-                            material.setAppName(appName);
-                            material.setVideoUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+material.getVid()+"&line=0"));
-                            if(material.getVideoUrl() == null){
-                                material.setVideoUrl(getVideoUrl(material.getVid()));
-                            }
-//                            material.setVideoUrl(getVideoUrl(material.getVid()));
-                            material.setCoverImage(getVideoFirstImage(material.getVideoUrl()));
-                            if(material.getCoverImage() == null){
-                                Thread.sleep(1000);
-                                material.setCoverImage(getVideoFirstImage(material.getVideoUrl()));
-                            }
-                            hotMaterialService.saveOrUpdate(material);
-                        }
-                        hotMaterialHandler(page+1,appCode,appName);
-                    }
-                }
-            }
-        }catch (Exception e){
-            e.printStackTrace();
-        }
-    }
-
-    public static String getVideoUrl(String vid) throws Exception {
-        String url = "http://i.snssdk.com/video/urls/1/toutiao/mp4/"+vid;
-        String result = HttpUtils2.httpGetRequest(url);
-        ObjectMapper mapper = new ObjectMapper();
-        if(result != null){
-            JsonNode node = mapper.readTree(result);
-            String mainUrl = node.get("data").get("video_list").get("video_1").get("main_url").asText();
-            return Base64Utils.decode(mainUrl);
-        }
-        return null;
-    }
-    private void effectCaseHandler(int page){
-        String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/case/effect_case_list?industry=0&style=0&mode=0&page="+page+"&limit=20");
-        ObjectMapper mapper = new ObjectMapper();
-        mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-        try {
-            JsonNode node = mapper.readTree(result);
-            int code = node.get("code").asInt();
-            if(code == 0) {
-                if (node.get("data") != null) {
-                    String dataJson = node.get("data").get("effect_cases").toString();
-                    List<EffectCase> list = mapper.readValue(dataJson,new TypeReference<List<EffectCase>>() {});
-                    if (list != null && list.size() > 0) {
-                        for (EffectCase effectCase:list){
-                            String resultDetail = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/case/effect_case/"+effectCase.getId());
-                            JsonNode detailNode = mapper.readTree(resultDetail);
-                            int detailCode = detailNode.get("code").asInt();
-                            if(detailCode == 0){
-                                if(detailNode.get("data") != null){
-//                                    String detailJson = node.get("data").get("effect_cases").toString();
-//                                    EffectCase effectCase1 = mapper.readValue(detailJson,EffectCase.class);
-                                    effectCase.setAuthor(detailNode.get("data").get("effect_case").get("author").asText());
-                                    effectCase.setCreateDate(detailNode.get("data").get("effect_case").get("create_date").asText());
-                                    effectCase.setCreativeId(detailNode.get("data").get("effect_case").get("creative_id").asLong());
-                                    effectCase.setCreativeScore(detailNode.get("data").get("effect_case").get("creative_score").asText());
-                                    effectCase.setEffect(detailNode.get("data").get("effect_case").get("effect").toString());
-                                    effectCase.setMethods(detailNode.get("data").get("effect_case").get("methods").toString());
-                                    effectCase.setVideoUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+effectCase.getVid()+"&line=0"));
-                                    if (effectCase.getVideoUrl() == null){
-                                        effectCase.setVideoUrl(getVideoUrl(effectCase.getVid()));
-                                    }
-//                                    effectCase.setVideoUrl(getVideoUrl(effectCase.getVid()));
-                                    effectCase.setCoverImage(getVideoFirstImage(effectCase.getVideoUrl()));
-                                    if (effectCase.getCoverImage() == null){
-                                        Thread.sleep(1000);
-                                        effectCase.setCoverImage(getVideoFirstImage(effectCase.getVideoUrl()));
-                                    }
-                                    effectCase.setCreativeLabel(detailNode.get("data").get("effect_case").get("creative_label").toString());
-                                }
-                            }
-                            effectCaseService.saveOrUpdate(effectCase);
-                        }
-                        effectCaseHandler(page+1);
-                    }
-                }
-            }
-        }catch (Exception e){
-            e.printStackTrace();
-        }
-    }
-
-    private String getVideoFirstImage(String url){
-        try {
-            FFmpegFrameGrabber ff = new FFmpegFrameGrabber(url);
-            ff.start();
-            int lenght = ff.getLengthInFrames();
-            int i = 0;
-            Frame f = null;
-            while (i < lenght) {
-// 过滤前5帧,避免出现全黑的图片,依自己情况而定
-                f = ff.grabFrame();
-                if ((i > 5) && (f.image != null)) {
-                    break;
-                }
-                i++;
-            }
-//            opencv_core.IplImage img = f.image;
-            int owidth = f.imageWidth;
-            int oheight = f.imageHeight;
-            Java2DFrameConverter converter =new Java2DFrameConverter();
-            BufferedImage fecthedImage =converter.getBufferedImage(f);
-// 对截取的帧进行等比例缩放
-            int width = 800;
-            int height = (int) (((double) width / owidth) * oheight);
-            BufferedImage bi = new BufferedImage(width, height, BufferedImage.TYPE_3BYTE_BGR);
-            bi.getGraphics().drawImage(fecthedImage.getScaledInstance(width, height, Image.SCALE_SMOOTH),
-                    0, 0, null);
-            ByteArrayOutputStream baos = new ByteArrayOutputStream();//io流
-            ImageIO.write(bi, "jpg", baos);
-            byte[] bytes = baos.toByteArray();//转换成字节
-            String png_base64 = Base64.getEncoder().encodeToString(bytes).trim();//转换成base64串
-            png_base64 = png_base64.replaceAll("\n", "").replaceAll("\r", "");//删除 \r\n
-            //        ImageIO.write(bufferedImage, "png", new File("D:/qrcode1.png"));
-            baos.close();
-            ff.close();
-            return "data:image/jpg;base64,"+png_base64;
-        }catch (Exception e){
-            e.printStackTrace();
-            return null;
-        }
-
-    }
-    private void douyinHotHandler(int page,int listType){
-        String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_video/user_side/list?page="+page+"&limit=20&list_type="+listType+"&aggr_duration_type=-1");
-        ObjectMapper mapper = new ObjectMapper();
-        try {
-            JsonNode node = mapper.readTree(result);
-            int code = node.get("code").asInt();
-            if(code == 0){
-                if(node.get("data")!= null){
-                    String dataJson = node.get("data").get("items").toString();
-                    List<DouyinHot> list = mapper.readValue(dataJson,new TypeReference<List<DouyinHot>>() {});
-                    if (list != null && list.size() > 0){
-                        for (DouyinHot douyinHot : list){
-                            douyinHot.setUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+douyinHot.getVideoId()+"&line=0"));
-                            if(douyinHot.getUrl() == null){
-                                douyinHot.setUrl(getVideoUrl(douyinHot.getVideoId()));
-                            }
-//                            douyinHot.setUrl(getVideoUrl(douyinHot.getVideoId()));
-                            douyinHot.setCoverImage(getVideoFirstImage(douyinHot.getUrl()));
-                            if(douyinHot.getCoverImage() == null){
-                                Thread.sleep(1000);
-                                douyinHot.setCoverImage(getVideoFirstImage(douyinHot.getUrl()));
-                            }
-                            douyinHotService.saveOrUpdate(douyinHot);
-                        }
-                        douyinHotHandler(page+1,listType);
-                    }else {
-                        if(listType < 4){
-                            douyinHotHandler(1,listType+1);
-                        }
-                    }
-                }
-            }
-        }catch (Exception e){
-            e.printStackTrace();
-        }
-
-    }
     @Override
     public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
         String account="3248395570@qq.com";
         String password = "Ydxq-704127411";
-        login(account,password);
-        douyinHotHandler(1,1);
-        effectCaseHandler(1);
-        hotMaterialHandler(1,1,"西瓜");
-        hotMaterialHandler(1,3,"火山");
-        hotMaterialHandler(1,4,"抖音");
-        hotMaterialHandler(1,8,"头条");
-        hotMaterialHandler(1,9,"穿山甲");
+        oceanEngineService.login(account,password);
+        oceanEngineService.douyinHotHandler(1,1);
+        oceanEngineService.effectCaseHandler(1);
+        oceanEngineService.hotMaterialHandler(1,1,"西瓜");
+        oceanEngineService.hotMaterialHandler(1,3,"火山");
+        oceanEngineService.hotMaterialHandler(1,4,"抖音");
+        oceanEngineService.hotMaterialHandler(1,8,"头条");
+        oceanEngineService.hotMaterialHandler(1,9,"穿山甲");
         log.info("巨量创意抓取完成");
     }
-
-    private void login(String account,String password){
-        String url = "https://cc.oceanengine.com/login";
-        System.getProperties().setProperty("webdriver.chrome.driver", chromeDriver);
-        ChromeOptions chromeOptions = new ChromeOptions();
-        chromeOptions.addArguments("--headless");
-        chromeOptions.addArguments("--no-sandbox");
-        chromeOptions.addArguments("--window-size=1920,1080");
-        chromeOptions.addArguments("--user-agent=" + HttpUtils2.USER_AGENT);
-        chromeOptions.setAcceptInsecureCerts(true);
-        WebDriver webDriver = new ChromeDriver(chromeOptions);
-        //清除所有的缓存
-        webDriver.manage().deleteAllCookies();
-        try {
-            //获取登录页面
-            webDriver.get(url);
-            for (Cookie cookie : webDriver.manage().getCookies()) {
-                BasicClientCookie ck = new BasicClientCookie(cookie.getName(), cookie.getValue());
-                ck.setDomain("oceanengine.com");
-                ck.setExpiryDate(cookie.getExpiry());
-                ck.setPath(cookie.getPath());
-                HttpUtils2.cookieStore.addCookie(ck);
-            }
-            Map<String, String> header = new HashMap<>();
-            header.put("Accept", "*/*");
-            header.put("Connection", "keep-alive");
-            header.put("Host", "sso.toutiao.com");
-            header.put("Content-Type", "application/x-www-form-urlencoded");
-            header.put("Origin", "https://cc.oceanengine.com");
-            header.put("Referer", "https://cc.oceanengine.com/login");
-            Map<String, Object> param = new HashMap<>();
-            param.put("mobile", "");
-            param.put("code", "");
-            param.put("account", account);
-            param.put("password", password);
-            param.put("captcha", "hqde");
-            param.put("is_30_days_no_login", "true");
-            param.put("service", "https://cc.oceanengine.com");
-            String res = HttpUtils2.httpPostParamRequest("https://sso.toutiao.com/account_login/", param, header);
-            ObjectMapper mapper = new ObjectMapper();
-            mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-            JsonNode jsonNode = mapper.readTree(res);
-            //errorcode == 1102 验证码错误,请重新输入验证码
-            //errorcode == 1101 验证码为空
-
-            Integer errorCode = jsonNode.get("error_code").asInt();
-            String orderId = null;
-            while (errorCode != 0) {
-                //用户名或者密码错误
-                if (null != errorCode && errorCode == 1009) {
-                    return;
-                }
-                FateadmUtil fateadmUtil = new FateadmUtil();
-                fateadmUtil.init();
-                if (orderId != null) {
-                    fateadmUtil.justice(orderId);
-                }
-                String captcha = jsonNode.get("captcha").asText();
-                Base64.Decoder decoder = Base64.getDecoder();
-                FateadmHttpUtil.HttpResp resp = fateadmUtil.Predict("30400", decoder.decode(captcha));
-                param.put("captcha", resp.predResl);
-                orderId = resp.reqId;
-                res = HttpUtils2.httpPostParamRequest("https://sso.toutiao.com/account_login/", param, header);
-                jsonNode = mapper.readTree(res);
-                errorCode = jsonNode.get("error_code").asInt();
-            }
-            if (errorCode == 0) {
-                //表示登录成功
-                List<org.apache.http.cookie.Cookie> cookies = HttpUtils2.cookieStore.getCookies();
-                for (org.apache.http.cookie.Cookie ck : cookies) {
-                    Cookie cookie = new Cookie(ck.getName(), ck.getValue(), "." + ck.getDomain(), ck.getPath(), ck.getExpiryDate());
-                    webDriver.manage().addCookie(cookie);
-                }
-                String redirectUrl = jsonNode.get("redirect_url").asText();
-                webDriver.get(redirectUrl);
-                //登录成功 进入推广页面
-                webDriver.get("https://cc.oceanengine.com/inspiration/douyin-rank-list");
-                Thread.sleep(3000);
-                for (Cookie cookie : webDriver.manage().getCookies()) {
-                    BasicClientCookie ck = new BasicClientCookie(cookie.getName(), cookie.getValue());
-                    ck.setDomain("cc.oceanengine.com");
-                    ck.setExpiryDate(cookie.getExpiry());
-                    ck.setPath(cookie.getPath());
-                    HttpUtils2.cookieStore.addCookie(ck);
-                }
-
-            }
-        } catch (Exception e) {
-            e.printStackTrace();
-        } finally {
-            //获得cookie
-            Set<Cookie> coo = webDriver.manage().getCookies();
-            //清除所有的缓存
-            webDriver.manage().deleteAllCookies();
-            webDriver.quit();
-        }
-    }
 }

+ 2 - 5
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/PangolinLoginJob.java

@@ -2,8 +2,8 @@ package org.jeecg.modules.ctop.job;
 
 import cn.com.ctop.common.module.entity.BindAccountLogin;
 import cn.com.ctop.common.module.service.IBindAccountLoginService;
+import cn.com.ctop.common.module.utils.CtopAdConstant;
 import cn.com.ctop.crawler.modules.pangolin.service.PangolinLoginService;
-import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import org.quartz.Job;
 import org.quartz.JobExecutionContext;
 import org.quartz.JobExecutionException;
@@ -18,10 +18,7 @@ public class PangolinLoginJob implements Job {
     private PangolinLoginService pangolinLoginService;
     @Override
     public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
-        QueryWrapper<BindAccountLogin> queryWrapper = new QueryWrapper<>();
-        queryWrapper.eq("login_type","pangolin");
-        queryWrapper.eq("status",1);
-        List<BindAccountLogin> list = bindAccountLoginService.list(queryWrapper);
+        List<BindAccountLogin> list = bindAccountLoginService.getListByParams(CtopAdConstant.PLATFORM_TYPE_PANGOLIN_PY,1);
         if(list!=null&&!list.isEmpty()){
             for (BindAccountLogin login:list) {
                 if(null==login.getCookie()||"".equals(login.getCookie().trim())){

+ 144 - 0
jeecg-boot-module-system/src/main/java/org/jeecg/modules/opencv/utils/ExtractionVideoKeyFrames.java

@@ -0,0 +1,144 @@
+package org.jeecg.modules.opencv.utils;
+
+import org.opencv.core.*;
+import org.opencv.imgcodecs.Imgcodecs;
+import org.opencv.imgproc.Imgproc;
+import org.opencv.videoio.VideoCapture;
+import org.opencv.videoio.Videoio;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
+public class ExtractionVideoKeyFrames {
+    Mat computeColorHist3D(Mat im, Size size)
+    {
+        Mat dst = new Mat(), hist = new Mat();
+        Imgproc.resize(im, dst, size, 0, 0, Imgproc.INTER_NEAREST);
+        Mat hsv_src = new Mat();
+        Imgproc.cvtColor(im, hsv_src, Imgproc.COLOR_BGR2HSV);
+        List<Mat> images=new ArrayList<Mat>(3);
+        images.add(hsv_src);
+
+        MatOfInt channels= new MatOfInt(0);
+        MatOfInt hSize = new MatOfInt(256);
+        MatOfFloat ranges= new MatOfFloat(0,255);
+        //获取图像直方图
+        Imgproc.calcHist(images, channels, new Mat(), hist, hSize, ranges);
+        return hist;
+    }
+
+    Mat computeColorHist(Mat im, Size size)
+    {
+        Mat dst = new Mat(), hist = new Mat();
+        Imgproc.resize(im, dst, size, 0, 0, Imgproc.INTER_NEAREST);
+        Mat gray = new Mat(im.size(),CvType.CV_8UC1);
+        Imgproc.cvtColor(im,gray,Imgproc.COLOR_RGB2GRAY);
+
+        List<Mat> images = new ArrayList<Mat>();
+        images.add(gray);
+
+        MatOfInt channels= new MatOfInt(0);
+        MatOfInt hSize = new MatOfInt(128);
+        MatOfFloat ranges= new MatOfFloat(0,128);
+        Imgproc.calcHist(images, channels, new Mat(), hist, hSize, ranges);
+        return hist;
+    }
+
+    double computeASE(Mat a, Mat b) {
+        return  1 - Imgproc.compareHist(a, b, 0);
+    }
+
+    int ExtractKeyframes(String videoPath, double rate, String outImgPath){
+        VideoCapture cap = new VideoCapture();
+        System.out.println(cap.isOpened());
+        if (!cap.open(videoPath)) {
+            System.out.println("Open video file error!");
+            return -1;
+        }
+
+        File file = new File(outImgPath);
+        if(!file.exists()){
+            file.mkdirs();
+        }
+        //  Videoio.CAP_PROP_FRAME_COUNT
+        int width = (int)cap.get(Videoio.CAP_PROP_FRAME_WIDTH);
+        int height = (int)cap.get(Videoio.CAP_PROP_FRAME_HEIGHT);
+        float fps = (float)cap.get(Videoio.CAP_PROP_FPS);
+        int nframes = (int)cap.get(Videoio.CAP_PROP_FRAME_COUNT);
+        System.out.println("W:"+width+";H:"+height+";FPS:"+fps+";N:"+nframes);
+
+        int nscale = 1;
+        if (width%4==0 && height%4==0){
+            nscale = 4;
+        }else if (width%2==0 && height%2==0){
+            nscale = 2;
+        }
+        Size size = new Size(width/nscale, height/nscale);
+
+        Mat frame = new Mat();
+        Mat lastFrame = new Mat();
+        Mat histA = new Mat();
+        Mat histB = new Mat();
+        long nCount = 0;
+        int frameID = 1;
+        int lastFrameID = 1;
+        double tic = (double)Core.getTickCount();
+        while (cap.read(frame)) {
+            if (frame.empty()){
+                break;
+            }
+
+            if (lastFrameID==1) {
+                frameID = (int)cap.get(Videoio.CAP_PROP_POS_FRAMES);
+                lastFrameID = frameID;
+                frame.copyTo(lastFrame);
+                continue;
+            }
+            if (++nCount % 15 != 0){
+                continue;	// Processing one frame in each second
+            }
+
+            frameID = (int)cap.get(Videoio.CAP_PROP_POS_FRAMES);
+            histA = computeColorHist3D(frame, size);
+
+            double aseV = rate;
+            if (histB.empty()){
+                System.out.println(aseV);
+            }
+            else{
+                aseV = computeASE(histA, histB);
+                System.out.println(aseV);
+            }
+
+            if (aseV >= rate) {
+                String imagePath = outImgPath+"/"+lastFrameID+".jpg";
+                if (!Imgcodecs.imwrite(imagePath, lastFrame)){
+                    return -2;
+                }
+                lastFrameID = frameID;
+                frame.copyTo(lastFrame);
+            }
+
+            histA.copyTo(histB);
+        }
+
+        if (!lastFrame.empty()) {
+            String imagePath=outImgPath+"/"+ lastFrameID+".jpg";
+            if (!Imgcodecs.imwrite(imagePath, lastFrame)){
+                return -2;
+            }
+        }
+        System.out.println("NFRAMES(in reality)="+nCount);
+
+        double toc = (double)Core.getTickCount();
+        System.out.println("ELAPSED TIME="+(toc-tic)/Core.getTickFrequency());
+        return 0;
+    }
+
+    public static void main(String[] args) {
+        System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
+        ExtractionVideoKeyFrames evf = new ExtractionVideoKeyFrames();
+        evf.ExtractKeyframes("D:\\data\\fill.mp4",0.5,"D:\\data\\");
+    }
+}

+ 1 - 1
jeecg-boot-module-system/src/main/resources/application-test.yml

@@ -152,7 +152,7 @@ jeecg:
     webapp: /data/webapp
     video-upload: /data/upload/video/
     image-upload: /data/upload/image/
-    chrome-driver: /usr/bin/chromedriver
+    chrome-driver: D://chromedriver.exe
     csv-upload: /data/upload/csv/
     report-history: /data/report/history/
     report-daily: /data/report/daily/

+ 70 - 78
jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java

@@ -1,25 +1,20 @@
 package org.jeecg;
 
 import cn.com.ctop.common.module.entity.BindAccountLogin;
-import cn.com.ctop.common.module.entity.CtopOauthToken;
 import cn.com.ctop.common.module.service.IBindAccountLoginService;
-import cn.com.ctop.common.module.service.ICtopOauthTokenService;
 import cn.com.ctop.common.module.utils.CtopAdConstant;
-import cn.com.ctop.crawler.modules.pangolin.entity.PangolinApp;
-import cn.com.ctop.crawler.modules.pangolin.service.PangolinAppService;
-import cn.com.ctop.crawler.modules.pangolin.service.PangolinCrawlerService;
-import cn.com.ctop.crawler.modules.pangolin.service.PangolinLoginService;
-import cn.com.ctop.kuaishou.modules.batch.service.impl.KuaishouInterfaceServiceImpl;
-import cn.com.ctop.toutiao.modules.report.service.IBytedanceFundDailyService;
-import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import cn.com.ctop.crawler.modules.oceanengine.entity.HotMaterial;
+import cn.com.ctop.crawler.modules.oceanengine.service.IHotMaterialService;
+import cn.com.ctop.crawler.modules.oceanengine.service.IOceanEngineService;
+import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouWebInterfaceService;
 import lombok.extern.slf4j.Slf4j;
-import org.jeecg.common.util.DateUtils;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.test.context.junit4.SpringRunner;
 
+import java.util.HashMap;
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
@@ -30,97 +25,94 @@ import java.util.concurrent.Executors;
 @Slf4j
 public class SampleTest {
     @Autowired
-    private IBindAccountLoginService bindAccountLoginService;
-    @Autowired
-    private PangolinLoginService pangolinLoginService;
-    @Autowired
-    private PangolinCrawlerService pangolinCrawlerService;
-    @Autowired
-    private PangolinAppService pangolinAppService;
+    private IOceanEngineService oceanEngineService;
     @Autowired
-    private KuaishouInterfaceServiceImpl interfaceService;
-
-
+    private IHotMaterialService hotMaterialService;
     @Test
-    public void suzhao() {
-        interfaceService.getSuZaoList("33e4f5b5fa460386e8d2f4d4113c1f27",161468L,1);
+    public void initImageUrl(){
+        List<HotMaterial> hotMaterials =  hotMaterialService.getId();
+        if(null!=hotMaterials&&!hotMaterials.isEmpty()){
+            hotMaterials.forEach(douyinHot -> {
+                HotMaterial getHot = hotMaterialService.getById(douyinHot.getId());
+//                String image = getHot.getCoverImage().replace("data:image/jpg;base64,", "");
+                String destPath = "D:\\data\\effectCase\\";
+                String fileName = getHot.getId()+".jpg";
+//                EffectCase.base64ToFile(destPath,image,fileName);
+            });
+        }
     }
-
-
     @Test
-    public void testPangolinLogin() {
-        QueryWrapper<BindAccountLogin> queryWrapper = new QueryWrapper<>();
-        queryWrapper.eq("login_type", "pangolin");
-        queryWrapper.eq("status", 1);
-        List<BindAccountLogin> list = bindAccountLoginService.list(queryWrapper);
-        if (list != null && !list.isEmpty()) {
-            for (BindAccountLogin login : list) {
-                if (null == login.getCookie() || "".equals(login.getCookie().trim())) {
-                    pangolinLoginService.pangolinLogin(login.getAccountName(), login.getPassword());
-                }
-            }
-        }
+    public void testOceanEngineJob(){
+        String account="3248395570@qq.com";
+        String password = "Ydxq-704127411";
+        oceanEngineService.login(account,password);
+        oceanEngineService.douyinHotHandler(1,1);
+        oceanEngineService.effectCaseHandler(1);
+        oceanEngineService.hotMaterialHandler(1,4,"抖音");
+        oceanEngineService.hotMaterialHandler(1,8,"头条");
+        oceanEngineService.hotMaterialHandler(1,1,"西瓜");
+        oceanEngineService.hotMaterialHandler(1,3,"火山");
+        oceanEngineService.hotMaterialHandler(1,9,"穿山甲");
+        log.info("巨量创意抓取完成");
     }
 
+    @Autowired
+    private IBindAccountLoginService bindAccountLoginService;
+    @Autowired
+    private IKuaishouWebInterfaceService kuaishouWebInterfaceService;
     @Test
-    public void testPangolinData() {
-        QueryWrapper<BindAccountLogin> queryWrapper = new QueryWrapper<>();
-        queryWrapper.eq("login_type", "pangolin");
-        queryWrapper.eq("status", 1);
-        queryWrapper.isNotNull("cookie");
-        List<BindAccountLogin> list = bindAccountLoginService.list(queryWrapper);
-        if (list != null && !list.isEmpty()) {
-            for (BindAccountLogin bindAccountLogin : list) {
-                pangolinCrawlerService.getChannelList(bindAccountLogin);
-                pangolinCrawlerService.getActivationList(bindAccountLogin, DateUtils.getNowDate("yyyy-MM-dd"), 1);
-                pangolinCrawlerService.getCheckList(bindAccountLogin);
-                QueryWrapper<PangolinApp> queryWrapper1 = new QueryWrapper<>();
-                queryWrapper1.eq("account_name", bindAccountLogin.getAccountName());
-                List<PangolinApp> appList = pangolinAppService.list(queryWrapper1);
-                if (appList != null && !appList.isEmpty()) {
-                    appList.forEach(pangolinApp -> pangolinCrawlerService.getRealTimeList(bindAccountLogin, pangolinApp.getId()));
+    public void kuaishouLogin(){
+        Long start =System.currentTimeMillis();
+        List<BindAccountLogin> list = bindAccountLoginService.getListByParams(CtopAdConstant.PLATFORM_TYPE_KUAISHOU_PY,1);
+        if(list!=null&&!list.isEmpty()){
+            for (BindAccountLogin login:list) {
+                if(null==login.getCookie()||"".equals(login.getCookie().trim())){
+                    kuaishouWebInterfaceService.getkuaishouWebLoginCookie(login);
                 }
             }
         }
+        Long end = System.currentTimeMillis();
+        log.info("总用时:{}毫秒",end-start);
     }
-
-    @Autowired
-    private ICtopOauthTokenService tokenService;
-    @Autowired
-    private IBytedanceFundDailyService bytedanceFundDailyService;
     static ExecutorService executorService = null;
-
-
+    static CountDownLatch countDownLatch = null;
     @Test
-    public void testflowFund() {
-        List<CtopOauthToken> tokens = tokenService.getTokenListByType(CtopAdConstant.PLATFORM_TYPE_BYTEDANCE);
-        if (null == tokens || tokens.size() <= 0) {
-            log.error("头条账户流水数据异常:未获取到可用的token");
-            return;
-        }
-        CountDownLatch countDownLatch = new CountDownLatch(tokens.size());
-        executorService = Executors.newFixedThreadPool(5);
-        tokens.forEach(token -> {
-            executorService.submit(() -> {
-                try {
-                    for (int i = 0; i < 60; i++) {
-                        String findDate = DateUtils.addDay(DateUtils.formatDate(), -i);
-                        bytedanceFundDailyService.loadFundDataByPage(token, findDate, findDate, 1);
+    public void kuaishouCommentDelete(){
+        Long start =System.currentTimeMillis();
+        List<BindAccountLogin> list = bindAccountLoginService.getListByParams(CtopAdConstant.PLATFORM_TYPE_KUAISHOU_PY,1);
+        if(list!=null&&!list.isEmpty()){
+            executorService = Executors.newFixedThreadPool(10);
+            countDownLatch = new CountDownLatch(list.size());
+            list.forEach(login -> {
+                executorService.submit(()->{
+                    try {
+                        if(null!=login.getCookie()&&!"".equals(login.getCookie().trim())){
+                            kuaishouWebInterfaceService.deleteAllComment(new HashMap<>(),login);
+                        }
+                    }catch (Exception e){
+
+                    }finally {
+                        countDownLatch.countDown();
                     }
-                } catch (Exception e) {
-                } finally {
-                    countDownLatch.countDown();
-                }
+                });
             });
-        });
+        }
         try {
             countDownLatch.await();
         } catch (InterruptedException e) {
             e.printStackTrace();
         }
+        Long end = System.currentTimeMillis();
+        log.info("总用时:{}毫秒",end-start);
     }
+    @Test
+    public void testBase64(){
 
+    }
 }
+//刷新token:885936毫秒
+
+
 
 
 

+ 8 - 0
module-common/src/main/java/cn/com/ctop/common/module/entity/BindAccountLogin.java

@@ -102,5 +102,13 @@ public class BindAccountLogin {
 
     private String cookie;
 
+    public BindAccountLogin(Long id,String accountName) {
+        this.id = id;
+        this.accountName = accountName;
+    }
+
+    public BindAccountLogin() {
+    }
+
     private Long projectId;
 }

+ 13 - 0
module-common/src/main/java/cn/com/ctop/common/module/service/IBindAccountLoginService.java

@@ -2,6 +2,11 @@ package cn.com.ctop.common.module.service;
 
 import cn.com.ctop.common.module.entity.BindAccountLogin;
 import com.baomidou.mybatisplus.extension.service.IService;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import org.apache.http.client.CookieStore;
+
+import java.util.List;
+import java.util.Map;
 
 /**
  * @Description: 账号绑定-登陆
@@ -10,6 +15,8 @@ import com.baomidou.mybatisplus.extension.service.IService;
  * @Version: V1.0
  */
 public interface IBindAccountLoginService extends IService<BindAccountLogin> {
+    boolean checkLogin(BindAccountLogin bindAccountLogin, String response);
+
     /**
      * 查询
      *
@@ -18,5 +25,11 @@ public interface IBindAccountLoginService extends IService<BindAccountLogin> {
      */
     BindAccountLogin selectByAccountId(String accountId);
 
+    Map<String, Object> getListByUserId(String userId);
+
+    List<BindAccountLogin> getListByParams(String loginType,Integer status);
+
+    CookieStore setCookie(BindAccountLogin bindAccountLogin,String domain) throws JsonProcessingException;
+
     BindAccountLogin getByParams(String accountName, String password, String advertiserId);
 }

+ 78 - 1
module-common/src/main/java/cn/com/ctop/common/module/service/impl/BindAccountLoginServiceImpl.java

@@ -3,12 +3,26 @@ package cn.com.ctop.common.module.service.impl;
 import cn.com.ctop.common.module.entity.BindAccountLogin;
 import cn.com.ctop.common.module.mapper.BindAccountLoginMapper;
 import cn.com.ctop.common.module.service.IBindAccountLoginService;
+import cn.com.ctop.common.module.utils.CtopAdConstant;
+import cn.com.ctop.common.module.utils.ResultMapUtils;
+import cn.com.ctop.common.module.utils.StatusCode;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.http.client.CookieStore;
+import org.apache.http.impl.client.BasicCookieStore;
+import org.apache.http.impl.cookie.BasicClientCookie;
 import org.springframework.context.annotation.Primary;
 import org.springframework.stereotype.Service;
 
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 /**
  * @Description: 账号绑定-登陆
  * @Author: jeecg-boot
@@ -17,9 +31,17 @@ import org.springframework.stereotype.Service;
  */
 @Slf4j
 @Service
-@Primary
 public class BindAccountLoginServiceImpl extends ServiceImpl<BindAccountLoginMapper, BindAccountLogin> implements IBindAccountLoginService {
     @Override
+    public boolean checkLogin(BindAccountLogin bindAccountLogin, String response) {
+        if (response.contains("<html>")) {
+            bindAccountLogin.setCookie("");
+            this.updateById(bindAccountLogin);
+            return false;
+        }
+        return true;
+    }
+    @Override
     public BindAccountLogin selectByAccountId(String accountId) {
         QueryWrapper<BindAccountLogin> queryWrapper = new QueryWrapper<>();
         queryWrapper.eq("account_id", accountId).orderByDesc("id").last("limit 1");
@@ -41,4 +63,59 @@ public class BindAccountLoginServiceImpl extends ServiceImpl<BindAccountLoginMap
         queryWrapper.orderByDesc("create_time").last("limit 1");
         return this.getOne(queryWrapper);
     }
+
+    @Override
+    public Map<String, Object> getListByUserId(String userId) {
+        Map<String,Object>result = new HashMap<>();
+        QueryWrapper<BindAccountLogin> queryWrapper = new QueryWrapper<>();
+        queryWrapper.eq("login_type", CtopAdConstant.PLATFORM_TYPE_PANGOLIN_PY);
+        List<BindAccountLogin> loginList = new ArrayList<>();
+        if(userId == null||"".equals(userId.trim())){
+            loginList.add(new BindAccountLogin(0L,"全部"));
+        }else{
+            queryWrapper.eq("user_id",userId);
+        }
+        loginList.addAll(this.list(queryWrapper));
+        result.put("data",loginList);
+        ResultMapUtils.setResultMap(result, StatusCode.COMMON_SUCCESS);
+        return result;
+    }
+
+    @Override
+    public List<BindAccountLogin> getListByParams(String loginType, Integer status) {
+        QueryWrapper<BindAccountLogin> queryWrapper = new QueryWrapper<>();
+        if(null!=loginType&&!loginType.equals("")){
+            queryWrapper.eq("login_type",loginType);
+        }
+        if(status!=null){
+            queryWrapper.eq("status",status);
+        }
+        return this.list(queryWrapper);
+    }
+
+    @Override
+    public CookieStore setCookie(BindAccountLogin bindAccountLogin,String domain) throws JsonProcessingException {
+        ObjectMapper mapper = new ObjectMapper();
+        List<String> cookieList = mapper.readValue(bindAccountLogin.getCookie(), new TypeReference<List<String>>() {});
+        if(cookieList == null||cookieList.isEmpty()){
+            return null;
+        }
+        CookieStore cookieStore = new BasicCookieStore();
+        for (String cookieStr : cookieList) {
+            String[] cookieArray = cookieStr.split(";");
+            Map<String, String> cookieMap = new HashMap<>();
+            for (String cookie : cookieArray) {
+                String[] kv = cookie.split("=");
+                if (kv.length > 1) {
+                    cookieMap.put(kv[0].trim(), kv[1].trim());
+                }
+            }
+            BasicClientCookie clientCookie = new BasicClientCookie(cookieArray[0].split("=")[0],
+                    cookieArray[0].split("=")[1]);
+            clientCookie.setDomain(domain);
+            clientCookie.setPath(cookieMap.get("path"));
+            cookieStore.addCookie(clientCookie);
+        }
+        return cookieStore;
+    }
 }

+ 51 - 1
module-common/src/main/java/cn/com/ctop/common/module/utils/HttpUtils2.java

@@ -45,7 +45,7 @@ public class HttpUtils2 {
     /**
      * setProxy(new HttpHost("106.125.239.179", 4245))
      */
-    public static Map<String, Map<String, String>> fontsMap = new HashMap<String, Map<String, String>>();
+    public static Map<String, Map<String, String>> fontsMap = new HashMap<>();
     public static String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36";
 
     public static CloseableHttpClient createSSLClientDefault() {
@@ -66,6 +66,24 @@ public class HttpUtils2 {
         return HttpClients.createDefault();
     }
 
+    public static CloseableHttpClient createSSLClientDefault(CookieStore setCookieStore) {
+        try {
+            SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {
+                //信任所有证书
+                @Override
+                public boolean isTrusted(X509Certificate[] chain, String authType) throws CertificateException {
+                    return true;
+                }
+            }).build();
+            RequestConfig globalConfig = RequestConfig.custom().setConnectTimeout(30000).setSocketTimeout(30000).setCookieSpec(CookieSpecs.STANDARD).build();
+            SSLConnectionSocketFactory sslFactory = new SSLConnectionSocketFactory(sslContext);
+            return HttpClients.custom().setDefaultCookieStore(setCookieStore).setDefaultRequestConfig(globalConfig).setConnectionReuseStrategy((response, context) -> false).setSSLSocketFactory(sslFactory).build();
+        } catch (Exception e) {
+            log.error("处理Https证书异常", e);
+        }
+        return HttpClients.createDefault();
+    }
+
     public static String httpPostParamRequest(String url, Map<String, Object> param, Map<String, String> headers) {
         HttpClient httpClient = createSSLClientDefault();
         String strReturn = "";
@@ -569,4 +587,36 @@ public class HttpUtils2 {
     }
 
 
+    public static String httpPostRequest(String url, Map<String, Object> param, Map<String, String> headers, CookieStore cookieStore) {
+        HttpClient httpClient = createSSLClientDefault(cookieStore);
+        String strReturn = "";
+        try {
+            HttpPost httppost = new HttpPost(url);
+            httppost.setHeader("User-Agent", USER_AGENT);
+            httppost.addHeader("Content-Type", "application/json");
+            if (headers != null) {
+                for (String key : headers.keySet()) {
+                    httppost.setHeader(key, headers.get(key));
+                }
+            }
+
+            httppost.setEntity(new StringEntity(new Gson().toJson(param), Charset.forName("UTF-8")));
+            HttpEntity respentity;
+
+            HttpResponse response = httpClient.execute(httppost);
+            int statusCode = response.getStatusLine().getStatusCode();
+            if (statusCode == HttpStatus.SC_MOVED_PERMANENTLY || statusCode == HttpStatus.SC_MOVED_TEMPORARILY) {
+                String newUrl = response.getFirstHeader("Location").getValue();
+                return httpPostRequest(newUrl, param, headers);
+            } else if (statusCode == HttpStatus.SC_OK) {
+                respentity = response.getEntity();
+                strReturn = EntityUtils.toString(respentity);
+                return strReturn;
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+            log.error(e.getMessage());
+        }
+        return strReturn;
+    }
 }

+ 54 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/core/util/FfmpegUtils.java

@@ -0,0 +1,54 @@
+package cn.com.ctop.crawler.modules.core.util;
+
+import org.bytedeco.javacv.FFmpegFrameGrabber;
+import org.bytedeco.javacv.Frame;
+import org.bytedeco.javacv.Java2DFrameConverter;
+
+import javax.imageio.ImageIO;
+import java.awt.*;
+import java.awt.image.BufferedImage;
+import java.io.ByteArrayOutputStream;
+import java.util.Base64;
+
+public class FfmpegUtils {
+    public static String getVideoFirstImage(String url){
+        try {
+            FFmpegFrameGrabber ff = new FFmpegFrameGrabber(url);
+            ff.start();
+            int lenght = ff.getLengthInFrames();
+            int i = 0;
+            Frame f = null;
+            while (i < lenght) {
+// 过滤前5帧,避免出现全黑的图片,依自己情况而定
+                f = ff.grabFrame();
+                if ((i > 5) && (f.image != null)) {
+                    break;
+                }
+                i++;
+            }
+//            opencv_core.IplImage img = f.image;
+            int owidth = f.imageWidth;
+            int oheight = f.imageHeight;
+            Java2DFrameConverter converter =new Java2DFrameConverter();
+            BufferedImage fecthedImage =converter.getBufferedImage(f);
+// 对截取的帧进行等比例缩放
+            int width = 800;
+            int height = (int) (((double) width / owidth) * oheight);
+            BufferedImage bi = new BufferedImage(width, height, BufferedImage.TYPE_3BYTE_BGR);
+            bi.getGraphics().drawImage(fecthedImage.getScaledInstance(width, height, Image.SCALE_SMOOTH),
+                    0, 0, null);
+            ByteArrayOutputStream baos = new ByteArrayOutputStream();//io流
+            ImageIO.write(bi, "jpg", baos);
+            byte[] bytes = baos.toByteArray();//转换成字节
+            String png_base64 = Base64.getEncoder().encodeToString(bytes).trim();//转换成base64串
+            png_base64 = png_base64.replaceAll("\n", "").replaceAll("\r", "");//删除 \r\n
+            //        ImageIO.write(bufferedImage, "png", new File("D:/qrcode1.png"));
+            baos.close();
+            ff.close();
+            return "data:image/jpg;base64,"+png_base64;
+        }catch (Exception e){
+            e.printStackTrace();
+            return null;
+        }
+    }
+}

+ 1 - 1
module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/entity/DouyinHot.java

@@ -20,7 +20,7 @@ public class DouyinHot {
     private Long userId;
     @JsonProperty("core_user_name")
     private String userName;
-    private String coverImage;
+    private String coverUrl;
     private String title;
     @JsonProperty("head_image_uri")
     private String headImageUri;

+ 196 - 3
module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/entity/EffectCase.java

@@ -6,16 +6,22 @@ import com.baomidou.mybatisplus.annotation.TableId;
 import com.baomidou.mybatisplus.annotation.TableName;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import lombok.Data;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.util.Base64;
 import java.util.Date;
 
-@Data
-@TableName("ctop_oceanengine_effect_case")
 /**
  * 效果案例
  */
+@Data
+@TableName("ctop_oceanengine_effect_case")
 public class EffectCase {
     @TableId(value = "id", type = IdType.INPUT)
     private String id;
+    private String coverUrl;
     private String vid;
     @JsonProperty("creative_label_1")
     @TableField(value = "creative_label_1")
@@ -34,7 +40,7 @@ public class EffectCase {
     private String effect;
     @JsonProperty("conversion_type")
     private String conversionType;
-    private String coverImage;
+//    private String coverImage;
     private String creativeLabel;
     private String tag;
     @JsonProperty("creative_title")
@@ -47,4 +53,191 @@ public class EffectCase {
     private String industryTag;
     private Date createTime;
     private Date updateTime;
+
+    public String getId() {
+        return id;
+    }
+
+    public void setId(String id) {
+        this.id = id;
+    }
+
+    public String getVid() {
+        return vid;
+    }
+
+    public void setVid(String vid) {
+        this.vid = vid;
+    }
+
+    public String getCreativeLabel1() {
+        return creativeLabel1;
+    }
+
+    public void setCreativeLabel1(String creativeLabel1) {
+        this.creativeLabel1 = creativeLabel1;
+    }
+
+    public Integer getCreativeLabel2() {
+        return creativeLabel2;
+    }
+
+    public void setCreativeLabel2(Integer creativeLabel2) {
+        this.creativeLabel2 = creativeLabel2;
+    }
+
+    public String getCreateDate() {
+        return createDate;
+    }
+
+    public void setCreateDate(String createDate) {
+        this.createDate = createDate;
+    }
+
+    public Long getCreativeId() {
+        return creativeId;
+    }
+
+    public void setCreativeId(Long creativeId) {
+        this.creativeId = creativeId;
+    }
+
+    public String getMethods() {
+        return methods;
+    }
+
+    public void setMethods(String methods) {
+        this.methods = methods;
+    }
+
+    public String getVideoUrl() {
+        return videoUrl;
+    }
+
+    public void setVideoUrl(String videoUrl) {
+        this.videoUrl = videoUrl;
+    }
+
+    public String getAuthor() {
+        return author;
+    }
+
+    public void setAuthor(String author) {
+        this.author = author;
+    }
+
+    public String getEffect() {
+        return effect;
+    }
+
+    public void setEffect(String effect) {
+        this.effect = effect;
+    }
+
+    public String getConversionType() {
+        return conversionType;
+    }
+
+    public void setConversionType(String conversionType) {
+        this.conversionType = conversionType;
+    }
+    public String getCreativeLabel() {
+        return creativeLabel;
+    }
+
+    public void setCreativeLabel(String creativeLabel) {
+        this.creativeLabel = creativeLabel;
+    }
+
+    public String getTag() {
+        return tag;
+    }
+
+    public void setTag(String tag) {
+        this.tag = tag;
+    }
+
+    public String getCreativeTitle() {
+        return creativeTitle;
+    }
+
+    public void setCreativeTitle(String creativeTitle) {
+        this.creativeTitle = creativeTitle;
+    }
+
+    public String getCreativeInfo() {
+        return creativeInfo;
+    }
+
+    public void setCreativeInfo(String creativeInfo) {
+        this.creativeInfo = creativeInfo;
+    }
+
+    public String getCreativeScore() {
+        return creativeScore;
+    }
+
+    public void setCreativeScore(String creativeScore) {
+        this.creativeScore = creativeScore;
+    }
+
+    public String getIndustryTag() {
+        return industryTag;
+    }
+
+    public void setIndustryTag(String industryTag) {
+        this.industryTag = industryTag;
+    }
+
+    public Date getCreateTime() {
+        return createTime;
+    }
+
+    public void setCreateTime(Date createTime) {
+        this.createTime = createTime;
+    }
+
+    public Date getUpdateTime() {
+        return updateTime;
+    }
+
+    public void setUpdateTime(Date updateTime) {
+        this.updateTime = updateTime;
+    }
+
+    public static void base64ToFile(String destPath,String base64, String fileName) {
+        File file = null;
+        //创建文件目录
+        String filePath=destPath;
+        File  dir=new File(filePath);
+        if (!dir.exists() && !dir.isDirectory()) {
+            dir.mkdirs();
+        }
+        BufferedOutputStream bos = null;
+        java.io.FileOutputStream fos = null;
+        try {
+            byte[] bytes = Base64.getMimeDecoder().decode(base64);
+            file=new File(filePath+"/"+fileName);
+            fos = new java.io.FileOutputStream(file);
+            bos = new BufferedOutputStream(fos);
+            bos.write(bytes);
+        } catch (Exception e) {
+            e.printStackTrace();
+        } finally {
+            if (bos != null) {
+                try {
+                    bos.close();
+                } catch (IOException e) {
+                    e.printStackTrace();
+                }
+            }
+            if (fos != null) {
+                try {
+                    fos.close();
+                } catch (IOException e) {
+                    e.printStackTrace();
+                }
+            }
+        }
+    }
 }

+ 1 - 1
module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/entity/HotMaterial.java

@@ -20,7 +20,7 @@ public class HotMaterial {
     private String videoUrl;
     private String title;
     private Integer appCode;
-    private String coverImage;
+    private String coverUrl;
     private String appName;
     private Date createTime;
     private Date updateTime;

+ 3 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/service/IDouyinHotService.java

@@ -10,4 +10,7 @@ public interface IDouyinHotService extends IService<DouyinHot> {
     public void replaceBatch(List<DouyinHot> list);
 
     IPage<DouyinHot> queryHotFeeds(String userName, String start, String end, String orderByColumn, String orderByType, Integer pageNo, Integer pageSize);
+
+    List<DouyinHot> getId();
+
 }

+ 5 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/service/IHotMaterialService.java

@@ -4,6 +4,11 @@ import cn.com.ctop.crawler.modules.oceanengine.entity.HotMaterial;
 import com.baomidou.mybatisplus.core.metadata.IPage;
 import com.baomidou.mybatisplus.extension.service.IService;
 
+import java.util.List;
+
 public interface IHotMaterialService extends IService<HotMaterial> {
     IPage<HotMaterial> queryHotFeeds(String userName, String start, String end, String orderByColumn, String orderByType, Integer pageNo, Integer pageSize);
+
+    List<HotMaterial> getId();
+
 }

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/service/IOceanEngineService.java

@@ -1,4 +1,11 @@
 package cn.com.ctop.crawler.modules.oceanengine.service;
 
 public interface IOceanEngineService {
+    void hotMaterialHandler(int page, int appCode, String appName);
+
+    void effectCaseHandler(int page);
+
+    void douyinHotHandler(int page, int listType);
+
+    void login(String account, String password);
 }

+ 8 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/service/impl/DouyinHotServiceImpl.java

@@ -40,4 +40,12 @@ public class DouyinHotServiceImpl extends ServiceImpl<DouyinHotMapper, DouyinHot
         Page<DouyinHot> page = new Page<>(pageNo, pageSize);
         return this.page(page, queryWrapper);
     }
+
+    @Override
+    public List<DouyinHot> getId() {
+        QueryWrapper<DouyinHot> queryWrapper = new QueryWrapper<>();
+        queryWrapper.select("id").orderByDesc("create_time");
+        return this.list(queryWrapper);
+
+    }
 }

+ 10 - 1
module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/service/impl/HotMaterialServiceImpl.java

@@ -10,6 +10,8 @@ import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import org.springframework.context.annotation.Primary;
 import org.springframework.stereotype.Service;
 
+import java.util.List;
+
 @Service
 @Primary
 
@@ -18,7 +20,7 @@ public class HotMaterialServiceImpl extends ServiceImpl<HotMaterialMapper, HotMa
     public IPage<HotMaterial> queryHotFeeds(String userName, String start, String end, String orderByColumn, String orderByType, Integer pageNo, Integer pageSize) {
         QueryWrapper<HotMaterial> queryWrapper = new QueryWrapper<>();
         if (null != userName && !"".equals(userName)) {
-            queryWrapper.like("author", "%" + userName + "%");
+            queryWrapper.like("author", userName);
         }
         if (null != start && !"".equals(start) && null != end && !"".equals(end)) {
             queryWrapper.between("create_time", start, end);
@@ -31,4 +33,11 @@ public class HotMaterialServiceImpl extends ServiceImpl<HotMaterialMapper, HotMa
         Page<HotMaterial> page = new Page<>(pageNo, pageSize);
         return this.page(page, queryWrapper);
     }
+
+    @Override
+    public List<HotMaterial> getId() {
+        QueryWrapper<HotMaterial> queryWrapper = new QueryWrapper<>();
+        queryWrapper.select("id").orderByDesc("create_time");
+        return this.list(queryWrapper);
+    }
 }

+ 299 - 340
module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/service/impl/OceanEngineServiceImpl.java

@@ -1,347 +1,306 @@
-//package cn.com.ctop.crawler.modules.oceanengine.service.impl;
-//
-//import cn.com.ctop.common.module.utils.Base64Utils;
-//import cn.com.ctop.common.module.utils.HttpUtils2;
-//import cn.com.ctop.crawler.modules.core.util.FateadmHttpUtil;
-//import cn.com.ctop.crawler.modules.core.util.FateadmUtil;
-//import cn.com.ctop.crawler.modules.oceanengine.entity.DouyinHot;
-//import cn.com.ctop.crawler.modules.oceanengine.entity.EffectCase;
-//import cn.com.ctop.crawler.modules.oceanengine.entity.HotMaterial;
-//import cn.com.ctop.crawler.modules.oceanengine.service.IOceanEngineService;
-//import com.fasterxml.jackson.core.type.TypeReference;
-//import com.fasterxml.jackson.databind.DeserializationFeature;
-//import com.fasterxml.jackson.databind.JsonNode;
-//import com.fasterxml.jackson.databind.ObjectMapper;
-//import org.apache.http.impl.cookie.BasicClientCookie;
-//import org.bytedeco.javacv.FFmpegFrameGrabber;
-//import org.bytedeco.javacv.Frame;
-//import org.bytedeco.javacv.Java2DFrameConverter;
-//import org.openqa.selenium.Cookie;
-//import org.openqa.selenium.WebDriver;
-//import org.openqa.selenium.chrome.ChromeDriver;
-//import org.openqa.selenium.chrome.ChromeOptions;
-//import org.quartz.JobExecutionContext;
-//import org.quartz.JobExecutionException;
-//import org.springframework.stereotype.Service;
-//
-//import javax.imageio.ImageIO;
-//import java.awt.*;
-//import java.awt.image.BufferedImage;
-//import java.io.ByteArrayOutputStream;
-//import java.net.HttpURLConnection;
-//import java.net.URL;
-//import java.util.*;
-//import java.util.List;
-//
-//@Service
-//public class OceanEngineServiceImpl implements IOceanEngineService {
-//    public static String getVideoUrlByVid(String vid) {
-//        String url = "https://aweme.snssdk.com/aweme/v1/playwm/?video_id=" + vid + "&line=0";
-//        return getRealVideoUrl(url);
-//    }
-//
-//    public static String getRealVideoUrl(String url) {
-//        try {
-//            URL serverUrl = new URL(url);
-//            HttpURLConnection conn = (HttpURLConnection) serverUrl
-//                    .openConnection();
-//            conn.setRequestMethod("GET");
-//            // 必须设置false,否则会自动redirect到Location的地址
-//            conn.setInstanceFollowRedirects(false);
-//            conn.addRequestProperty("User-Agent",
-//                    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.2.8) Firefox/3.6.8");
-//            conn.connect();
-//            String location = conn.getHeaderField("Location");
-//            return location;
-//        } catch (Exception e) {
-//            e.printStackTrace();
-//        }
-//        return null;
-//    }
-//    private void hotMaterialHandler(int page,int appCode,String appName){
-//        String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_material/list?material_type=3&order_by=convert_show_rate&period_type=3&aggr_app_code="+appCode+"&page="+page+"&limit=20");
-//        ObjectMapper mapper = new ObjectMapper();
-//        mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-//        try {
-//            JsonNode node = mapper.readTree(result);
-//            int code = node.get("code").asInt();
-//            if (code == 0) {
-//                if (node.get("data") != null&&node.get("data").get("materials")!=null) {
-//                    String dataJson = node.get("data").get("materials").toString();
-//                    List<HotMaterial> list = mapper.readValue(dataJson,new TypeReference<List<HotMaterial>>() {});
-//                    if(list != null && list.size() > 0){
-//                        for (HotMaterial material : list){
-//                            material.setAppCode(appCode);
-//                            material.setAppName(appName);
-//                            material.setVideoUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+material.getVid()+"&line=0"));
-//                            if(material.getVideoUrl() == null){
-//                                material.setVideoUrl(getVideoUrl(material.getVid()));
-//                            }
-////                            material.setVideoUrl(getVideoUrl(material.getVid()));
-//                            material.setCoverImage(getVideoFirstImage(material.getVideoUrl()));
+package cn.com.ctop.crawler.modules.oceanengine.service.impl;
+
+import cn.com.ctop.common.module.utils.Base64Utils;
+import cn.com.ctop.common.module.utils.HttpUtils2;
+import cn.com.ctop.crawler.modules.core.util.FateadmHttpUtil;
+import cn.com.ctop.crawler.modules.core.util.FateadmUtil;
+import cn.com.ctop.crawler.modules.oceanengine.entity.DouyinHot;
+import cn.com.ctop.crawler.modules.oceanengine.entity.EffectCase;
+import cn.com.ctop.crawler.modules.oceanengine.entity.HotMaterial;
+import cn.com.ctop.crawler.modules.oceanengine.service.IDouyinHotService;
+import cn.com.ctop.crawler.modules.oceanengine.service.IEffectCaseService;
+import cn.com.ctop.crawler.modules.oceanengine.service.IHotMaterialService;
+import cn.com.ctop.crawler.modules.oceanengine.service.IOceanEngineService;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.http.impl.cookie.BasicClientCookie;
+import org.openqa.selenium.Cookie;
+import org.openqa.selenium.WebDriver;
+import org.openqa.selenium.chrome.ChromeDriver;
+import org.openqa.selenium.chrome.ChromeOptions;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.*;
+
+@Service
+@Slf4j
+public class OceanEngineServiceImpl implements IOceanEngineService {
+    @Value("${jeecg.path.chrome-driver}")
+    private String chromeDriver;
+    private String getVideoUrlByVid(String vid) {
+        String url = "https://aweme.snssdk.com/aweme/v1/playwm/?video_id=" + vid + "&line=0";
+        return getRealVideoUrl(url);
+    }
+
+    private String getRealVideoUrl(String url) {
+        try {
+            URL serverUrl = new URL(url);
+            HttpURLConnection conn = (HttpURLConnection) serverUrl
+                    .openConnection();
+            conn.setRequestMethod("GET");
+            // 必须设置false,否则会自动redirect到Location的地址
+            conn.setInstanceFollowRedirects(false);
+            conn.addRequestProperty("User-Agent",
+                    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.2.8) Firefox/3.6.8");
+            conn.connect();
+            String location = conn.getHeaderField("Location");
+            return location;
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+        return null;
+    }
+
+    @Override
+    public void hotMaterialHandler(int page, int appCode, String appName){
+        String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_material/list?material_type=3&order_by=click_show_rate&period_type=30&aggr_app_code="+appCode+"&page="+page+"&limit=20");
+        System.out.println(result);
+        ObjectMapper mapper = new ObjectMapper();
+        mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+        try {
+            JsonNode node = mapper.readTree(result);
+            int code = node.get("code").asInt();
+            if (code == 0) {
+                if (node.get("data") != null&&node.get("data").get("materials")!=null) {
+                    String dataJson = node.get("data").get("materials").toString();
+                    List<HotMaterial> list = mapper.readValue(dataJson,new TypeReference<List<HotMaterial>>() {});
+                    if(list != null && list.size() > 0){
+                        for (HotMaterial material : list){
+                            material.setAppCode(appCode);
+                            material.setAppName(appName);
+                            material.setVideoUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+material.getVid()+"&line=0"));
+                            if(material.getVideoUrl() == null){
+                                material.setVideoUrl(getVideoUrl(material.getVid()));
+                            }
+//                            material.setVideoUrl(getVideoUrl(material.getVid()));
+//                            material.setCoverImage(FfmpegUtils.getVideoFirstImage(material.getVideoUrl()));
 //                            if(material.getCoverImage() == null){
 //                                Thread.sleep(1000);
-//                                material.setCoverImage(getVideoFirstImage(material.getVideoUrl()));
-//                            }
-//                            hotMaterialService.saveOrUpdate(material);
-//                        }
-//                        hotMaterialHandler(page+1,appCode,appName);
-//                    }
-//                }
-//            }
-//        }catch (Exception e){
-//            e.printStackTrace();
-//        }
-//    }
-//
-//    public static String getVideoUrl(String vid) throws Exception {
-//        String url = "http://i.snssdk.com/video/urls/1/toutiao/mp4/"+vid;
-//        String result = HttpUtils2.httpGetRequest(url);
-//        ObjectMapper mapper = new ObjectMapper();
-//        if(result != null){
-//            JsonNode node = mapper.readTree(result);
-//            String mainUrl = node.get("data").get("video_list").get("video_1").get("main_url").asText();
-//            return Base64Utils.decode(mainUrl);
-//        }
-//        return null;
-//    }
-//    private void effectCaseHandler(int page){
-//        String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/case/effect_case_list?industry=0&style=0&mode=0&page="+page+"&limit=20");
-//        ObjectMapper mapper = new ObjectMapper();
-//        mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-//        try {
-//            JsonNode node = mapper.readTree(result);
-//            int code = node.get("code").asInt();
-//            if(code == 0) {
-//                if (node.get("data") != null) {
-//                    String dataJson = node.get("data").get("effect_cases").toString();
-//                    List<EffectCase> list = mapper.readValue(dataJson,new TypeReference<List<EffectCase>>() {});
-//                    if (list != null && list.size() > 0) {
-//                        for (EffectCase effectCase:list){
-//                            String resultDetail = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/case/effect_case/"+effectCase.getId());
-//                            JsonNode detailNode = mapper.readTree(resultDetail);
-//                            int detailCode = detailNode.get("code").asInt();
-//                            if(detailCode == 0){
-//                                if(detailNode.get("data") != null){
-////                                    String detailJson = node.get("data").get("effect_cases").toString();
-////                                    EffectCase effectCase1 = mapper.readValue(detailJson,EffectCase.class);
-//                                    effectCase.setAuthor(detailNode.get("data").get("effect_case").get("author").asText());
-//                                    effectCase.setCreateDate(detailNode.get("data").get("effect_case").get("create_date").asText());
-//                                    effectCase.setCreativeId(detailNode.get("data").get("effect_case").get("creative_id").asLong());
-//                                    effectCase.setCreativeScore(detailNode.get("data").get("effect_case").get("creative_score").asText());
-//                                    effectCase.setEffect(detailNode.get("data").get("effect_case").get("effect").toString());
-//                                    effectCase.setMethods(detailNode.get("data").get("effect_case").get("methods").toString());
-//                                    effectCase.setVideoUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+effectCase.getVid()+"&line=0"));
-//                                    if (effectCase.getVideoUrl() == null){
-//                                        effectCase.setVideoUrl(getVideoUrl(effectCase.getVid()));
-//                                    }
-////                                    effectCase.setVideoUrl(getVideoUrl(effectCase.getVid()));
-//                                    effectCase.setCoverImage(getVideoFirstImage(effectCase.getVideoUrl()));
-//                                    if (effectCase.getCoverImage() == null){
-//                                        Thread.sleep(1000);
-//                                        effectCase.setCoverImage(getVideoFirstImage(effectCase.getVideoUrl()));
-//                                    }
-//                                    effectCase.setCreativeLabel(detailNode.get("data").get("effect_case").get("creative_label").toString());
-//                                }
+//                                material.setCoverImage(FfmpegUtils.getVideoFirstImage(material.getVideoUrl()));
 //                            }
-//                            effectCaseService.saveOrUpdate(effectCase);
-//                        }
-//                        effectCaseHandler(page+1);
+                            hotMaterialService.saveOrUpdate(material);
+                        }
+                        hotMaterialHandler(page+1,appCode,appName);
+                    }
+                }
+            }
+        }catch (Exception e){
+            e.printStackTrace();
+        }
+    }
+
+    @Autowired
+    private IHotMaterialService hotMaterialService;
+
+    private String getVideoUrl(String vid) throws Exception {
+        String url = "http://i.snssdk.com/video/urls/1/toutiao/mp4/"+vid;
+        String result = HttpUtils2.httpGetRequest(url);
+        ObjectMapper mapper = new ObjectMapper();
+        if(result != null){
+            JsonNode node = mapper.readTree(result);
+            String mainUrl = node.get("data").get("video_list").get("video_1").get("main_url").asText();
+            return Base64Utils.decode(mainUrl);
+        }
+        return null;
+    }
+
+    @Override
+    public void effectCaseHandler(int page){
+        String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/case/effect_case_list?industry=0&style=0&mode=0&page="+page+"&limit=20");
+        try {
+            JSONObject node = JSONObject.parseObject(result);
+            Integer code = node.getInteger("code");
+            String message = node.getString("msg");
+            if(null==code||code != 0) {
+                log.error("巨量创意获取效果案例异常:{}",message);
+                return;
+            }
+            JSONArray effectCaseList = node.getJSONObject("data").getJSONArray("effect_cases");
+            if (effectCaseList == null || effectCaseList.isEmpty()) {
+                log.error("巨量创意获取效果案例数据为空:{}",message);
+                return;
+            }
+            List<EffectCase> effectCases = effectCaseList.toJavaList(EffectCase.class);
+            for (EffectCase effectCase:effectCases){
+                String resultDetail = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/case/effect_case/"+effectCase.getId());
+                JSONObject detailNode = JSONObject.parseObject(resultDetail);
+                Integer detailCode = detailNode.getInteger("code");
+                if(null ==detailCode|| detailCode != 0){
+                    continue;
+                }
+                JSONObject getCase = detailNode.getJSONObject("data").getJSONObject("effect_case");
+                if(getCase != null){
+                    effectCase.setAuthor(getCase.getString("author"));
+                    effectCase.setCreateDate(getCase.getString("create_date"));
+                    effectCase.setCreativeId(getCase.getLong("creative_id"));
+                    effectCase.setCreativeScore(getCase.getString("creative_score"));
+                    effectCase.setEffect(getCase.getString("effect"));
+                    effectCase.setMethods(getCase.getString("methods"));
+                    effectCase.setVideoUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+effectCase.getVid()+"&line=0"));
+                    if (effectCase.getVideoUrl() == null){
+                        effectCase.setVideoUrl(getVideoUrl(effectCase.getVid()));
+                    }
+//                    effectCase.setCoverImage(FfmpegUtils.getVideoFirstImage(effectCase.getVideoUrl()));
+//                    if (effectCase.getCoverImage() == null){
+//                        Thread.sleep(1000);
+//                        effectCase.setCoverImage(FfmpegUtils.getVideoFirstImage(effectCase.getVideoUrl()));
 //                    }
-//                }
-//            }
-//        }catch (Exception e){
-//            e.printStackTrace();
-//        }
-//    }
-//
-//    private String getVideoFirstImage(String url){
-//        try {
-//            FFmpegFrameGrabber ff = new FFmpegFrameGrabber(url);
-//            ff.start();
-//            int lenght = ff.getLengthInFrames();
-//            int i = 0;
-//            Frame f = null;
-//            while (i < lenght) {
-//// 过滤前5帧,避免出现全黑的图片,依自己情况而定
-//                f = ff.grabFrame();
-//                if ((i > 5) && (f.image != null)) {
-//                    break;
-//                }
-//                i++;
-//            }
-////            opencv_core.IplImage img = f.image;
-//            int owidth = f.imageWidth;
-//            int oheight = f.imageHeight;
-//            Java2DFrameConverter converter =new Java2DFrameConverter();
-//            BufferedImage fecthedImage =converter.getBufferedImage(f);
-//// 对截取的帧进行等比例缩放
-//            int width = 800;
-//            int height = (int) (((double) width / owidth) * oheight);
-//            BufferedImage bi = new BufferedImage(width, height, BufferedImage.TYPE_3BYTE_BGR);
-//            bi.getGraphics().drawImage(fecthedImage.getScaledInstance(width, height, Image.SCALE_SMOOTH),
-//                    0, 0, null);
-//            ByteArrayOutputStream baos = new ByteArrayOutputStream();//io流
-//            ImageIO.write(bi, "jpg", baos);
-//            byte[] bytes = baos.toByteArray();//转换成字节
-//            String png_base64 = Base64.getEncoder().encodeToString(bytes).trim();//转换成base64串
-//            png_base64 = png_base64.replaceAll("\n", "").replaceAll("\r", "");//删除 \r\n
-//            //        ImageIO.write(bufferedImage, "png", new File("D:/qrcode1.png"));
-//            baos.close();
-//            ff.close();
-//            return "data:image/jpg;base64,"+png_base64;
-//        }catch (Exception e){
-//            e.printStackTrace();
-//            return null;
-//        }
-//
-//    }
-//    private void douyinHotHandler(int page,int listType){
-//        String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_video/user_side/list?page="+page+"&limit=20&list_type="+listType+"&aggr_duration_type=-1");
-//        ObjectMapper mapper = new ObjectMapper();
-//        try {
-//            JsonNode node = mapper.readTree(result);
-//            int code = node.get("code").asInt();
-//            if(code == 0){
-//                if(node.get("data")!= null){
-//                    String dataJson = node.get("data").get("items").toString();
-//                    List<DouyinHot> list = mapper.readValue(dataJson,new TypeReference<List<DouyinHot>>() {});
-//                    if (list != null && list.size() > 0){
-//                        for (DouyinHot douyinHot : list){
-//                            douyinHot.setUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+douyinHot.getVideoId()+"&line=0"));
-//                            if(douyinHot.getUrl() == null){
-//                                douyinHot.setUrl(getVideoUrl(douyinHot.getVideoId()));
-//                            }
-////                            douyinHot.setUrl(getVideoUrl(douyinHot.getVideoId()));
-//                            douyinHot.setCoverImage(getVideoFirstImage(douyinHot.getUrl()));
+                    effectCase.setCreativeLabel(getCase.getString("creative_label"));
+                }
+                effectCaseService.saveOrUpdate(effectCase);
+            }
+            effectCaseHandler(page+1);
+        }catch (Exception e){
+            e.printStackTrace();
+        }
+    }
+
+    @Autowired
+    private IEffectCaseService effectCaseService;
+
+    @Autowired
+    private IDouyinHotService douyinHotService;
+    @Override
+    public void douyinHotHandler(int page, int listType){
+        String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_video/user_side/list?page="+page+"&limit=20&list_type="+listType+"&aggr_duration_type=-1");
+        ObjectMapper mapper = new ObjectMapper();
+        try {
+            JsonNode node = mapper.readTree(result);
+            int code = node.get("code").asInt();
+            if(code == 0){
+                if(node.get("data")!= null){
+                    String dataJson = node.get("data").get("items").toString();
+                    List<DouyinHot> list = mapper.readValue(dataJson,new TypeReference<List<DouyinHot>>() {});
+                    if (list != null && list.size() > 0){
+                        for (DouyinHot douyinHot : list){
+                            douyinHot.setUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+douyinHot.getVideoId()+"&line=0"));
+                            if(douyinHot.getUrl() == null){
+                                douyinHot.setUrl(getVideoUrl(douyinHot.getVideoId()));
+                            }
+//                            douyinHot.setCoverImage(FfmpegUtils.getVideoFirstImage(douyinHot.getUrl()));
 //                            if(douyinHot.getCoverImage() == null){
 //                                Thread.sleep(1000);
-//                                douyinHot.setCoverImage(getVideoFirstImage(douyinHot.getUrl()));
+//                                douyinHot.setCoverImage(FfmpegUtils.getVideoFirstImage(douyinHot.getUrl()));
 //                            }
-//                            douyinHotService.saveOrUpdate(douyinHot);
-//                        }
-//                        douyinHotHandler(page+1,listType);
-//                    }else {
-//                        if(listType < 4){
-//                            douyinHotHandler(1,listType+1);
-//                        }
-//                    }
-//                }
-//            }
-//        }catch (Exception e){
-//            e.printStackTrace();
-//        }
-//
-//    }
-//    @Override
-//    public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
-//        String account="3248395570@qq.com";
-//        String password = "Ydxq-704127411";
-//        login(account,password);
-//        douyinHotHandler(1,1);
-//        effectCaseHandler(1);
-//        hotMaterialHandler(1,1,"西瓜");
-//        hotMaterialHandler(1,3,"火山");
-//        hotMaterialHandler(1,4,"抖音");
-//        hotMaterialHandler(1,8,"头条");
-//        hotMaterialHandler(1,9,"穿山甲");
-//        log.info("巨量创意抓取完成");
-//    }
-//
-//    private void login(String account,String password){
-//        String url = "https://cc.oceanengine.com/login";
-//        System.getProperties().setProperty("webdriver.chrome.driver", chromeDriver);
-//        ChromeOptions chromeOptions = new ChromeOptions();
-//        chromeOptions.addArguments("--headless");
-//        chromeOptions.addArguments("--no-sandbox");
-//        chromeOptions.addArguments("--window-size=1920,1080");
-//        chromeOptions.addArguments("--user-agent=" + HttpUtils2.USER_AGENT);
-//        chromeOptions.setAcceptInsecureCerts(true);
-//        WebDriver webDriver = new ChromeDriver(chromeOptions);
-//        //清除所有的缓存
-//        webDriver.manage().deleteAllCookies();
-//        try {
-//            //获取登录页面
-//            webDriver.get(url);
-//            for (Cookie cookie : webDriver.manage().getCookies()) {
-//                BasicClientCookie ck = new BasicClientCookie(cookie.getName(), cookie.getValue());
-//                ck.setDomain("oceanengine.com");
-//                ck.setExpiryDate(cookie.getExpiry());
-//                ck.setPath(cookie.getPath());
-//                HttpUtils2.cookieStore.addCookie(ck);
-//            }
-//            Map<String, String> header = new HashMap<>();
-//            header.put("Accept", "*/*");
-//            header.put("Connection", "keep-alive");
-//            header.put("Host", "sso.toutiao.com");
-//            header.put("Content-Type", "application/x-www-form-urlencoded");
-//            header.put("Origin", "https://cc.oceanengine.com");
-//            header.put("Referer", "https://cc.oceanengine.com/login");
-//            Map<String, Object> param = new HashMap<>();
-//            param.put("mobile", "");
-//            param.put("code", "");
-//            param.put("account", account);
-//            param.put("password", password);
-//            param.put("captcha", "hqde");
-//            param.put("is_30_days_no_login", "true");
-//            param.put("service", "https://cc.oceanengine.com");
-//            String res = HttpUtils2.httpPostParamRequest("https://sso.toutiao.com/account_login/", param, header);
-//            ObjectMapper mapper = new ObjectMapper();
-//            mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-//            JsonNode jsonNode = mapper.readTree(res);
-//            //errorcode == 1102 验证码错误,请重新输入验证码
-//            //errorcode == 1101 验证码为空
-//
-//            Integer errorCode = jsonNode.get("error_code").asInt();
-//            String orderId = null;
-//            while (errorCode != 0) {
-//                //用户名或者密码错误
-//                if (null != errorCode && errorCode == 1009) {
-//                    return;
-//                }
-//                FateadmUtil fateadmUtil = new FateadmUtil();
-//                fateadmUtil.init();
-//                if (orderId != null) {
-//                    fateadmUtil.justice(orderId);
-//                }
-//                String captcha = jsonNode.get("captcha").asText();
-//                Base64.Decoder decoder = Base64.getDecoder();
-//                FateadmHttpUtil.HttpResp resp = fateadmUtil.Predict("30400", decoder.decode(captcha));
-//                param.put("captcha", resp.predResl);
-//                orderId = resp.reqId;
-//                res = HttpUtils2.httpPostParamRequest("https://sso.toutiao.com/account_login/", param, header);
-//                jsonNode = mapper.readTree(res);
-//                errorCode = jsonNode.get("error_code").asInt();
-//            }
-//            if (errorCode == 0) {
-//                //表示登录成功
-//                List<org.apache.http.cookie.Cookie> cookies = HttpUtils2.cookieStore.getCookies();
-//                for (org.apache.http.cookie.Cookie ck : cookies) {
-//                    Cookie cookie = new Cookie(ck.getName(), ck.getValue(), "." + ck.getDomain(), ck.getPath(), ck.getExpiryDate());
-//                    webDriver.manage().addCookie(cookie);
-//                }
-//                String redirectUrl = jsonNode.get("redirect_url").asText();
-//                webDriver.get(redirectUrl);
-//                //登录成功 进入推广页面
-//                webDriver.get("https://cc.oceanengine.com/inspiration/douyin-rank-list");
-//                Thread.sleep(3000);
-//                for (Cookie cookie : webDriver.manage().getCookies()) {
-//                    BasicClientCookie ck = new BasicClientCookie(cookie.getName(), cookie.getValue());
-//                    ck.setDomain("cc.oceanengine.com");
-//                    ck.setExpiryDate(cookie.getExpiry());
-//                    ck.setPath(cookie.getPath());
-//                    HttpUtils2.cookieStore.addCookie(ck);
-//                }
-//
-//            }
-//        } catch (Exception e) {
-//            e.printStackTrace();
-//        } finally {
-//            //获得cookie
-//            Set<Cookie> coo = webDriver.manage().getCookies();
-//            //清除所有的缓存
-//            webDriver.manage().deleteAllCookies();
-//            webDriver.quit();
-//        }
-//    }
-//}
+                            douyinHotService.saveOrUpdate(douyinHot);
+                        }
+                        douyinHotHandler(page+1,listType);
+                    }else {
+                        if(listType < 4){
+                            douyinHotHandler(1,listType+1);
+                        }
+                    }
+                }
+            }
+        }catch (Exception e){
+            e.printStackTrace();
+        }
+    }
+
+    @Override
+    public void login(String account, String password){
+        String url = "https://cc.oceanengine.com/login";
+        System.getProperties().setProperty("webdriver.chrome.driver", chromeDriver);
+        ChromeOptions chromeOptions = new ChromeOptions();
+        chromeOptions.addArguments("--headless");
+        chromeOptions.addArguments("--no-sandbox");
+        chromeOptions.addArguments("--window-size=1920,1080");
+        chromeOptions.addArguments("--user-agent=" + HttpUtils2.USER_AGENT);
+        chromeOptions.setAcceptInsecureCerts(true);
+        WebDriver webDriver = new ChromeDriver(chromeOptions);
+        //清除所有的缓存
+        webDriver.manage().deleteAllCookies();
+        try {
+            //获取登录页面
+            webDriver.get(url);
+            for (Cookie cookie : webDriver.manage().getCookies()) {
+                BasicClientCookie ck = new BasicClientCookie(cookie.getName(), cookie.getValue());
+                ck.setDomain("oceanengine.com");
+                ck.setExpiryDate(cookie.getExpiry());
+                ck.setPath(cookie.getPath());
+                HttpUtils2.cookieStore.addCookie(ck);
+            }
+            Map<String, String> header = new HashMap<>();
+            header.put("Accept", "*/*");
+            header.put("Connection", "keep-alive");
+            header.put("Host", "sso.toutiao.com");
+            header.put("Content-Type", "application/x-www-form-urlencoded");
+            header.put("Origin", "https://cc.oceanengine.com");
+            header.put("Referer", "https://cc.oceanengine.com/login");
+            Map<String, Object> param = new HashMap<>();
+            param.put("mobile", "");
+            param.put("code", "");
+            param.put("account", account);
+            param.put("password", password);
+            param.put("captcha", "hqde");
+            param.put("is_30_days_no_login", "true");
+            param.put("service", "https://cc.oceanengine.com");
+            String res = HttpUtils2.httpPostParamRequest("https://sso.toutiao.com/account_login/", param, header);
+            ObjectMapper mapper = new ObjectMapper();
+            mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+            JsonNode jsonNode = mapper.readTree(res);
+            //errorcode == 1102 验证码错误,请重新输入验证码
+            //errorcode == 1101 验证码为空
+
+            Integer errorCode = jsonNode.get("error_code").asInt();
+            String orderId = null;
+            while (errorCode != 0) {
+                //用户名或者密码错误
+                if (null != errorCode && errorCode == 1009) {
+                    return;
+                }
+                FateadmUtil fateadmUtil = new FateadmUtil();
+                fateadmUtil.init();
+                if (orderId != null) {
+                    fateadmUtil.justice(orderId);
+                }
+                String captcha = jsonNode.get("captcha").asText();
+                Base64.Decoder decoder = Base64.getDecoder();
+                FateadmHttpUtil.HttpResp resp = fateadmUtil.Predict("30400", decoder.decode(captcha));
+                param.put("captcha", resp.predResl);
+                orderId = resp.reqId;
+                res = HttpUtils2.httpPostParamRequest("https://sso.toutiao.com/account_login/", param, header);
+                jsonNode = mapper.readTree(res);
+                errorCode = jsonNode.get("error_code").asInt();
+            }
+            if (errorCode == 0) {
+                //表示登录成功
+                List<org.apache.http.cookie.Cookie> cookies = HttpUtils2.cookieStore.getCookies();
+                for (org.apache.http.cookie.Cookie ck : cookies) {
+                    Cookie cookie = new Cookie(ck.getName(), ck.getValue(), "." + ck.getDomain(), ck.getPath(), ck.getExpiryDate());
+                    webDriver.manage().addCookie(cookie);
+                }
+                String redirectUrl = jsonNode.get("redirect_url").asText();
+                webDriver.get(redirectUrl);
+                //登录成功 进入推广页面
+                webDriver.get("https://cc.oceanengine.com/inspiration/douyin-rank-list");
+                Thread.sleep(3000);
+                for (Cookie cookie : webDriver.manage().getCookies()) {
+                    BasicClientCookie ck = new BasicClientCookie(cookie.getName(), cookie.getValue());
+                    ck.setDomain("cc.oceanengine.com");
+                    ck.setExpiryDate(cookie.getExpiry());
+                    ck.setPath(cookie.getPath());
+                    HttpUtils2.cookieStore.addCookie(ck);
+                }
+
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        } finally {
+            //获得cookie
+            Set<Cookie> coo = webDriver.manage().getCookies();
+            //清除所有的缓存
+            webDriver.manage().deleteAllCookies();
+            webDriver.quit();
+        }
+    }
+}

+ 4 - 13
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinCrawlerServiceImpl.java

@@ -58,7 +58,7 @@ public class PangolinCrawlerServiceImpl implements PangolinCrawlerService {
                 }
             }
             String result = HttpUtils2.httpPostRequest(url, null, null);
-            if (checkLogin(bindAccountLogin, result)) {
+            if (bindAccountLoginService.checkLogin(bindAccountLogin, result)) {
                 JsonNode resultNode = mapper.readTree(result);
                 Iterator<JsonNode> appIterator = resultNode.get("data").get("List").elements();
                 while (appIterator.hasNext()) {
@@ -122,7 +122,7 @@ public class PangolinCrawlerServiceImpl implements PangolinCrawlerService {
                 }
             }
             String result = HttpUtils2.httpGetRequest(url);
-            if (checkLogin(bindAccountLogin, result)) {
+            if (bindAccountLoginService.checkLogin(bindAccountLogin, result)) {
                 JsonNode resultNode = mapper.readTree(result);
                 Iterator<JsonNode> appIterator = resultNode.get("data").get("AppNameList").elements();
                 while (appIterator.hasNext()) {
@@ -188,7 +188,7 @@ public class PangolinCrawlerServiceImpl implements PangolinCrawlerService {
             paramMap.put("search", searchMap);
             paramMap.put("page", pageMap);
             String result = HttpUtils2.httpPostRequest(url, paramMap, null);
-            if (checkLogin(bindAccountLogin, result)) {
+            if (bindAccountLoginService.checkLogin(bindAccountLogin, result)) {
                 JsonNode resultNode = mapper.readTree(result);
                 Iterator<JsonNode> listIterator = resultNode.get("data").get("list").elements();
                 while (listIterator.hasNext()) {
@@ -259,7 +259,7 @@ public class PangolinCrawlerServiceImpl implements PangolinCrawlerService {
             paramMap.put("channelName", "");
             paramMap.put("appId", appIds);
             String result = HttpUtils2.httpPostRequest(url, paramMap, null);
-            if (checkLogin(bindAccountLogin, result)) {
+            if (bindAccountLoginService.checkLogin(bindAccountLogin, result)) {
                 JsonNode resultNode = mapper.readTree(result);
                 Iterator<String> clickIterator = resultNode.get("data").get("ChartData").get("click_num").fieldNames();
                 while (clickIterator.hasNext()) {
@@ -341,13 +341,4 @@ public class PangolinCrawlerServiceImpl implements PangolinCrawlerService {
             e.printStackTrace();
         }
     }
-
-    public boolean checkLogin(BindAccountLogin bindAccountLogin, String response) {
-        if (response.contains("<html>")) {
-            bindAccountLogin.setCookie("");
-            bindAccountLoginService.updateById(bindAccountLogin);
-            return false;
-        }
-        return true;
-    }
 }

+ 1 - 2
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinLoginServiceImpl.java

@@ -79,7 +79,7 @@ public class PangolinLoginServiceImpl implements PangolinLoginService {
                 String cookieStr = iterator.next().toString();
                 cookieList.add(cookieStr);
                 String[] cookieArray = cookieStr.split(";");
-                Map<String, String> cookieMap = new HashMap<String, String>();
+                Map<String, String> cookieMap = new HashMap<>();
                 for (String cookie : cookieArray) {
                     String[] kv = cookie.split("=");
                     if (kv.length > 1) {
@@ -105,7 +105,6 @@ public class PangolinLoginServiceImpl implements PangolinLoginService {
             e.printStackTrace();
         } finally {
             webDriver.manage().deleteAllCookies();
-//            webDriver.close();
             webDriver.quit();
         }
     }

+ 4 - 2
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/controller/BatchController.java

@@ -907,7 +907,9 @@ public class BatchController {
                         Map<String, Object> updateMap = updateService.updateUnitBid(token.getAccessToken(), accountId, unitId, multiply, userId);
                         if (!Check.isNull(updateMap)) {
                             Integer code = (Integer) updateMap.get("code");
-                            if (code != 0) {
+                            if (code == 0) {
+                                iKuaishouInterfaceService.getGroup(token.getAccessToken(), accountId, unitId);
+                            } else {
                                 JSONObject failJson = new JSONObject();
                                 failJson.put("message", updateMap.get("message"));
                                 if (!Check.isNull(group)) {
@@ -970,8 +972,8 @@ public class BatchController {
                             if (!Check.isNull(updateDeepConversionBid) && updateDeepConversionBid != 0) {
                                 Map<String, Object> updateDeepMap = updateService.updateUnitDeepConversionBid(token.getAccessToken(), accountId, unitId, deepConversionBid, userId);
                             }
-                            iKuaishouInterfaceService.getGroup(token.getAccessToken(), accountId, unitId);
                         }
+                        iKuaishouInterfaceService.getGroup(token.getAccessToken(), accountId, unitId);
                         if (!Check.isNull(updateMap)) {
                             Integer code = (Integer) updateMap.get("code");
                             if (code != 0) {

+ 18 - 13
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/graphql/service/IKuaishouWebInterfaceService.java

@@ -1,31 +1,36 @@
 package cn.com.ctop.kuaishou.modules.graphql.service;
 
+import cn.com.ctop.common.module.entity.BindAccountLogin;
+
 import java.io.IOException;
 import java.util.Map;
 
 public interface IKuaishouWebInterfaceService {
-    public void getVideoList(String uid) throws IOException;
+    void getVideoList(String uid) throws IOException;
+
+    String checkKuaishouUser(String videoUrl);
 
-    public String checkKuaishouUser(String videoUrl);
+    void adkuaishouWebLogin(String phone, String password) throws IOException;
 
-    public void adkuaishouWebLogin(String phone, String password) throws IOException;
+    void deleteAllComment(Map<String, String> pcursorMap);
+    void deleteAllComment(Map<String,String>pcursorMap,BindAccountLogin login);
 
-    public void deleteAllComment(Map<String, String> pcursorMap);
+    String qrLogin(String taskId) throws Exception;
 
-    public String qrLogin(String taskId) throws Exception;
+    Map<String, Object> loginStatus(String taskId);
 
-    public Map<String, Object> loginStatus(String taskId);
+    Map<String, Object> commentDelete(String userId, String ksid, String photoId, Long commentId);
 
-    public Map<String, Object> commentDelete(String userId, String ksid, String photoId, Long commentId);
+    Map<String, Object> videoList(String userId, String ksid, String pcursor);
 
-    public Map<String, Object> videoList(String userId, String ksid, String pcursor);
+    Map<String, Object> commentList(String userId, String ksid, String photoId, String pcursor);
 
-    public Map<String, Object> commentList(String userId, String ksid, String photoId, String pcursor);
+    Map<String, Object> commentAdd(String userId, String ksid, String photoId, String content, Long replyToCommentId, Long replyTo);
 
-    public Map<String, Object> commentAdd(String userId, String ksid, String photoId, String content, Long replyToCommentId, Long replyTo);
+    void getkuaishouWebLoginCookie(BindAccountLogin login);
 
-    public void sendPreview(Integer creativeId, String kwid);
+    void sendPreview(Integer creativeId, String kwid);
 
-    public void adekuaishouWebLogin(String phone, String password) throws IOException;
-    public Map<String, Object> subCommentList(String userId, String ksid, String photoId, Long rootCommentId, String pcursor);
+    void adekuaishouWebLogin(String phone, String password) throws IOException;
+    Map<String, Object> subCommentList(String userId, String ksid, String photoId, Long rootCommentId, String pcursor);
 }

Rozdílová data souboru nebyla zobrazena, protože soubor je příliš velký
+ 203 - 163
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/graphql/service/impl/KuaishouWebInterfaceServiceImpl.java