yumeng 5 năm trước cách đây
mục cha
commit
0e8ef43d4e

+ 366 - 366
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/OceanengineJob.java

@@ -1,366 +1,366 @@
-//package org.jeecg.modules.ctop.job;
-//
-//import cn.com.ctop.common.module.utils.Base64Utils;
-//import cn.com.ctop.common.module.utils.HttpUtils2;
-//import cn.com.ctop.crawler.modules.core.util.FateadmHttpUtil;
-//import cn.com.ctop.crawler.modules.core.util.FateadmUtil;
-//import cn.com.ctop.crawler.modules.oceanengine.entity.DouyinHot;
-//import cn.com.ctop.crawler.modules.oceanengine.entity.EffectCase;
-//import cn.com.ctop.crawler.modules.oceanengine.entity.HotMaterial;
-//import cn.com.ctop.crawler.modules.oceanengine.service.IDouyinHotService;
-//import cn.com.ctop.crawler.modules.oceanengine.service.IEffectCaseService;
-//import cn.com.ctop.crawler.modules.oceanengine.service.IHotMaterialService;
-//import com.fasterxml.jackson.core.type.TypeReference;
-//import com.fasterxml.jackson.databind.DeserializationFeature;
-//import com.fasterxml.jackson.databind.JsonNode;
-//import com.fasterxml.jackson.databind.ObjectMapper;
-//import org.apache.http.impl.cookie.BasicClientCookie;
-//import org.bytedeco.javacpp.opencv_core;
-//import org.bytedeco.javacv.FFmpegFrameGrabber;
-//import org.bytedeco.javacv.Frame;
-//import org.bytedeco.javacv.Java2DFrameConverter;
-//import org.openqa.selenium.Cookie;
-//import org.openqa.selenium.WebDriver;
-//import org.openqa.selenium.chrome.ChromeDriver;
-//import org.openqa.selenium.chrome.ChromeOptions;
-//import org.quartz.Job;
-//import org.quartz.JobExecutionContext;
-//import org.quartz.JobExecutionException;
-//import org.springframework.beans.factory.annotation.Autowired;
-//import org.springframework.beans.factory.annotation.Value;
-//
-//import javax.imageio.ImageIO;
-//import java.awt.*;
-//import java.awt.image.BufferedImage;
-//import java.io.ByteArrayOutputStream;
-//import java.net.HttpURLConnection;
-//import java.net.URL;
-//import java.util.List;
-//import java.util.*;
-//
-//public class OceanengineJob implements Job {
-//    @Value("${jeecg.path.chrome-driver}")
-//    private String chromeDriver;
-//    @Autowired
-//    private IDouyinHotService douyinHotService;
-//    @Autowired
-//    private IEffectCaseService effectCaseService;
-//    @Autowired
-//    private IHotMaterialService hotMaterialService;
-//    public static void main(String[] args){
-//        System.out.println(getRealVideoUrl(""));
-//    }
-//
-//    public static String getVideoUrlByVid(String vid) {
-//        String url = "https://aweme.snssdk.com/aweme/v1/playwm/?video_id=" + vid + "&line=0";
-//        return getRealVideoUrl(url);
-//    }
-//
-//    public static String getRealVideoUrl(String url) {
-//        try {
-//            URL serverUrl = new URL(url);
-//            HttpURLConnection conn = (HttpURLConnection) serverUrl
-//                    .openConnection();
-//            conn.setRequestMethod("GET");
-//            // 必须设置false,否则会自动redirect到Location的地址
-//            conn.setInstanceFollowRedirects(false);
-//            conn.addRequestProperty("User-Agent",
-//                    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.2.8) Firefox/3.6.8");
-//            conn.connect();
-//            String location = conn.getHeaderField("Location");
-//            return location;
-//        } catch (Exception e) {
-//            e.printStackTrace();
-//        }
-//        return null;
-//    }
-//    private void hotMaterialHandler(int page,int appCode,String appName){
-//        String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_material/list?material_type=3&order_by=convert_show_rate&period_type=3&aggr_app_code="+appCode+"&page="+page+"&limit=20");
-//        ObjectMapper mapper = new ObjectMapper();
-//        mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-//        try {
-//            JsonNode node = mapper.readTree(result);
-//            int code = node.get("code").asInt();
-//            if (code == 0) {
-//                if (node.get("data") != null&&node.get("data").get("materials")!=null) {
-//                    String dataJson = node.get("data").get("materials").toString();
-//                    List<HotMaterial> list = mapper.readValue(dataJson,new TypeReference<List<HotMaterial>>() {});
-//                    if(list != null && list.size() > 0){
-//                        for (HotMaterial material : list){
-//                            material.setAppCode(appCode);
-//                            material.setAppName(appName);
-//                            material.setVideoUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+material.getVid()+"&line=0"));
-//                            if(material.getVideoUrl() == null){
-//                                material.setVideoUrl(getVideoUrl(material.getVid()));
-//                            }
-////                            material.setVideoUrl(getVideoUrl(material.getVid()));
-//                            material.setCoverImage(getVideoFirstImage(material.getVideoUrl()));
-//                            if(material.getCoverImage() == null){
-//                                Thread.sleep(1000);
-//                                material.setCoverImage(getVideoFirstImage(material.getVideoUrl()));
-//                            }
-//                            hotMaterialService.saveOrUpdate(material);
-//                        }
-//                        hotMaterialHandler(page+1,appCode,appName);
-//                    }
-//                }
-//            }
-//        }catch (Exception e){
-//            e.printStackTrace();
-//        }
-//    }
-//
-//    public static String getVideoUrl(String vid) throws Exception {
-//        String url = "http://i.snssdk.com/video/urls/1/toutiao/mp4/"+vid;
-//        String result = HttpUtils2.httpGetRequest(url);
-//        ObjectMapper mapper = new ObjectMapper();
-//        if(result != null){
-//            JsonNode node = mapper.readTree(result);
-//            String mainUrl = node.get("data").get("video_list").get("video_1").get("main_url").asText();
-//            return Base64Utils.decode(mainUrl);
-//        }
-//        return null;
-//    }
-//    private void effectCaseHandler(int page){
-//        String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/case/effect_case_list?industry=0&style=0&mode=0&page="+page+"&limit=20");
-//        ObjectMapper mapper = new ObjectMapper();
-//        mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-//        try {
-//            JsonNode node = mapper.readTree(result);
-//            int code = node.get("code").asInt();
-//            if(code == 0) {
-//                if (node.get("data") != null) {
-//                    String dataJson = node.get("data").get("effect_cases").toString();
-//                    List<EffectCase> list = mapper.readValue(dataJson,new TypeReference<List<EffectCase>>() {});
-//                    if (list != null && list.size() > 0) {
-//                        for (EffectCase effectCase:list){
-//                            String resultDetail = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/case/effect_case/"+effectCase.getId());
-//                            JsonNode detailNode = mapper.readTree(resultDetail);
-//                            int detailCode = detailNode.get("code").asInt();
-//                            if(detailCode == 0){
-//                                if(detailNode.get("data") != null){
-////                                    String detailJson = node.get("data").get("effect_cases").toString();
-////                                    EffectCase effectCase1 = mapper.readValue(detailJson,EffectCase.class);
-//                                    effectCase.setAuthor(detailNode.get("data").get("effect_case").get("author").asText());
-//                                    effectCase.setCreateDate(detailNode.get("data").get("effect_case").get("create_date").asText());
-//                                    effectCase.setCreativeId(detailNode.get("data").get("effect_case").get("creative_id").asLong());
-//                                    effectCase.setCreativeScore(detailNode.get("data").get("effect_case").get("creative_score").asText());
-//                                    effectCase.setEffect(detailNode.get("data").get("effect_case").get("effect").toString());
-//                                    effectCase.setMethods(detailNode.get("data").get("effect_case").get("methods").toString());
-//                                    effectCase.setVideoUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+effectCase.getVid()+"&line=0"));
-//                                    if (effectCase.getVideoUrl() == null){
-//                                        effectCase.setVideoUrl(getVideoUrl(effectCase.getVid()));
-//                                    }
-////                                    effectCase.setVideoUrl(getVideoUrl(effectCase.getVid()));
-//                                    effectCase.setCoverImage(getVideoFirstImage(effectCase.getVideoUrl()));
-//                                    if (effectCase.getCoverImage() == null){
-//                                        Thread.sleep(1000);
-//                                        effectCase.setCoverImage(getVideoFirstImage(effectCase.getVideoUrl()));
-//                                    }
-//                                    effectCase.setCreativeLabel(detailNode.get("data").get("effect_case").get("creative_label").toString());
-//                                }
-//                            }
-//                            effectCaseService.saveOrUpdate(effectCase);
-//                        }
-//                        effectCaseHandler(page+1);
-//                    }
-//                }
-//            }
-//        }catch (Exception e){
-//            e.printStackTrace();
-//        }
-//    }
-//
-//    private String getVideoFirstImage(String url){
-//        try {
-//            FFmpegFrameGrabber ff = new FFmpegFrameGrabber(url);
-//            ff.start();
-//            int lenght = ff.getLengthInFrames();
-//            int i = 0;
-//            Frame f = null;
-//            while (i < lenght) {
-//// 过滤前5帧,避免出现全黑的图片,依自己情况而定
-//                f = ff.grabFrame();
-//                if ((i > 5) && (f.image != null)) {
-//                    break;
-//                }
-//                i++;
-//            }
-////            opencv_core.IplImage img = f.image;
-//            int owidth = f.imageWidth;
-//            int oheight = f.imageHeight;
-//            Java2DFrameConverter converter =new Java2DFrameConverter();
-//            BufferedImage fecthedImage =converter.getBufferedImage(f);
-//// 对截取的帧进行等比例缩放
-//            int width = 800;
-//            int height = (int) (((double) width / owidth) * oheight);
-//            BufferedImage bi = new BufferedImage(width, height, BufferedImage.TYPE_3BYTE_BGR);
-//            bi.getGraphics().drawImage(fecthedImage.getScaledInstance(width, height, Image.SCALE_SMOOTH),
-//                    0, 0, null);
-//            ByteArrayOutputStream baos = new ByteArrayOutputStream();//io流
-//            ImageIO.write(bi, "jpg", baos);
-//            byte[] bytes = baos.toByteArray();//转换成字节
-//            String png_base64 = Base64.getEncoder().encodeToString(bytes).trim();//转换成base64串
-//            png_base64 = png_base64.replaceAll("\n", "").replaceAll("\r", "");//删除 \r\n
-//            //        ImageIO.write(bufferedImage, "png", new File("D:/qrcode1.png"));
-//            baos.close();
-//            ff.close();
-//            return "data:image/jpg;base64,"+png_base64;
-//        }catch (Exception e){
-//            e.printStackTrace();
-//            return null;
-//        }
-//
-//    }
-//    private void douyinHotHandler(int page,int listType){
-//        String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_video/user_side/list?page="+page+"&limit=20&list_type="+listType+"&aggr_duration_type=-1");
-//        ObjectMapper mapper = new ObjectMapper();
-//        try {
-//            JsonNode node = mapper.readTree(result);
-//            int code = node.get("code").asInt();
-//            if(code == 0){
-//                if(node.get("data")!= null){
-//                    String dataJson = node.get("data").get("items").toString();
-//                    List<DouyinHot> list = mapper.readValue(dataJson,new TypeReference<List<DouyinHot>>() {});
-//                    if (list != null && list.size() > 0){
-//                        for (DouyinHot douyinHot : list){
-//                            douyinHot.setUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+douyinHot.getVideoId()+"&line=0"));
-//                            if(douyinHot.getUrl() == null){
-//                                douyinHot.setUrl(getVideoUrl(douyinHot.getVideoId()));
-//                            }
-////                            douyinHot.setUrl(getVideoUrl(douyinHot.getVideoId()));
-//                            douyinHot.setCoverImage(getVideoFirstImage(douyinHot.getUrl()));
-//                            if(douyinHot.getCoverImage() == null){
-//                                Thread.sleep(1000);
-//                                douyinHot.setCoverImage(getVideoFirstImage(douyinHot.getUrl()));
-//                            }
-//                            douyinHotService.saveOrUpdate(douyinHot);
-//                        }
-//                        douyinHotHandler(page+1,listType);
-//                    }else {
-//                        if(listType < 4){
-//                            douyinHotHandler(1,listType+1);
-//                        }
-//                    }
-//                }
-//            }
-//        }catch (Exception e){
-//            e.printStackTrace();
-//        }
-//
-//    }
-//    @Override
-//    public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
-//        String account="3248395570@qq.com";
-//        String password = "Ydxq-704127411";
-//        login(account,password);
-//        douyinHotHandler(1,1);
-//        effectCaseHandler(1);
-//        hotMaterialHandler(1,1,"西瓜");
-//        hotMaterialHandler(1,3,"火山");
-//        hotMaterialHandler(1,4,"抖音");
-//        hotMaterialHandler(1,8,"头条");
-//        hotMaterialHandler(1,9,"穿山甲");
-//        System.out.println("巨量创意抓取完成");
-//    }
-//
-//    private void login(String account,String password){
-//        String url = "https://cc.oceanengine.com/login";
-//        System.getProperties().setProperty("webdriver.chrome.driver", chromeDriver);
-//        ChromeOptions chromeOptions = new ChromeOptions();
-//        chromeOptions.addArguments("--headless");
-//        chromeOptions.addArguments("--no-sandbox");
-//        chromeOptions.addArguments("--window-size=1920,1080");
-//        chromeOptions.addArguments("--user-agent=" + HttpUtils2.USER_AGENT);
-//        chromeOptions.setAcceptInsecureCerts(true);
-//        WebDriver webDriver = new ChromeDriver(chromeOptions);
-//        //清除所有的缓存
-//        webDriver.manage().deleteAllCookies();
-//        try {
-//            //获取登录页面
-//            webDriver.get(url);
-//            for (Cookie cookie : webDriver.manage().getCookies()) {
-//                BasicClientCookie ck = new BasicClientCookie(cookie.getName(), cookie.getValue());
-//                ck.setDomain("oceanengine.com");
-//                ck.setExpiryDate(cookie.getExpiry());
-//                ck.setPath(cookie.getPath());
-//                HttpUtils2.cookieStore.addCookie(ck);
-//            }
-//            Map<String, String> header = new HashMap<>();
-//            header.put("Accept", "*/*");
-//            header.put("Connection", "keep-alive");
-//            header.put("Host", "sso.toutiao.com");
-//            header.put("Content-Type", "application/x-www-form-urlencoded");
-//            header.put("Origin", "https://cc.oceanengine.com");
-//            header.put("Referer", "https://cc.oceanengine.com/login");
-//            Map<String, Object> param = new HashMap<>();
-//            param.put("mobile", "");
-//            param.put("code", "");
-//            param.put("account", account);
-//            param.put("password", password);
-//            param.put("captcha", "hqde");
-//            param.put("is_30_days_no_login", "true");
-//            param.put("service", "https://cc.oceanengine.com");
-//            String res = HttpUtils2.httpPostParamRequest("https://sso.toutiao.com/account_login/", param, header);
-//            System.out.println(res);
-//            ObjectMapper mapper = new ObjectMapper();
-//            mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-//            JsonNode jsonNode = mapper.readTree(res);
-//            //errorcode == 1102 验证码错误,请重新输入验证码
-//            //errorcode == 1101 验证码为空
-//
-//            Integer errorCode = jsonNode.get("error_code").asInt();
-//            String orderId = null;
-//            while (errorCode != 0) {
-//                //用户名或者密码错误
-//                if (null != errorCode && errorCode == 1009) {
-//                    return;
-//                }
-//                FateadmUtil fateadmUtil = new FateadmUtil();
-//                fateadmUtil.init();
-//                if (orderId != null) {
-//                    fateadmUtil.Justice(orderId);
-//                }
-//                String captcha = jsonNode.get("captcha").asText();
-//                Base64.Decoder decoder = Base64.getDecoder();
-//                FateadmHttpUtil.HttpResp resp = fateadmUtil.Predict("30400", decoder.decode(captcha));
-//                param.put("captcha", resp.pred_resl);
-//                orderId = resp.req_id;
-//                res = HttpUtils2.httpPostParamRequest("https://sso.toutiao.com/account_login/", param, header);
-//                System.out.println(res);
-//                jsonNode = mapper.readTree(res);
-//                errorCode = jsonNode.get("error_code").asInt();
-//            }
-//            if (errorCode == 0) {
-//                //表示登录成功
-//
-//                List<org.apache.http.cookie.Cookie> cookies = HttpUtils2.cookieStore.getCookies();
-//                for (org.apache.http.cookie.Cookie ck : cookies) {
-//                    Cookie cookie = new Cookie(ck.getName(), ck.getValue(), "." + ck.getDomain(), ck.getPath(), ck.getExpiryDate());
-//                    webDriver.manage().addCookie(cookie);
-//                }
-//                String redirectUrl = jsonNode.get("redirect_url").asText();
-//                webDriver.get(redirectUrl);
-//                //登录成功 进入推广页面
-//                webDriver.get("https://cc.oceanengine.com/inspiration/douyin-rank-list");
-//                Thread.sleep(3000);
-//                for (Cookie cookie : webDriver.manage().getCookies()) {
-//                    BasicClientCookie ck = new BasicClientCookie(cookie.getName(), cookie.getValue());
-//                    ck.setDomain("cc.oceanengine.com");
-//                    ck.setExpiryDate(cookie.getExpiry());
-//                    ck.setPath(cookie.getPath());
-//                    HttpUtils2.cookieStore.addCookie(ck);
-//                }
-//
-//            }
-//        } catch (Exception e) {
-//            e.printStackTrace();
-//        } finally {
-//            //获得cookie
-//            Set<Cookie> coo = webDriver.manage().getCookies();
-//            //清除所有的缓存
-//            webDriver.manage().deleteAllCookies();
-//            webDriver.quit();
-//        }
-//    }
-//}
+package org.jeecg.modules.ctop.job;
+
+import cn.com.ctop.common.module.utils.Base64Utils;
+import cn.com.ctop.common.module.utils.HttpUtils2;
+import cn.com.ctop.crawler.modules.core.util.FateadmHttpUtil;
+import cn.com.ctop.crawler.modules.core.util.FateadmUtil;
+import cn.com.ctop.crawler.modules.oceanengine.entity.DouyinHot;
+import cn.com.ctop.crawler.modules.oceanengine.entity.EffectCase;
+import cn.com.ctop.crawler.modules.oceanengine.entity.HotMaterial;
+import cn.com.ctop.crawler.modules.oceanengine.service.IDouyinHotService;
+import cn.com.ctop.crawler.modules.oceanengine.service.IEffectCaseService;
+import cn.com.ctop.crawler.modules.oceanengine.service.IHotMaterialService;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.http.impl.cookie.BasicClientCookie;
+import org.bytedeco.javacpp.opencv_core;
+import org.bytedeco.javacv.FFmpegFrameGrabber;
+import org.bytedeco.javacv.Frame;
+import org.bytedeco.javacv.Java2DFrameConverter;
+import org.openqa.selenium.Cookie;
+import org.openqa.selenium.WebDriver;
+import org.openqa.selenium.chrome.ChromeDriver;
+import org.openqa.selenium.chrome.ChromeOptions;
+import org.quartz.Job;
+import org.quartz.JobExecutionContext;
+import org.quartz.JobExecutionException;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+
+import javax.imageio.ImageIO;
+import java.awt.*;
+import java.awt.image.BufferedImage;
+import java.io.ByteArrayOutputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.List;
+import java.util.*;
+
+public class OceanengineJob implements Job {
+    @Value("${jeecg.path.chrome-driver}")
+    private String chromeDriver;
+    @Autowired
+    private IDouyinHotService douyinHotService;
+    @Autowired
+    private IEffectCaseService effectCaseService;
+    @Autowired
+    private IHotMaterialService hotMaterialService;
+    public static void main(String[] args){
+        System.out.println(getRealVideoUrl(""));
+    }
+
+    public static String getVideoUrlByVid(String vid) {
+        String url = "https://aweme.snssdk.com/aweme/v1/playwm/?video_id=" + vid + "&line=0";
+        return getRealVideoUrl(url);
+    }
+
+    public static String getRealVideoUrl(String url) {
+        try {
+            URL serverUrl = new URL(url);
+            HttpURLConnection conn = (HttpURLConnection) serverUrl
+                    .openConnection();
+            conn.setRequestMethod("GET");
+            // 必须设置false,否则会自动redirect到Location的地址
+            conn.setInstanceFollowRedirects(false);
+            conn.addRequestProperty("User-Agent",
+                    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.2.8) Firefox/3.6.8");
+            conn.connect();
+            String location = conn.getHeaderField("Location");
+            return location;
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+        return null;
+    }
+    private void hotMaterialHandler(int page,int appCode,String appName){
+        String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_material/list?material_type=3&order_by=convert_show_rate&period_type=3&aggr_app_code="+appCode+"&page="+page+"&limit=20");
+        ObjectMapper mapper = new ObjectMapper();
+        mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+        try {
+            JsonNode node = mapper.readTree(result);
+            int code = node.get("code").asInt();
+            if (code == 0) {
+                if (node.get("data") != null&&node.get("data").get("materials")!=null) {
+                    String dataJson = node.get("data").get("materials").toString();
+                    List<HotMaterial> list = mapper.readValue(dataJson,new TypeReference<List<HotMaterial>>() {});
+                    if(list != null && list.size() > 0){
+                        for (HotMaterial material : list){
+                            material.setAppCode(appCode);
+                            material.setAppName(appName);
+                            material.setVideoUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+material.getVid()+"&line=0"));
+                            if(material.getVideoUrl() == null){
+                                material.setVideoUrl(getVideoUrl(material.getVid()));
+                            }
+//                            material.setVideoUrl(getVideoUrl(material.getVid()));
+                            material.setCoverImage(getVideoFirstImage(material.getVideoUrl()));
+                            if(material.getCoverImage() == null){
+                                Thread.sleep(1000);
+                                material.setCoverImage(getVideoFirstImage(material.getVideoUrl()));
+                            }
+                            hotMaterialService.saveOrUpdate(material);
+                        }
+                        hotMaterialHandler(page+1,appCode,appName);
+                    }
+                }
+            }
+        }catch (Exception e){
+            e.printStackTrace();
+        }
+    }
+
+    public static String getVideoUrl(String vid) throws Exception {
+        String url = "http://i.snssdk.com/video/urls/1/toutiao/mp4/"+vid;
+        String result = HttpUtils2.httpGetRequest(url);
+        ObjectMapper mapper = new ObjectMapper();
+        if(result != null){
+            JsonNode node = mapper.readTree(result);
+            String mainUrl = node.get("data").get("video_list").get("video_1").get("main_url").asText();
+            return Base64Utils.decode(mainUrl);
+        }
+        return null;
+    }
+    private void effectCaseHandler(int page){
+        String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/case/effect_case_list?industry=0&style=0&mode=0&page="+page+"&limit=20");
+        ObjectMapper mapper = new ObjectMapper();
+        mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+        try {
+            JsonNode node = mapper.readTree(result);
+            int code = node.get("code").asInt();
+            if(code == 0) {
+                if (node.get("data") != null) {
+                    String dataJson = node.get("data").get("effect_cases").toString();
+                    List<EffectCase> list = mapper.readValue(dataJson,new TypeReference<List<EffectCase>>() {});
+                    if (list != null && list.size() > 0) {
+                        for (EffectCase effectCase:list){
+                            String resultDetail = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/case/effect_case/"+effectCase.getId());
+                            JsonNode detailNode = mapper.readTree(resultDetail);
+                            int detailCode = detailNode.get("code").asInt();
+                            if(detailCode == 0){
+                                if(detailNode.get("data") != null){
+//                                    String detailJson = node.get("data").get("effect_cases").toString();
+//                                    EffectCase effectCase1 = mapper.readValue(detailJson,EffectCase.class);
+                                    effectCase.setAuthor(detailNode.get("data").get("effect_case").get("author").asText());
+                                    effectCase.setCreateDate(detailNode.get("data").get("effect_case").get("create_date").asText());
+                                    effectCase.setCreativeId(detailNode.get("data").get("effect_case").get("creative_id").asLong());
+                                    effectCase.setCreativeScore(detailNode.get("data").get("effect_case").get("creative_score").asText());
+                                    effectCase.setEffect(detailNode.get("data").get("effect_case").get("effect").toString());
+                                    effectCase.setMethods(detailNode.get("data").get("effect_case").get("methods").toString());
+                                    effectCase.setVideoUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+effectCase.getVid()+"&line=0"));
+                                    if (effectCase.getVideoUrl() == null){
+                                        effectCase.setVideoUrl(getVideoUrl(effectCase.getVid()));
+                                    }
+//                                    effectCase.setVideoUrl(getVideoUrl(effectCase.getVid()));
+                                    effectCase.setCoverImage(getVideoFirstImage(effectCase.getVideoUrl()));
+                                    if (effectCase.getCoverImage() == null){
+                                        Thread.sleep(1000);
+                                        effectCase.setCoverImage(getVideoFirstImage(effectCase.getVideoUrl()));
+                                    }
+                                    effectCase.setCreativeLabel(detailNode.get("data").get("effect_case").get("creative_label").toString());
+                                }
+                            }
+                            effectCaseService.saveOrUpdate(effectCase);
+                        }
+                        effectCaseHandler(page+1);
+                    }
+                }
+            }
+        }catch (Exception e){
+            e.printStackTrace();
+        }
+    }
+
+    private String getVideoFirstImage(String url){
+        try {
+            FFmpegFrameGrabber ff = new FFmpegFrameGrabber(url);
+            ff.start();
+            int lenght = ff.getLengthInFrames();
+            int i = 0;
+            Frame f = null;
+            while (i < lenght) {
+// 过滤前5帧,避免出现全黑的图片,依自己情况而定
+                f = ff.grabFrame();
+                if ((i > 5) && (f.image != null)) {
+                    break;
+                }
+                i++;
+            }
+//            opencv_core.IplImage img = f.image;
+            int owidth = f.imageWidth;
+            int oheight = f.imageHeight;
+            Java2DFrameConverter converter =new Java2DFrameConverter();
+            BufferedImage fecthedImage =converter.getBufferedImage(f);
+// 对截取的帧进行等比例缩放
+            int width = 800;
+            int height = (int) (((double) width / owidth) * oheight);
+            BufferedImage bi = new BufferedImage(width, height, BufferedImage.TYPE_3BYTE_BGR);
+            bi.getGraphics().drawImage(fecthedImage.getScaledInstance(width, height, Image.SCALE_SMOOTH),
+                    0, 0, null);
+            ByteArrayOutputStream baos = new ByteArrayOutputStream();//io流
+            ImageIO.write(bi, "jpg", baos);
+            byte[] bytes = baos.toByteArray();//转换成字节
+            String png_base64 = Base64.getEncoder().encodeToString(bytes).trim();//转换成base64串
+            png_base64 = png_base64.replaceAll("\n", "").replaceAll("\r", "");//删除 \r\n
+            //        ImageIO.write(bufferedImage, "png", new File("D:/qrcode1.png"));
+            baos.close();
+            ff.close();
+            return "data:image/jpg;base64,"+png_base64;
+        }catch (Exception e){
+            e.printStackTrace();
+            return null;
+        }
+
+    }
+    private void douyinHotHandler(int page,int listType){
+        String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_video/user_side/list?page="+page+"&limit=20&list_type="+listType+"&aggr_duration_type=-1");
+        ObjectMapper mapper = new ObjectMapper();
+        try {
+            JsonNode node = mapper.readTree(result);
+            int code = node.get("code").asInt();
+            if(code == 0){
+                if(node.get("data")!= null){
+                    String dataJson = node.get("data").get("items").toString();
+                    List<DouyinHot> list = mapper.readValue(dataJson,new TypeReference<List<DouyinHot>>() {});
+                    if (list != null && list.size() > 0){
+                        for (DouyinHot douyinHot : list){
+                            douyinHot.setUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+douyinHot.getVideoId()+"&line=0"));
+                            if(douyinHot.getUrl() == null){
+                                douyinHot.setUrl(getVideoUrl(douyinHot.getVideoId()));
+                            }
+//                            douyinHot.setUrl(getVideoUrl(douyinHot.getVideoId()));
+                            douyinHot.setCoverImage(getVideoFirstImage(douyinHot.getUrl()));
+                            if(douyinHot.getCoverImage() == null){
+                                Thread.sleep(1000);
+                                douyinHot.setCoverImage(getVideoFirstImage(douyinHot.getUrl()));
+                            }
+                            douyinHotService.saveOrUpdate(douyinHot);
+                        }
+                        douyinHotHandler(page+1,listType);
+                    }else {
+                        if(listType < 4){
+                            douyinHotHandler(1,listType+1);
+                        }
+                    }
+                }
+            }
+        }catch (Exception e){
+            e.printStackTrace();
+        }
+
+    }
+    @Override
+    public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
+        String account="3248395570@qq.com";
+        String password = "Ydxq-704127411";
+        login(account,password);
+        douyinHotHandler(1,1);
+        effectCaseHandler(1);
+        hotMaterialHandler(1,1,"西瓜");
+        hotMaterialHandler(1,3,"火山");
+        hotMaterialHandler(1,4,"抖音");
+        hotMaterialHandler(1,8,"头条");
+        hotMaterialHandler(1,9,"穿山甲");
+        System.out.println("巨量创意抓取完成");
+    }
+
+    private void login(String account,String password){
+        String url = "https://cc.oceanengine.com/login";
+        System.getProperties().setProperty("webdriver.chrome.driver", chromeDriver);
+        ChromeOptions chromeOptions = new ChromeOptions();
+        chromeOptions.addArguments("--headless");
+        chromeOptions.addArguments("--no-sandbox");
+        chromeOptions.addArguments("--window-size=1920,1080");
+        chromeOptions.addArguments("--user-agent=" + HttpUtils2.USER_AGENT);
+        chromeOptions.setAcceptInsecureCerts(true);
+        WebDriver webDriver = new ChromeDriver(chromeOptions);
+        //清除所有的缓存
+        webDriver.manage().deleteAllCookies();
+        try {
+            //获取登录页面
+            webDriver.get(url);
+            for (Cookie cookie : webDriver.manage().getCookies()) {
+                BasicClientCookie ck = new BasicClientCookie(cookie.getName(), cookie.getValue());
+                ck.setDomain("oceanengine.com");
+                ck.setExpiryDate(cookie.getExpiry());
+                ck.setPath(cookie.getPath());
+                HttpUtils2.cookieStore.addCookie(ck);
+            }
+            Map<String, String> header = new HashMap<>();
+            header.put("Accept", "*/*");
+            header.put("Connection", "keep-alive");
+            header.put("Host", "sso.toutiao.com");
+            header.put("Content-Type", "application/x-www-form-urlencoded");
+            header.put("Origin", "https://cc.oceanengine.com");
+            header.put("Referer", "https://cc.oceanengine.com/login");
+            Map<String, Object> param = new HashMap<>();
+            param.put("mobile", "");
+            param.put("code", "");
+            param.put("account", account);
+            param.put("password", password);
+            param.put("captcha", "hqde");
+            param.put("is_30_days_no_login", "true");
+            param.put("service", "https://cc.oceanengine.com");
+            String res = HttpUtils2.httpPostParamRequest("https://sso.toutiao.com/account_login/", param, header);
+            System.out.println(res);
+            ObjectMapper mapper = new ObjectMapper();
+            mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+            JsonNode jsonNode = mapper.readTree(res);
+            //errorcode == 1102 验证码错误,请重新输入验证码
+            //errorcode == 1101 验证码为空
+
+            Integer errorCode = jsonNode.get("error_code").asInt();
+            String orderId = null;
+            while (errorCode != 0) {
+                //用户名或者密码错误
+                if (null != errorCode && errorCode == 1009) {
+                    return;
+                }
+                FateadmUtil fateadmUtil = new FateadmUtil();
+                fateadmUtil.init();
+                if (orderId != null) {
+                    fateadmUtil.Justice(orderId);
+                }
+                String captcha = jsonNode.get("captcha").asText();
+                Base64.Decoder decoder = Base64.getDecoder();
+                FateadmHttpUtil.HttpResp resp = fateadmUtil.Predict("30400", decoder.decode(captcha));
+                param.put("captcha", resp.pred_resl);
+                orderId = resp.req_id;
+                res = HttpUtils2.httpPostParamRequest("https://sso.toutiao.com/account_login/", param, header);
+                System.out.println(res);
+                jsonNode = mapper.readTree(res);
+                errorCode = jsonNode.get("error_code").asInt();
+            }
+            if (errorCode == 0) {
+                //表示登录成功
+
+                List<org.apache.http.cookie.Cookie> cookies = HttpUtils2.cookieStore.getCookies();
+                for (org.apache.http.cookie.Cookie ck : cookies) {
+                    Cookie cookie = new Cookie(ck.getName(), ck.getValue(), "." + ck.getDomain(), ck.getPath(), ck.getExpiryDate());
+                    webDriver.manage().addCookie(cookie);
+                }
+                String redirectUrl = jsonNode.get("redirect_url").asText();
+                webDriver.get(redirectUrl);
+                //登录成功 进入推广页面
+                webDriver.get("https://cc.oceanengine.com/inspiration/douyin-rank-list");
+                Thread.sleep(3000);
+                for (Cookie cookie : webDriver.manage().getCookies()) {
+                    BasicClientCookie ck = new BasicClientCookie(cookie.getName(), cookie.getValue());
+                    ck.setDomain("cc.oceanengine.com");
+                    ck.setExpiryDate(cookie.getExpiry());
+                    ck.setPath(cookie.getPath());
+                    HttpUtils2.cookieStore.addCookie(ck);
+                }
+
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        } finally {
+            //获得cookie
+            Set<Cookie> coo = webDriver.manage().getCookies();
+            //清除所有的缓存
+            webDriver.manage().deleteAllCookies();
+            webDriver.quit();
+        }
+    }
+}