Forráskód Böngészése

抽取巨量创意爬虫逻辑

syh 5 éve
szülő
commit
ec7acea9d9

+ 21 - 6
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/PangolinController.java

@@ -1,5 +1,6 @@
 package org.jeecg.modules.ctop.controller;
 
+import cn.com.ctop.common.module.service.IBindAccountLoginService;
 import cn.com.ctop.common.module.service.ISysRoleService;
 import cn.com.ctop.common.module.utils.ResultMapUtils;
 import cn.com.ctop.common.module.utils.StatusCode;
@@ -33,11 +34,14 @@ public class PangolinController {
     private PangolinChannelService pangolinChannelService;
     @Autowired
     private ISysRoleService sysRoleService;
+    @Autowired
+    private IBindAccountLoginService bindAccountLoginService;
+
     @PostMapping("api/channel/list")
     public Map<String, Object> channelList(@RequestBody JSONObject data) {
-//        LoginUser user = (LoginUser) SecurityUtils.getSubject().getPrincipal();
-//        String userId = user.getId();
-        String userId = "6b4c821adc414dc8b7718ab63ccbfcaf";
+        LoginUser user = (LoginUser) SecurityUtils.getSubject().getPrincipal();
+        String userId = user.getId();
+//        String userId = "6b4c821adc414dc8b7718ab63ccbfcaf";
         String roleCode = sysRoleService.getRoleCodeByUserId(userId);
         if(null!=roleCode&&(roleCode.equals("admin"))){
             userId = null;
@@ -45,12 +49,23 @@ public class PangolinController {
         return pangolinChannelService.getListByUserId(userId);
     }
 
+    @PostMapping("api/user/list")
+    public Map<String, Object> userList() {
+        LoginUser user = (LoginUser) SecurityUtils.getSubject().getPrincipal();
+        String userId = user.getId();
+//        String userId = "6b4c821adc414dc8b7718ab63ccbfcaf";
+        String roleCode = sysRoleService.getRoleCodeByUserId(userId);
+        if(null!=roleCode&&(roleCode.equals("admin"))){
+            userId = null;
+        }
+        return bindAccountLoginService.getListByUserId(userId);
+    }
 
     @PostMapping("api/app/list")
     public Map<String, Object> appList() {
-//        LoginUser user = (LoginUser) SecurityUtils.getSubject().getPrincipal();
-//        String userId = user.getId();
-        String userId = "6b4c821adc414dc8b7718ab63ccbfcaf";
+        LoginUser user = (LoginUser) SecurityUtils.getSubject().getPrincipal();
+        String userId = user.getId();
+//        String userId = "6b4c821adc414dc8b7718ab63ccbfcaf";
         String roleCode = sysRoleService.getRoleCodeByUserId(userId);
         if(null!=roleCode&&(roleCode.equals("admin"))){
             userId = null;

+ 10 - 342
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/OceanengineJob.java

@@ -1,361 +1,29 @@
 package org.jeecg.modules.ctop.job;
 
-import cn.com.ctop.common.module.utils.Base64Utils;
-import cn.com.ctop.common.module.utils.HttpUtils2;
-import cn.com.ctop.crawler.modules.core.util.FateadmHttpUtil;
-import cn.com.ctop.crawler.modules.core.util.FateadmUtil;
-import cn.com.ctop.crawler.modules.oceanengine.entity.DouyinHot;
-import cn.com.ctop.crawler.modules.oceanengine.entity.EffectCase;
-import cn.com.ctop.crawler.modules.oceanengine.entity.HotMaterial;
-import cn.com.ctop.crawler.modules.oceanengine.service.IDouyinHotService;
-import cn.com.ctop.crawler.modules.oceanengine.service.IEffectCaseService;
-import cn.com.ctop.crawler.modules.oceanengine.service.IHotMaterialService;
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
+import cn.com.ctop.crawler.modules.oceanengine.service.IOceanEngineService;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.http.impl.cookie.BasicClientCookie;
-import org.bytedeco.javacv.FFmpegFrameGrabber;
-import org.bytedeco.javacv.Frame;
-import org.bytedeco.javacv.Java2DFrameConverter;
-import org.openqa.selenium.Cookie;
-import org.openqa.selenium.WebDriver;
-import org.openqa.selenium.chrome.ChromeDriver;
-import org.openqa.selenium.chrome.ChromeOptions;
 import org.quartz.Job;
 import org.quartz.JobExecutionContext;
 import org.quartz.JobExecutionException;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
-
-import javax.imageio.ImageIO;
-import java.awt.*;
-import java.awt.image.BufferedImage;
-import java.io.ByteArrayOutputStream;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.util.List;
-import java.util.*;
 
 @Slf4j
 public class OceanengineJob implements Job {
-    @Value("${jeecg.path.chrome-driver}")
-    private String chromeDriver;
-    @Autowired
-    private IDouyinHotService douyinHotService;
-    @Autowired
-    private IEffectCaseService effectCaseService;
     @Autowired
-    private IHotMaterialService hotMaterialService;
-
-    public static String getVideoUrlByVid(String vid) {
-        String url = "https://aweme.snssdk.com/aweme/v1/playwm/?video_id=" + vid + "&line=0";
-        return getRealVideoUrl(url);
-    }
+    private IOceanEngineService oceanEngineService;
 
-    public static String getRealVideoUrl(String url) {
-        try {
-            URL serverUrl = new URL(url);
-            HttpURLConnection conn = (HttpURLConnection) serverUrl
-                    .openConnection();
-            conn.setRequestMethod("GET");
-            // 必须设置false,否则会自动redirect到Location的地址
-            conn.setInstanceFollowRedirects(false);
-            conn.addRequestProperty("User-Agent",
-                    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.2.8) Firefox/3.6.8");
-            conn.connect();
-            String location = conn.getHeaderField("Location");
-            return location;
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-        return null;
-    }
-    private void hotMaterialHandler(int page,int appCode,String appName){
-        String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_material/list?material_type=3&order_by=convert_show_rate&period_type=3&aggr_app_code="+appCode+"&page="+page+"&limit=20");
-        ObjectMapper mapper = new ObjectMapper();
-        mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-        try {
-            JsonNode node = mapper.readTree(result);
-            int code = node.get("code").asInt();
-            if (code == 0) {
-                if (node.get("data") != null&&node.get("data").get("materials")!=null) {
-                    String dataJson = node.get("data").get("materials").toString();
-                    List<HotMaterial> list = mapper.readValue(dataJson,new TypeReference<List<HotMaterial>>() {});
-                    if(list != null && list.size() > 0){
-                        for (HotMaterial material : list){
-                            material.setAppCode(appCode);
-                            material.setAppName(appName);
-                            material.setVideoUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+material.getVid()+"&line=0"));
-                            if(material.getVideoUrl() == null){
-                                material.setVideoUrl(getVideoUrl(material.getVid()));
-                            }
-//                            material.setVideoUrl(getVideoUrl(material.getVid()));
-                            material.setCoverImage(getVideoFirstImage(material.getVideoUrl()));
-                            if(material.getCoverImage() == null){
-                                Thread.sleep(1000);
-                                material.setCoverImage(getVideoFirstImage(material.getVideoUrl()));
-                            }
-                            hotMaterialService.saveOrUpdate(material);
-                        }
-                        hotMaterialHandler(page+1,appCode,appName);
-                    }
-                }
-            }
-        }catch (Exception e){
-            e.printStackTrace();
-        }
-    }
-
-    public static String getVideoUrl(String vid) throws Exception {
-        String url = "http://i.snssdk.com/video/urls/1/toutiao/mp4/"+vid;
-        String result = HttpUtils2.httpGetRequest(url);
-        ObjectMapper mapper = new ObjectMapper();
-        if(result != null){
-            JsonNode node = mapper.readTree(result);
-            String mainUrl = node.get("data").get("video_list").get("video_1").get("main_url").asText();
-            return Base64Utils.decode(mainUrl);
-        }
-        return null;
-    }
-    private void effectCaseHandler(int page){
-        String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/case/effect_case_list?industry=0&style=0&mode=0&page="+page+"&limit=20");
-        ObjectMapper mapper = new ObjectMapper();
-        mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-        try {
-            JsonNode node = mapper.readTree(result);
-            int code = node.get("code").asInt();
-            if(code == 0) {
-                if (node.get("data") != null) {
-                    String dataJson = node.get("data").get("effect_cases").toString();
-                    List<EffectCase> list = mapper.readValue(dataJson,new TypeReference<List<EffectCase>>() {});
-                    if (list != null && list.size() > 0) {
-                        for (EffectCase effectCase:list){
-                            String resultDetail = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/case/effect_case/"+effectCase.getId());
-                            JsonNode detailNode = mapper.readTree(resultDetail);
-                            int detailCode = detailNode.get("code").asInt();
-                            if(detailCode == 0){
-                                if(detailNode.get("data") != null){
-//                                    String detailJson = node.get("data").get("effect_cases").toString();
-//                                    EffectCase effectCase1 = mapper.readValue(detailJson,EffectCase.class);
-                                    effectCase.setAuthor(detailNode.get("data").get("effect_case").get("author").asText());
-                                    effectCase.setCreateDate(detailNode.get("data").get("effect_case").get("create_date").asText());
-                                    effectCase.setCreativeId(detailNode.get("data").get("effect_case").get("creative_id").asLong());
-                                    effectCase.setCreativeScore(detailNode.get("data").get("effect_case").get("creative_score").asText());
-                                    effectCase.setEffect(detailNode.get("data").get("effect_case").get("effect").toString());
-                                    effectCase.setMethods(detailNode.get("data").get("effect_case").get("methods").toString());
-                                    effectCase.setVideoUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+effectCase.getVid()+"&line=0"));
-                                    if (effectCase.getVideoUrl() == null){
-                                        effectCase.setVideoUrl(getVideoUrl(effectCase.getVid()));
-                                    }
-//                                    effectCase.setVideoUrl(getVideoUrl(effectCase.getVid()));
-                                    effectCase.setCoverImage(getVideoFirstImage(effectCase.getVideoUrl()));
-                                    if (effectCase.getCoverImage() == null){
-                                        Thread.sleep(1000);
-                                        effectCase.setCoverImage(getVideoFirstImage(effectCase.getVideoUrl()));
-                                    }
-                                    effectCase.setCreativeLabel(detailNode.get("data").get("effect_case").get("creative_label").toString());
-                                }
-                            }
-                            effectCaseService.saveOrUpdate(effectCase);
-                        }
-                        effectCaseHandler(page+1);
-                    }
-                }
-            }
-        }catch (Exception e){
-            e.printStackTrace();
-        }
-    }
-
-    private String getVideoFirstImage(String url){
-        try {
-            FFmpegFrameGrabber ff = new FFmpegFrameGrabber(url);
-            ff.start();
-            int lenght = ff.getLengthInFrames();
-            int i = 0;
-            Frame f = null;
-            while (i < lenght) {
-// 过滤前5帧,避免出现全黑的图片,依自己情况而定
-                f = ff.grabFrame();
-                if ((i > 5) && (f.image != null)) {
-                    break;
-                }
-                i++;
-            }
-//            opencv_core.IplImage img = f.image;
-            int owidth = f.imageWidth;
-            int oheight = f.imageHeight;
-            Java2DFrameConverter converter =new Java2DFrameConverter();
-            BufferedImage fecthedImage =converter.getBufferedImage(f);
-// 对截取的帧进行等比例缩放
-            int width = 800;
-            int height = (int) (((double) width / owidth) * oheight);
-            BufferedImage bi = new BufferedImage(width, height, BufferedImage.TYPE_3BYTE_BGR);
-            bi.getGraphics().drawImage(fecthedImage.getScaledInstance(width, height, Image.SCALE_SMOOTH),
-                    0, 0, null);
-            ByteArrayOutputStream baos = new ByteArrayOutputStream();//io流
-            ImageIO.write(bi, "jpg", baos);
-            byte[] bytes = baos.toByteArray();//转换成字节
-            String png_base64 = Base64.getEncoder().encodeToString(bytes).trim();//转换成base64串
-            png_base64 = png_base64.replaceAll("\n", "").replaceAll("\r", "");//删除 \r\n
-            //        ImageIO.write(bufferedImage, "png", new File("D:/qrcode1.png"));
-            baos.close();
-            ff.close();
-            return "data:image/jpg;base64,"+png_base64;
-        }catch (Exception e){
-            e.printStackTrace();
-            return null;
-        }
-
-    }
-    private void douyinHotHandler(int page,int listType){
-        String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_video/user_side/list?page="+page+"&limit=20&list_type="+listType+"&aggr_duration_type=-1");
-        ObjectMapper mapper = new ObjectMapper();
-        try {
-            JsonNode node = mapper.readTree(result);
-            int code = node.get("code").asInt();
-            if(code == 0){
-                if(node.get("data")!= null){
-                    String dataJson = node.get("data").get("items").toString();
-                    List<DouyinHot> list = mapper.readValue(dataJson,new TypeReference<List<DouyinHot>>() {});
-                    if (list != null && list.size() > 0){
-                        for (DouyinHot douyinHot : list){
-                            douyinHot.setUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+douyinHot.getVideoId()+"&line=0"));
-                            if(douyinHot.getUrl() == null){
-                                douyinHot.setUrl(getVideoUrl(douyinHot.getVideoId()));
-                            }
-//                            douyinHot.setUrl(getVideoUrl(douyinHot.getVideoId()));
-                            douyinHot.setCoverImage(getVideoFirstImage(douyinHot.getUrl()));
-                            if(douyinHot.getCoverImage() == null){
-                                Thread.sleep(1000);
-                                douyinHot.setCoverImage(getVideoFirstImage(douyinHot.getUrl()));
-                            }
-                            douyinHotService.saveOrUpdate(douyinHot);
-                        }
-                        douyinHotHandler(page+1,listType);
-                    }else {
-                        if(listType < 4){
-                            douyinHotHandler(1,listType+1);
-                        }
-                    }
-                }
-            }
-        }catch (Exception e){
-            e.printStackTrace();
-        }
-
-    }
     @Override
     public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
         String account="3248395570@qq.com";
         String password = "Ydxq-704127411";
-        login(account,password);
-        douyinHotHandler(1,1);
-        effectCaseHandler(1);
-        hotMaterialHandler(1,1,"西瓜");
-        hotMaterialHandler(1,3,"火山");
-        hotMaterialHandler(1,4,"抖音");
-        hotMaterialHandler(1,8,"头条");
-        hotMaterialHandler(1,9,"穿山甲");
+        oceanEngineService.login(account,password);
+        oceanEngineService.douyinHotHandler(1,1);
+        oceanEngineService.effectCaseHandler(1);
+        oceanEngineService.hotMaterialHandler(1,1,"西瓜");
+        oceanEngineService.hotMaterialHandler(1,3,"火山");
+        oceanEngineService.hotMaterialHandler(1,4,"抖音");
+        oceanEngineService.hotMaterialHandler(1,8,"头条");
+        oceanEngineService.hotMaterialHandler(1,9,"穿山甲");
         log.info("巨量创意抓取完成");
     }
-
-    private void login(String account,String password){
-        String url = "https://cc.oceanengine.com/login";
-        System.getProperties().setProperty("webdriver.chrome.driver", chromeDriver);
-        ChromeOptions chromeOptions = new ChromeOptions();
-        chromeOptions.addArguments("--headless");
-        chromeOptions.addArguments("--no-sandbox");
-        chromeOptions.addArguments("--window-size=1920,1080");
-        chromeOptions.addArguments("--user-agent=" + HttpUtils2.USER_AGENT);
-        chromeOptions.setAcceptInsecureCerts(true);
-        WebDriver webDriver = new ChromeDriver(chromeOptions);
-        //清除所有的缓存
-        webDriver.manage().deleteAllCookies();
-        try {
-            //获取登录页面
-            webDriver.get(url);
-            for (Cookie cookie : webDriver.manage().getCookies()) {
-                BasicClientCookie ck = new BasicClientCookie(cookie.getName(), cookie.getValue());
-                ck.setDomain("oceanengine.com");
-                ck.setExpiryDate(cookie.getExpiry());
-                ck.setPath(cookie.getPath());
-                HttpUtils2.cookieStore.addCookie(ck);
-            }
-            Map<String, String> header = new HashMap<>();
-            header.put("Accept", "*/*");
-            header.put("Connection", "keep-alive");
-            header.put("Host", "sso.toutiao.com");
-            header.put("Content-Type", "application/x-www-form-urlencoded");
-            header.put("Origin", "https://cc.oceanengine.com");
-            header.put("Referer", "https://cc.oceanengine.com/login");
-            Map<String, Object> param = new HashMap<>();
-            param.put("mobile", "");
-            param.put("code", "");
-            param.put("account", account);
-            param.put("password", password);
-            param.put("captcha", "hqde");
-            param.put("is_30_days_no_login", "true");
-            param.put("service", "https://cc.oceanengine.com");
-            String res = HttpUtils2.httpPostParamRequest("https://sso.toutiao.com/account_login/", param, header);
-            ObjectMapper mapper = new ObjectMapper();
-            mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-            JsonNode jsonNode = mapper.readTree(res);
-            //errorcode == 1102 验证码错误,请重新输入验证码
-            //errorcode == 1101 验证码为空
-
-            Integer errorCode = jsonNode.get("error_code").asInt();
-            String orderId = null;
-            while (errorCode != 0) {
-                //用户名或者密码错误
-                if (null != errorCode && errorCode == 1009) {
-                    return;
-                }
-                FateadmUtil fateadmUtil = new FateadmUtil();
-                fateadmUtil.init();
-                if (orderId != null) {
-                    fateadmUtil.justice(orderId);
-                }
-                String captcha = jsonNode.get("captcha").asText();
-                Base64.Decoder decoder = Base64.getDecoder();
-                FateadmHttpUtil.HttpResp resp = fateadmUtil.Predict("30400", decoder.decode(captcha));
-                param.put("captcha", resp.predResl);
-                orderId = resp.reqId;
-                res = HttpUtils2.httpPostParamRequest("https://sso.toutiao.com/account_login/", param, header);
-                jsonNode = mapper.readTree(res);
-                errorCode = jsonNode.get("error_code").asInt();
-            }
-            if (errorCode == 0) {
-                //表示登录成功
-                List<org.apache.http.cookie.Cookie> cookies = HttpUtils2.cookieStore.getCookies();
-                for (org.apache.http.cookie.Cookie ck : cookies) {
-                    Cookie cookie = new Cookie(ck.getName(), ck.getValue(), "." + ck.getDomain(), ck.getPath(), ck.getExpiryDate());
-                    webDriver.manage().addCookie(cookie);
-                }
-                String redirectUrl = jsonNode.get("redirect_url").asText();
-                webDriver.get(redirectUrl);
-                //登录成功 进入推广页面
-                webDriver.get("https://cc.oceanengine.com/inspiration/douyin-rank-list");
-                Thread.sleep(3000);
-                for (Cookie cookie : webDriver.manage().getCookies()) {
-                    BasicClientCookie ck = new BasicClientCookie(cookie.getName(), cookie.getValue());
-                    ck.setDomain("cc.oceanengine.com");
-                    ck.setExpiryDate(cookie.getExpiry());
-                    ck.setPath(cookie.getPath());
-                    HttpUtils2.cookieStore.addCookie(ck);
-                }
-
-            }
-        } catch (Exception e) {
-            e.printStackTrace();
-        } finally {
-            //获得cookie
-            Set<Cookie> coo = webDriver.manage().getCookies();
-            //清除所有的缓存
-            webDriver.manage().deleteAllCookies();
-            webDriver.quit();
-        }
-    }
 }

+ 14 - 95
jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java

@@ -1,114 +1,33 @@
 package org.jeecg;
 
-import cn.com.ctop.common.module.entity.BindAccountLogin;
-import cn.com.ctop.common.module.entity.CtopOauthToken;
-import cn.com.ctop.common.module.service.IBindAccountLoginService;
-import cn.com.ctop.common.module.service.ICtopOauthTokenService;
-import cn.com.ctop.common.module.utils.CtopAdConstant;
-import cn.com.ctop.crawler.modules.pangolin.entity.PangolinApp;
-import cn.com.ctop.crawler.modules.pangolin.service.PangolinAppService;
-import cn.com.ctop.crawler.modules.pangolin.service.PangolinCrawlerService;
-import cn.com.ctop.crawler.modules.pangolin.service.PangolinLoginService;
-import cn.com.ctop.toutiao.modules.report.service.IBytedanceFundDailyService;
-import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import cn.com.ctop.crawler.modules.oceanengine.service.IOceanEngineService;
 import lombok.extern.slf4j.Slf4j;
-import org.jeecg.common.util.DateUtils;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.test.context.junit4.SpringRunner;
 
-import java.util.List;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
 @RunWith(SpringRunner.class)
 @SpringBootTest
 @Slf4j
 public class SampleTest {
     @Autowired
-    private IBindAccountLoginService bindAccountLoginService;
-    @Autowired
-    private PangolinLoginService pangolinLoginService;
-    @Autowired
-    private PangolinCrawlerService pangolinCrawlerService;
-    @Autowired
-    private PangolinAppService pangolinAppService;
+    private IOceanEngineService oceanEngineService;
     @Test
-    public void testPangolinLogin(){
-        QueryWrapper<BindAccountLogin> queryWrapper = new QueryWrapper<>();
-        queryWrapper.eq("login_type","pangolin");
-        queryWrapper.eq("status",1);
-        List<BindAccountLogin> list = bindAccountLoginService.list(queryWrapper);
-        if(list!=null&&!list.isEmpty()){
-            for (BindAccountLogin login:list) {
-                if(null==login.getCookie()||"".equals(login.getCookie().trim())){
-                    pangolinLoginService.pangolinLogin(login.getAccountName(), login.getPassword());
-                }
-            }
-        }
+    public void testOceanEngineJob(){
+        String account="3248395570@qq.com";
+        String password = "Ydxq-704127411";
+        oceanEngineService.login(account,password);
+        oceanEngineService.douyinHotHandler(1,1);
+        oceanEngineService.effectCaseHandler(1);
+        oceanEngineService.hotMaterialHandler(1,1,"西瓜");
+        oceanEngineService.hotMaterialHandler(1,3,"火山");
+        oceanEngineService.hotMaterialHandler(1,4,"抖音");
+        oceanEngineService.hotMaterialHandler(1,8,"头条");
+        oceanEngineService.hotMaterialHandler(1,9,"穿山甲");
+        log.info("巨量创意抓取完成");
     }
-
-    @Test
-    public void testPangolinData(){
-        QueryWrapper<BindAccountLogin> queryWrapper = new QueryWrapper<>();
-        queryWrapper.eq("login_type", "pangolin");
-        queryWrapper.eq("status", 1);
-        queryWrapper.isNotNull("cookie");
-        List<BindAccountLogin> list = bindAccountLoginService.list(queryWrapper);
-        if (list != null && !list.isEmpty()) {
-            for (BindAccountLogin bindAccountLogin : list) {
-                pangolinCrawlerService.getChannelList(bindAccountLogin);
-                pangolinCrawlerService.getActivationList(bindAccountLogin, DateUtils.getNowDate("yyyy-MM-dd"), 1);
-                pangolinCrawlerService.getCheckList(bindAccountLogin);
-                QueryWrapper<PangolinApp> queryWrapper1 = new QueryWrapper<>();
-                queryWrapper1.eq("account_name", bindAccountLogin.getAccountName());
-                List<PangolinApp> appList = pangolinAppService.list(queryWrapper1);
-                if (appList != null && !appList.isEmpty()) {
-                    appList.forEach(pangolinApp -> pangolinCrawlerService.getRealTimeList(bindAccountLogin, pangolinApp.getId()));
-                }
-            }
-        }
-    }
-
-    @Autowired
-    private ICtopOauthTokenService tokenService;
-    @Autowired
-    private IBytedanceFundDailyService bytedanceFundDailyService;
-    static ExecutorService executorService = null;
-
-
-    @Test
-    public void testflowFund(){
-        List<CtopOauthToken> tokens = tokenService.getTokenListByType(CtopAdConstant.PLATFORM_TYPE_BYTEDANCE);
-        if (null == tokens || tokens.size() <= 0) {
-            log.error("头条账户流水数据异常:未获取到可用的token");
-            return ;
-        }
-        CountDownLatch countDownLatch   = new CountDownLatch(tokens.size());
-        executorService = Executors.newFixedThreadPool(5);
-        tokens.forEach(token -> {
-            executorService.submit(() -> {
-                try {
-                    for(int i=0;i<60;i++){
-                        String findDate = DateUtils.addDay(DateUtils.formatDate(),-i);
-                        bytedanceFundDailyService.loadFundDataByPage(token, findDate, findDate,1);
-                    }
-                }catch (Exception e){
-                }finally {
-                    countDownLatch.countDown();
-                }
-            });
-        });
-        try {
-            countDownLatch.await();
-        } catch (InterruptedException e) {
-            e.printStackTrace();
-        }
-    }
-
 }
 
 

+ 8 - 0
module-common/src/main/java/cn/com/ctop/common/module/entity/BindAccountLogin.java

@@ -101,4 +101,12 @@ public class BindAccountLogin {
     private Date updateTime;
 
     private String cookie;
+
+    public BindAccountLogin(Long id,String accountName) {
+        this.id = id;
+        this.accountName = accountName;
+    }
+
+    public BindAccountLogin() {
+    }
 }

+ 4 - 0
module-common/src/main/java/cn/com/ctop/common/module/service/IBindAccountLoginService.java

@@ -3,6 +3,8 @@ package cn.com.ctop.common.module.service;
 import cn.com.ctop.common.module.entity.BindAccountLogin;
 import com.baomidou.mybatisplus.extension.service.IService;
 
+import java.util.Map;
+
 /**
  * @Description: 账号绑定-登陆
  * @Author: jeecg-boot
@@ -17,4 +19,6 @@ public interface IBindAccountLoginService extends IService<BindAccountLogin> {
      * @return
      */
     BindAccountLogin selectByAccountId(String accountId);
+
+    Map<String, Object> getListByUserId(String userId);
 }

+ 25 - 0
module-common/src/main/java/cn/com/ctop/common/module/service/impl/BindAccountLoginServiceImpl.java

@@ -3,11 +3,19 @@ package cn.com.ctop.common.module.service.impl;
 import cn.com.ctop.common.module.entity.BindAccountLogin;
 import cn.com.ctop.common.module.mapper.BindAccountLoginMapper;
 import cn.com.ctop.common.module.service.IBindAccountLoginService;
+import cn.com.ctop.common.module.utils.CtopAdConstant;
+import cn.com.ctop.common.module.utils.ResultMapUtils;
+import cn.com.ctop.common.module.utils.StatusCode;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Service;
 
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 /**
  * @Description: 账号绑定-登陆
  * @Author: jeecg-boot
@@ -23,4 +31,21 @@ public class BindAccountLoginServiceImpl extends ServiceImpl<BindAccountLoginMap
         queryWrapper.eq("account_id", accountId).orderByDesc("id").last("limit 1");
         return this.getOne(queryWrapper);
     }
+
+    @Override
+    public Map<String, Object> getListByUserId(String userId) {
+        Map<String,Object>result = new HashMap<>();
+        QueryWrapper<BindAccountLogin> queryWrapper = new QueryWrapper<>();
+        queryWrapper.eq("login_type", CtopAdConstant.PLATFORM_TYPE_PANGOLIN_PY);
+        List<BindAccountLogin> loginList = new ArrayList<>();
+        if(userId == null||"".equals(userId.trim())){
+            loginList.add(new BindAccountLogin(0L,"全部"));
+        }else{
+            queryWrapper.eq("user_id",userId);
+        }
+        loginList.addAll(this.list(queryWrapper));
+        result.put("data",loginList);
+        ResultMapUtils.setResultMap(result, StatusCode.COMMON_SUCCESS);
+        return result;
+    }
 }

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/service/IOceanEngineService.java

@@ -1,4 +1,11 @@
 package cn.com.ctop.crawler.modules.oceanengine.service;
 
 public interface IOceanEngineService {
+    void hotMaterialHandler(int page, int appCode, String appName);
+
+    void effectCaseHandler(int page);
+
+    void douyinHotHandler(int page, int listType);
+
+    void login(String account, String password);
 }

+ 351 - 347
module-crawler/src/main/java/cn/com/ctop/crawler/modules/oceanengine/service/impl/OceanEngineServiceImpl.java

@@ -1,347 +1,351 @@
-//package cn.com.ctop.crawler.modules.oceanengine.service.impl;
-//
-//import cn.com.ctop.common.module.utils.Base64Utils;
-//import cn.com.ctop.common.module.utils.HttpUtils2;
-//import cn.com.ctop.crawler.modules.core.util.FateadmHttpUtil;
-//import cn.com.ctop.crawler.modules.core.util.FateadmUtil;
-//import cn.com.ctop.crawler.modules.oceanengine.entity.DouyinHot;
-//import cn.com.ctop.crawler.modules.oceanengine.entity.EffectCase;
-//import cn.com.ctop.crawler.modules.oceanengine.entity.HotMaterial;
-//import cn.com.ctop.crawler.modules.oceanengine.service.IOceanEngineService;
-//import com.fasterxml.jackson.core.type.TypeReference;
-//import com.fasterxml.jackson.databind.DeserializationFeature;
-//import com.fasterxml.jackson.databind.JsonNode;
-//import com.fasterxml.jackson.databind.ObjectMapper;
-//import org.apache.http.impl.cookie.BasicClientCookie;
-//import org.bytedeco.javacv.FFmpegFrameGrabber;
-//import org.bytedeco.javacv.Frame;
-//import org.bytedeco.javacv.Java2DFrameConverter;
-//import org.openqa.selenium.Cookie;
-//import org.openqa.selenium.WebDriver;
-//import org.openqa.selenium.chrome.ChromeDriver;
-//import org.openqa.selenium.chrome.ChromeOptions;
-//import org.quartz.JobExecutionContext;
-//import org.quartz.JobExecutionException;
-//import org.springframework.stereotype.Service;
-//
-//import javax.imageio.ImageIO;
-//import java.awt.*;
-//import java.awt.image.BufferedImage;
-//import java.io.ByteArrayOutputStream;
-//import java.net.HttpURLConnection;
-//import java.net.URL;
-//import java.util.*;
-//import java.util.List;
-//
-//@Service
-//public class OceanEngineServiceImpl implements IOceanEngineService {
-//    public static String getVideoUrlByVid(String vid) {
-//        String url = "https://aweme.snssdk.com/aweme/v1/playwm/?video_id=" + vid + "&line=0";
-//        return getRealVideoUrl(url);
-//    }
-//
-//    public static String getRealVideoUrl(String url) {
-//        try {
-//            URL serverUrl = new URL(url);
-//            HttpURLConnection conn = (HttpURLConnection) serverUrl
-//                    .openConnection();
-//            conn.setRequestMethod("GET");
-//            // 必须设置false,否则会自动redirect到Location的地址
-//            conn.setInstanceFollowRedirects(false);
-//            conn.addRequestProperty("User-Agent",
-//                    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.2.8) Firefox/3.6.8");
-//            conn.connect();
-//            String location = conn.getHeaderField("Location");
-//            return location;
-//        } catch (Exception e) {
-//            e.printStackTrace();
-//        }
-//        return null;
-//    }
-//    private void hotMaterialHandler(int page,int appCode,String appName){
-//        String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_material/list?material_type=3&order_by=convert_show_rate&period_type=3&aggr_app_code="+appCode+"&page="+page+"&limit=20");
-//        ObjectMapper mapper = new ObjectMapper();
-//        mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-//        try {
-//            JsonNode node = mapper.readTree(result);
-//            int code = node.get("code").asInt();
-//            if (code == 0) {
-//                if (node.get("data") != null&&node.get("data").get("materials")!=null) {
-//                    String dataJson = node.get("data").get("materials").toString();
-//                    List<HotMaterial> list = mapper.readValue(dataJson,new TypeReference<List<HotMaterial>>() {});
-//                    if(list != null && list.size() > 0){
-//                        for (HotMaterial material : list){
-//                            material.setAppCode(appCode);
-//                            material.setAppName(appName);
-//                            material.setVideoUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+material.getVid()+"&line=0"));
-//                            if(material.getVideoUrl() == null){
-//                                material.setVideoUrl(getVideoUrl(material.getVid()));
-//                            }
-////                            material.setVideoUrl(getVideoUrl(material.getVid()));
-//                            material.setCoverImage(getVideoFirstImage(material.getVideoUrl()));
-//                            if(material.getCoverImage() == null){
-//                                Thread.sleep(1000);
-//                                material.setCoverImage(getVideoFirstImage(material.getVideoUrl()));
-//                            }
-//                            hotMaterialService.saveOrUpdate(material);
-//                        }
-//                        hotMaterialHandler(page+1,appCode,appName);
-//                    }
-//                }
-//            }
-//        }catch (Exception e){
-//            e.printStackTrace();
-//        }
-//    }
-//
-//    public static String getVideoUrl(String vid) throws Exception {
-//        String url = "http://i.snssdk.com/video/urls/1/toutiao/mp4/"+vid;
-//        String result = HttpUtils2.httpGetRequest(url);
-//        ObjectMapper mapper = new ObjectMapper();
-//        if(result != null){
-//            JsonNode node = mapper.readTree(result);
-//            String mainUrl = node.get("data").get("video_list").get("video_1").get("main_url").asText();
-//            return Base64Utils.decode(mainUrl);
-//        }
-//        return null;
-//    }
-//    private void effectCaseHandler(int page){
-//        String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/case/effect_case_list?industry=0&style=0&mode=0&page="+page+"&limit=20");
-//        ObjectMapper mapper = new ObjectMapper();
-//        mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-//        try {
-//            JsonNode node = mapper.readTree(result);
-//            int code = node.get("code").asInt();
-//            if(code == 0) {
-//                if (node.get("data") != null) {
-//                    String dataJson = node.get("data").get("effect_cases").toString();
-//                    List<EffectCase> list = mapper.readValue(dataJson,new TypeReference<List<EffectCase>>() {});
-//                    if (list != null && list.size() > 0) {
-//                        for (EffectCase effectCase:list){
-//                            String resultDetail = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/case/effect_case/"+effectCase.getId());
-//                            JsonNode detailNode = mapper.readTree(resultDetail);
-//                            int detailCode = detailNode.get("code").asInt();
-//                            if(detailCode == 0){
-//                                if(detailNode.get("data") != null){
-////                                    String detailJson = node.get("data").get("effect_cases").toString();
-////                                    EffectCase effectCase1 = mapper.readValue(detailJson,EffectCase.class);
-//                                    effectCase.setAuthor(detailNode.get("data").get("effect_case").get("author").asText());
-//                                    effectCase.setCreateDate(detailNode.get("data").get("effect_case").get("create_date").asText());
-//                                    effectCase.setCreativeId(detailNode.get("data").get("effect_case").get("creative_id").asLong());
-//                                    effectCase.setCreativeScore(detailNode.get("data").get("effect_case").get("creative_score").asText());
-//                                    effectCase.setEffect(detailNode.get("data").get("effect_case").get("effect").toString());
-//                                    effectCase.setMethods(detailNode.get("data").get("effect_case").get("methods").toString());
-//                                    effectCase.setVideoUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+effectCase.getVid()+"&line=0"));
-//                                    if (effectCase.getVideoUrl() == null){
-//                                        effectCase.setVideoUrl(getVideoUrl(effectCase.getVid()));
-//                                    }
-////                                    effectCase.setVideoUrl(getVideoUrl(effectCase.getVid()));
-//                                    effectCase.setCoverImage(getVideoFirstImage(effectCase.getVideoUrl()));
-//                                    if (effectCase.getCoverImage() == null){
-//                                        Thread.sleep(1000);
-//                                        effectCase.setCoverImage(getVideoFirstImage(effectCase.getVideoUrl()));
-//                                    }
-//                                    effectCase.setCreativeLabel(detailNode.get("data").get("effect_case").get("creative_label").toString());
-//                                }
-//                            }
-//                            effectCaseService.saveOrUpdate(effectCase);
-//                        }
-//                        effectCaseHandler(page+1);
-//                    }
-//                }
-//            }
-//        }catch (Exception e){
-//            e.printStackTrace();
-//        }
-//    }
-//
-//    private String getVideoFirstImage(String url){
-//        try {
-//            FFmpegFrameGrabber ff = new FFmpegFrameGrabber(url);
-//            ff.start();
-//            int lenght = ff.getLengthInFrames();
-//            int i = 0;
-//            Frame f = null;
-//            while (i < lenght) {
-//// 过滤前5帧,避免出现全黑的图片,依自己情况而定
-//                f = ff.grabFrame();
-//                if ((i > 5) && (f.image != null)) {
-//                    break;
-//                }
-//                i++;
-//            }
-////            opencv_core.IplImage img = f.image;
-//            int owidth = f.imageWidth;
-//            int oheight = f.imageHeight;
-//            Java2DFrameConverter converter =new Java2DFrameConverter();
-//            BufferedImage fecthedImage =converter.getBufferedImage(f);
-//// 对截取的帧进行等比例缩放
-//            int width = 800;
-//            int height = (int) (((double) width / owidth) * oheight);
-//            BufferedImage bi = new BufferedImage(width, height, BufferedImage.TYPE_3BYTE_BGR);
-//            bi.getGraphics().drawImage(fecthedImage.getScaledInstance(width, height, Image.SCALE_SMOOTH),
-//                    0, 0, null);
-//            ByteArrayOutputStream baos = new ByteArrayOutputStream();//io流
-//            ImageIO.write(bi, "jpg", baos);
-//            byte[] bytes = baos.toByteArray();//转换成字节
-//            String png_base64 = Base64.getEncoder().encodeToString(bytes).trim();//转换成base64串
-//            png_base64 = png_base64.replaceAll("\n", "").replaceAll("\r", "");//删除 \r\n
-//            //        ImageIO.write(bufferedImage, "png", new File("D:/qrcode1.png"));
-//            baos.close();
-//            ff.close();
-//            return "data:image/jpg;base64,"+png_base64;
-//        }catch (Exception e){
-//            e.printStackTrace();
-//            return null;
-//        }
-//
-//    }
-//    private void douyinHotHandler(int page,int listType){
-//        String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_video/user_side/list?page="+page+"&limit=20&list_type="+listType+"&aggr_duration_type=-1");
-//        ObjectMapper mapper = new ObjectMapper();
-//        try {
-//            JsonNode node = mapper.readTree(result);
-//            int code = node.get("code").asInt();
-//            if(code == 0){
-//                if(node.get("data")!= null){
-//                    String dataJson = node.get("data").get("items").toString();
-//                    List<DouyinHot> list = mapper.readValue(dataJson,new TypeReference<List<DouyinHot>>() {});
-//                    if (list != null && list.size() > 0){
-//                        for (DouyinHot douyinHot : list){
-//                            douyinHot.setUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+douyinHot.getVideoId()+"&line=0"));
-//                            if(douyinHot.getUrl() == null){
-//                                douyinHot.setUrl(getVideoUrl(douyinHot.getVideoId()));
-//                            }
-////                            douyinHot.setUrl(getVideoUrl(douyinHot.getVideoId()));
-//                            douyinHot.setCoverImage(getVideoFirstImage(douyinHot.getUrl()));
-//                            if(douyinHot.getCoverImage() == null){
-//                                Thread.sleep(1000);
-//                                douyinHot.setCoverImage(getVideoFirstImage(douyinHot.getUrl()));
-//                            }
-//                            douyinHotService.saveOrUpdate(douyinHot);
-//                        }
-//                        douyinHotHandler(page+1,listType);
-//                    }else {
-//                        if(listType < 4){
-//                            douyinHotHandler(1,listType+1);
-//                        }
-//                    }
-//                }
-//            }
-//        }catch (Exception e){
-//            e.printStackTrace();
-//        }
-//
-//    }
-//    @Override
-//    public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
-//        String account="3248395570@qq.com";
-//        String password = "Ydxq-704127411";
-//        login(account,password);
-//        douyinHotHandler(1,1);
-//        effectCaseHandler(1);
-//        hotMaterialHandler(1,1,"西瓜");
-//        hotMaterialHandler(1,3,"火山");
-//        hotMaterialHandler(1,4,"抖音");
-//        hotMaterialHandler(1,8,"头条");
-//        hotMaterialHandler(1,9,"穿山甲");
-//        log.info("巨量创意抓取完成");
-//    }
-//
-//    private void login(String account,String password){
-//        String url = "https://cc.oceanengine.com/login";
-//        System.getProperties().setProperty("webdriver.chrome.driver", chromeDriver);
-//        ChromeOptions chromeOptions = new ChromeOptions();
-//        chromeOptions.addArguments("--headless");
-//        chromeOptions.addArguments("--no-sandbox");
-//        chromeOptions.addArguments("--window-size=1920,1080");
-//        chromeOptions.addArguments("--user-agent=" + HttpUtils2.USER_AGENT);
-//        chromeOptions.setAcceptInsecureCerts(true);
-//        WebDriver webDriver = new ChromeDriver(chromeOptions);
-//        //清除所有的缓存
-//        webDriver.manage().deleteAllCookies();
-//        try {
-//            //获取登录页面
-//            webDriver.get(url);
-//            for (Cookie cookie : webDriver.manage().getCookies()) {
-//                BasicClientCookie ck = new BasicClientCookie(cookie.getName(), cookie.getValue());
-//                ck.setDomain("oceanengine.com");
-//                ck.setExpiryDate(cookie.getExpiry());
-//                ck.setPath(cookie.getPath());
-//                HttpUtils2.cookieStore.addCookie(ck);
-//            }
-//            Map<String, String> header = new HashMap<>();
-//            header.put("Accept", "*/*");
-//            header.put("Connection", "keep-alive");
-//            header.put("Host", "sso.toutiao.com");
-//            header.put("Content-Type", "application/x-www-form-urlencoded");
-//            header.put("Origin", "https://cc.oceanengine.com");
-//            header.put("Referer", "https://cc.oceanengine.com/login");
-//            Map<String, Object> param = new HashMap<>();
-//            param.put("mobile", "");
-//            param.put("code", "");
-//            param.put("account", account);
-//            param.put("password", password);
-//            param.put("captcha", "hqde");
-//            param.put("is_30_days_no_login", "true");
-//            param.put("service", "https://cc.oceanengine.com");
-//            String res = HttpUtils2.httpPostParamRequest("https://sso.toutiao.com/account_login/", param, header);
-//            ObjectMapper mapper = new ObjectMapper();
-//            mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-//            JsonNode jsonNode = mapper.readTree(res);
-//            //errorcode == 1102 验证码错误,请重新输入验证码
-//            //errorcode == 1101 验证码为空
-//
-//            Integer errorCode = jsonNode.get("error_code").asInt();
-//            String orderId = null;
-//            while (errorCode != 0) {
-//                //用户名或者密码错误
-//                if (null != errorCode && errorCode == 1009) {
-//                    return;
-//                }
-//                FateadmUtil fateadmUtil = new FateadmUtil();
-//                fateadmUtil.init();
-//                if (orderId != null) {
-//                    fateadmUtil.justice(orderId);
-//                }
-//                String captcha = jsonNode.get("captcha").asText();
-//                Base64.Decoder decoder = Base64.getDecoder();
-//                FateadmHttpUtil.HttpResp resp = fateadmUtil.Predict("30400", decoder.decode(captcha));
-//                param.put("captcha", resp.predResl);
-//                orderId = resp.reqId;
-//                res = HttpUtils2.httpPostParamRequest("https://sso.toutiao.com/account_login/", param, header);
-//                jsonNode = mapper.readTree(res);
-//                errorCode = jsonNode.get("error_code").asInt();
-//            }
-//            if (errorCode == 0) {
-//                //表示登录成功
-//                List<org.apache.http.cookie.Cookie> cookies = HttpUtils2.cookieStore.getCookies();
-//                for (org.apache.http.cookie.Cookie ck : cookies) {
-//                    Cookie cookie = new Cookie(ck.getName(), ck.getValue(), "." + ck.getDomain(), ck.getPath(), ck.getExpiryDate());
-//                    webDriver.manage().addCookie(cookie);
-//                }
-//                String redirectUrl = jsonNode.get("redirect_url").asText();
-//                webDriver.get(redirectUrl);
-//                //登录成功 进入推广页面
-//                webDriver.get("https://cc.oceanengine.com/inspiration/douyin-rank-list");
-//                Thread.sleep(3000);
-//                for (Cookie cookie : webDriver.manage().getCookies()) {
-//                    BasicClientCookie ck = new BasicClientCookie(cookie.getName(), cookie.getValue());
-//                    ck.setDomain("cc.oceanengine.com");
-//                    ck.setExpiryDate(cookie.getExpiry());
-//                    ck.setPath(cookie.getPath());
-//                    HttpUtils2.cookieStore.addCookie(ck);
-//                }
-//
-//            }
-//        } catch (Exception e) {
-//            e.printStackTrace();
-//        } finally {
-//            //获得cookie
-//            Set<Cookie> coo = webDriver.manage().getCookies();
-//            //清除所有的缓存
-//            webDriver.manage().deleteAllCookies();
-//            webDriver.quit();
-//        }
-//    }
-//}
+package cn.com.ctop.crawler.modules.oceanengine.service.impl;
+
+import cn.com.ctop.common.module.utils.Base64Utils;
+import cn.com.ctop.common.module.utils.HttpUtils2;
+import cn.com.ctop.crawler.modules.core.util.FateadmHttpUtil;
+import cn.com.ctop.crawler.modules.core.util.FateadmUtil;
+import cn.com.ctop.crawler.modules.oceanengine.entity.DouyinHot;
+import cn.com.ctop.crawler.modules.oceanengine.entity.EffectCase;
+import cn.com.ctop.crawler.modules.oceanengine.entity.HotMaterial;
+import cn.com.ctop.crawler.modules.oceanengine.service.IDouyinHotService;
+import cn.com.ctop.crawler.modules.oceanengine.service.IEffectCaseService;
+import cn.com.ctop.crawler.modules.oceanengine.service.IHotMaterialService;
+import cn.com.ctop.crawler.modules.oceanengine.service.IOceanEngineService;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.http.impl.cookie.BasicClientCookie;
+import org.bytedeco.javacv.FFmpegFrameGrabber;
+import org.bytedeco.javacv.Frame;
+import org.bytedeco.javacv.Java2DFrameConverter;
+import org.openqa.selenium.Cookie;
+import org.openqa.selenium.WebDriver;
+import org.openqa.selenium.chrome.ChromeDriver;
+import org.openqa.selenium.chrome.ChromeOptions;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+import javax.imageio.ImageIO;
+import java.awt.*;
+import java.awt.image.BufferedImage;
+import java.io.ByteArrayOutputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.List;
+import java.util.*;
+
+@Service
+public class OceanEngineServiceImpl implements IOceanEngineService {
+    @Value("${jeecg.path.chrome-driver}")
+    private String chromeDriver;
+    private String getVideoUrlByVid(String vid) {
+        String url = "https://aweme.snssdk.com/aweme/v1/playwm/?video_id=" + vid + "&line=0";
+        return getRealVideoUrl(url);
+    }
+
+    private String getRealVideoUrl(String url) {
+        try {
+            URL serverUrl = new URL(url);
+            HttpURLConnection conn = (HttpURLConnection) serverUrl
+                    .openConnection();
+            conn.setRequestMethod("GET");
+            // 必须设置false,否则会自动redirect到Location的地址
+            conn.setInstanceFollowRedirects(false);
+            conn.addRequestProperty("User-Agent",
+                    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.2.8) Firefox/3.6.8");
+            conn.connect();
+            String location = conn.getHeaderField("Location");
+            return location;
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+        return null;
+    }
+
+    @Override
+    public void hotMaterialHandler(int page, int appCode, String appName){
+        String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_material/list?material_type=3&order_by=convert_show_rate&period_type=3&aggr_app_code="+appCode+"&page="+page+"&limit=20");
+        ObjectMapper mapper = new ObjectMapper();
+        mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+        try {
+            JsonNode node = mapper.readTree(result);
+            int code = node.get("code").asInt();
+            if (code == 0) {
+                if (node.get("data") != null&&node.get("data").get("materials")!=null) {
+                    String dataJson = node.get("data").get("materials").toString();
+                    List<HotMaterial> list = mapper.readValue(dataJson,new TypeReference<List<HotMaterial>>() {});
+                    if(list != null && list.size() > 0){
+                        for (HotMaterial material : list){
+                            material.setAppCode(appCode);
+                            material.setAppName(appName);
+                            material.setVideoUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+material.getVid()+"&line=0"));
+                            if(material.getVideoUrl() == null){
+                                material.setVideoUrl(getVideoUrl(material.getVid()));
+                            }
+//                            material.setVideoUrl(getVideoUrl(material.getVid()));
+                            material.setCoverImage(getVideoFirstImage(material.getVideoUrl()));
+                            if(material.getCoverImage() == null){
+                                Thread.sleep(1000);
+                                material.setCoverImage(getVideoFirstImage(material.getVideoUrl()));
+                            }
+                            hotMaterialService.saveOrUpdate(material);
+                        }
+                        hotMaterialHandler(page+1,appCode,appName);
+                    }
+                }
+            }
+        }catch (Exception e){
+            e.printStackTrace();
+        }
+    }
+
+    @Autowired
+    private IHotMaterialService hotMaterialService;
+
+    private String getVideoUrl(String vid) throws Exception {
+        String url = "http://i.snssdk.com/video/urls/1/toutiao/mp4/"+vid;
+        String result = HttpUtils2.httpGetRequest(url);
+        ObjectMapper mapper = new ObjectMapper();
+        if(result != null){
+            JsonNode node = mapper.readTree(result);
+            String mainUrl = node.get("data").get("video_list").get("video_1").get("main_url").asText();
+            return Base64Utils.decode(mainUrl);
+        }
+        return null;
+    }
+
+    @Override
+    public void effectCaseHandler(int page){
+        String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/case/effect_case_list?industry=0&style=0&mode=0&page="+page+"&limit=20");
+        ObjectMapper mapper = new ObjectMapper();
+        mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+        try {
+            JsonNode node = mapper.readTree(result);
+            int code = node.get("code").asInt();
+            if(code == 0) {
+                if (node.get("data") != null) {
+                    String dataJson = node.get("data").get("effect_cases").toString();
+                    List<EffectCase> list = mapper.readValue(dataJson,new TypeReference<List<EffectCase>>() {});
+                    if (list != null && list.size() > 0) {
+                        for (EffectCase effectCase:list){
+                            String resultDetail = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/case/effect_case/"+effectCase.getId());
+                            JsonNode detailNode = mapper.readTree(resultDetail);
+                            int detailCode = detailNode.get("code").asInt();
+                            if(detailCode == 0){
+                                if(detailNode.get("data") != null){
+//                                    String detailJson = node.get("data").get("effect_cases").toString();
+//                                    EffectCase effectCase1 = mapper.readValue(detailJson,EffectCase.class);
+                                    effectCase.setAuthor(detailNode.get("data").get("effect_case").get("author").asText());
+                                    effectCase.setCreateDate(detailNode.get("data").get("effect_case").get("create_date").asText());
+                                    effectCase.setCreativeId(detailNode.get("data").get("effect_case").get("creative_id").asLong());
+                                    effectCase.setCreativeScore(detailNode.get("data").get("effect_case").get("creative_score").asText());
+                                    effectCase.setEffect(detailNode.get("data").get("effect_case").get("effect").toString());
+                                    effectCase.setMethods(detailNode.get("data").get("effect_case").get("methods").toString());
+                                    effectCase.setVideoUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+effectCase.getVid()+"&line=0"));
+                                    if (effectCase.getVideoUrl() == null){
+                                        effectCase.setVideoUrl(getVideoUrl(effectCase.getVid()));
+                                    }
+//                                    effectCase.setVideoUrl(getVideoUrl(effectCase.getVid()));
+                                    effectCase.setCoverImage(getVideoFirstImage(effectCase.getVideoUrl()));
+                                    if (effectCase.getCoverImage() == null){
+                                        Thread.sleep(1000);
+                                        effectCase.setCoverImage(getVideoFirstImage(effectCase.getVideoUrl()));
+                                    }
+                                    effectCase.setCreativeLabel(detailNode.get("data").get("effect_case").get("creative_label").toString());
+                                }
+                            }
+                            effectCaseService.saveOrUpdate(effectCase);
+                        }
+                        effectCaseHandler(page+1);
+                    }
+                }
+            }
+        }catch (Exception e){
+            e.printStackTrace();
+        }
+    }
+
+    @Autowired
+    private IEffectCaseService effectCaseService;
+
+    public String getVideoFirstImage(String url){
+        try {
+            FFmpegFrameGrabber ff = new FFmpegFrameGrabber(url);
+            ff.start();
+            int lenght = ff.getLengthInFrames();
+            int i = 0;
+            Frame f = null;
+            while (i < lenght) {
+// 过滤前5帧,避免出现全黑的图片,依自己情况而定
+                f = ff.grabFrame();
+                if ((i > 5) && (f.image != null)) {
+                    break;
+                }
+                i++;
+            }
+//            opencv_core.IplImage img = f.image;
+            int owidth = f.imageWidth;
+            int oheight = f.imageHeight;
+            Java2DFrameConverter converter =new Java2DFrameConverter();
+            BufferedImage fecthedImage =converter.getBufferedImage(f);
+// 对截取的帧进行等比例缩放
+            int width = 800;
+            int height = (int) (((double) width / owidth) * oheight);
+            BufferedImage bi = new BufferedImage(width, height, BufferedImage.TYPE_3BYTE_BGR);
+            bi.getGraphics().drawImage(fecthedImage.getScaledInstance(width, height, Image.SCALE_SMOOTH),
+                    0, 0, null);
+            ByteArrayOutputStream baos = new ByteArrayOutputStream();//io流
+            ImageIO.write(bi, "jpg", baos);
+            byte[] bytes = baos.toByteArray();//转换成字节
+            String png_base64 = Base64.getEncoder().encodeToString(bytes).trim();//转换成base64串
+            png_base64 = png_base64.replaceAll("\n", "").replaceAll("\r", "");//删除 \r\n
+            //        ImageIO.write(bufferedImage, "png", new File("D:/qrcode1.png"));
+            baos.close();
+            ff.close();
+            return "data:image/jpg;base64,"+png_base64;
+        }catch (Exception e){
+            e.printStackTrace();
+            return null;
+        }
+
+    }
+
+    @Autowired
+    private IDouyinHotService douyinHotService;
+    @Override
+    public void douyinHotHandler(int page, int listType){
+        String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_video/user_side/list?page="+page+"&limit=20&list_type="+listType+"&aggr_duration_type=-1");
+        ObjectMapper mapper = new ObjectMapper();
+        try {
+            JsonNode node = mapper.readTree(result);
+            int code = node.get("code").asInt();
+            if(code == 0){
+                if(node.get("data")!= null){
+                    String dataJson = node.get("data").get("items").toString();
+                    List<DouyinHot> list = mapper.readValue(dataJson,new TypeReference<List<DouyinHot>>() {});
+                    if (list != null && list.size() > 0){
+                        for (DouyinHot douyinHot : list){
+                            douyinHot.setUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+douyinHot.getVideoId()+"&line=0"));
+                            if(douyinHot.getUrl() == null){
+                                douyinHot.setUrl(getVideoUrl(douyinHot.getVideoId()));
+                            }
+                            douyinHot.setCoverImage(getVideoFirstImage(douyinHot.getUrl()));
+                            if(douyinHot.getCoverImage() == null){
+                                Thread.sleep(1000);
+                                douyinHot.setCoverImage(getVideoFirstImage(douyinHot.getUrl()));
+                            }
+                            douyinHotService.saveOrUpdate(douyinHot);
+                        }
+                        douyinHotHandler(page+1,listType);
+                    }else {
+                        if(listType < 4){
+                            douyinHotHandler(1,listType+1);
+                        }
+                    }
+                }
+            }
+        }catch (Exception e){
+            e.printStackTrace();
+        }
+    }
+
+    @Override
+    public void login(String account, String password){
+        String url = "https://cc.oceanengine.com/login";
+        System.getProperties().setProperty("webdriver.chrome.driver", chromeDriver);
+        ChromeOptions chromeOptions = new ChromeOptions();
+        chromeOptions.addArguments("--headless");
+        chromeOptions.addArguments("--no-sandbox");
+        chromeOptions.addArguments("--window-size=1920,1080");
+        chromeOptions.addArguments("--user-agent=" + HttpUtils2.USER_AGENT);
+        chromeOptions.setAcceptInsecureCerts(true);
+        WebDriver webDriver = new ChromeDriver(chromeOptions);
+        //清除所有的缓存
+        webDriver.manage().deleteAllCookies();
+        try {
+            //获取登录页面
+            webDriver.get(url);
+            for (Cookie cookie : webDriver.manage().getCookies()) {
+                BasicClientCookie ck = new BasicClientCookie(cookie.getName(), cookie.getValue());
+                ck.setDomain("oceanengine.com");
+                ck.setExpiryDate(cookie.getExpiry());
+                ck.setPath(cookie.getPath());
+                HttpUtils2.cookieStore.addCookie(ck);
+            }
+            Map<String, String> header = new HashMap<>();
+            header.put("Accept", "*/*");
+            header.put("Connection", "keep-alive");
+            header.put("Host", "sso.toutiao.com");
+            header.put("Content-Type", "application/x-www-form-urlencoded");
+            header.put("Origin", "https://cc.oceanengine.com");
+            header.put("Referer", "https://cc.oceanengine.com/login");
+            Map<String, Object> param = new HashMap<>();
+            param.put("mobile", "");
+            param.put("code", "");
+            param.put("account", account);
+            param.put("password", password);
+            param.put("captcha", "hqde");
+            param.put("is_30_days_no_login", "true");
+            param.put("service", "https://cc.oceanengine.com");
+            String res = HttpUtils2.httpPostParamRequest("https://sso.toutiao.com/account_login/", param, header);
+            ObjectMapper mapper = new ObjectMapper();
+            mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+            JsonNode jsonNode = mapper.readTree(res);
+            //errorcode == 1102 验证码错误,请重新输入验证码
+            //errorcode == 1101 验证码为空
+
+            Integer errorCode = jsonNode.get("error_code").asInt();
+            String orderId = null;
+            while (errorCode != 0) {
+                //用户名或者密码错误
+                if (null != errorCode && errorCode == 1009) {
+                    return;
+                }
+                FateadmUtil fateadmUtil = new FateadmUtil();
+                fateadmUtil.init();
+                if (orderId != null) {
+                    fateadmUtil.justice(orderId);
+                }
+                String captcha = jsonNode.get("captcha").asText();
+                Base64.Decoder decoder = Base64.getDecoder();
+                FateadmHttpUtil.HttpResp resp = fateadmUtil.Predict("30400", decoder.decode(captcha));
+                param.put("captcha", resp.predResl);
+                orderId = resp.reqId;
+                res = HttpUtils2.httpPostParamRequest("https://sso.toutiao.com/account_login/", param, header);
+                jsonNode = mapper.readTree(res);
+                errorCode = jsonNode.get("error_code").asInt();
+            }
+            if (errorCode == 0) {
+                //表示登录成功
+                List<org.apache.http.cookie.Cookie> cookies = HttpUtils2.cookieStore.getCookies();
+                for (org.apache.http.cookie.Cookie ck : cookies) {
+                    Cookie cookie = new Cookie(ck.getName(), ck.getValue(), "." + ck.getDomain(), ck.getPath(), ck.getExpiryDate());
+                    webDriver.manage().addCookie(cookie);
+                }
+                String redirectUrl = jsonNode.get("redirect_url").asText();
+                webDriver.get(redirectUrl);
+                //登录成功 进入推广页面
+                webDriver.get("https://cc.oceanengine.com/inspiration/douyin-rank-list");
+                Thread.sleep(3000);
+                for (Cookie cookie : webDriver.manage().getCookies()) {
+                    BasicClientCookie ck = new BasicClientCookie(cookie.getName(), cookie.getValue());
+                    ck.setDomain("cc.oceanengine.com");
+                    ck.setExpiryDate(cookie.getExpiry());
+                    ck.setPath(cookie.getPath());
+                    HttpUtils2.cookieStore.addCookie(ck);
+                }
+
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        } finally {
+            //获得cookie
+            Set<Cookie> coo = webDriver.manage().getCookies();
+            //清除所有的缓存
+            webDriver.manage().deleteAllCookies();
+            webDriver.quit();
+        }
+    }
+}