|
@@ -1,347 +1,351 @@
|
|
|
-//package cn.com.ctop.crawler.modules.oceanengine.service.impl;
|
|
|
-//
|
|
|
-//import cn.com.ctop.common.module.utils.Base64Utils;
|
|
|
-//import cn.com.ctop.common.module.utils.HttpUtils2;
|
|
|
-//import cn.com.ctop.crawler.modules.core.util.FateadmHttpUtil;
|
|
|
-//import cn.com.ctop.crawler.modules.core.util.FateadmUtil;
|
|
|
-//import cn.com.ctop.crawler.modules.oceanengine.entity.DouyinHot;
|
|
|
-//import cn.com.ctop.crawler.modules.oceanengine.entity.EffectCase;
|
|
|
-//import cn.com.ctop.crawler.modules.oceanengine.entity.HotMaterial;
|
|
|
-//import cn.com.ctop.crawler.modules.oceanengine.service.IOceanEngineService;
|
|
|
-//import com.fasterxml.jackson.core.type.TypeReference;
|
|
|
-//import com.fasterxml.jackson.databind.DeserializationFeature;
|
|
|
-//import com.fasterxml.jackson.databind.JsonNode;
|
|
|
-//import com.fasterxml.jackson.databind.ObjectMapper;
|
|
|
-//import org.apache.http.impl.cookie.BasicClientCookie;
|
|
|
-//import org.bytedeco.javacv.FFmpegFrameGrabber;
|
|
|
-//import org.bytedeco.javacv.Frame;
|
|
|
-//import org.bytedeco.javacv.Java2DFrameConverter;
|
|
|
-//import org.openqa.selenium.Cookie;
|
|
|
-//import org.openqa.selenium.WebDriver;
|
|
|
-//import org.openqa.selenium.chrome.ChromeDriver;
|
|
|
-//import org.openqa.selenium.chrome.ChromeOptions;
|
|
|
-//import org.quartz.JobExecutionContext;
|
|
|
-//import org.quartz.JobExecutionException;
|
|
|
-//import org.springframework.stereotype.Service;
|
|
|
-//
|
|
|
-//import javax.imageio.ImageIO;
|
|
|
-//import java.awt.*;
|
|
|
-//import java.awt.image.BufferedImage;
|
|
|
-//import java.io.ByteArrayOutputStream;
|
|
|
-//import java.net.HttpURLConnection;
|
|
|
-//import java.net.URL;
|
|
|
-//import java.util.*;
|
|
|
-//import java.util.List;
|
|
|
-//
|
|
|
-//@Service
|
|
|
-//public class OceanEngineServiceImpl implements IOceanEngineService {
|
|
|
-// public static String getVideoUrlByVid(String vid) {
|
|
|
-// String url = "https://aweme.snssdk.com/aweme/v1/playwm/?video_id=" + vid + "&line=0";
|
|
|
-// return getRealVideoUrl(url);
|
|
|
-// }
|
|
|
-//
|
|
|
-// public static String getRealVideoUrl(String url) {
|
|
|
-// try {
|
|
|
-// URL serverUrl = new URL(url);
|
|
|
-// HttpURLConnection conn = (HttpURLConnection) serverUrl
|
|
|
-// .openConnection();
|
|
|
-// conn.setRequestMethod("GET");
|
|
|
-// // 必须设置false,否则会自动redirect到Location的地址
|
|
|
-// conn.setInstanceFollowRedirects(false);
|
|
|
-// conn.addRequestProperty("User-Agent",
|
|
|
-// "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.2.8) Firefox/3.6.8");
|
|
|
-// conn.connect();
|
|
|
-// String location = conn.getHeaderField("Location");
|
|
|
-// return location;
|
|
|
-// } catch (Exception e) {
|
|
|
-// e.printStackTrace();
|
|
|
-// }
|
|
|
-// return null;
|
|
|
-// }
|
|
|
-// private void hotMaterialHandler(int page,int appCode,String appName){
|
|
|
-// String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_material/list?material_type=3&order_by=convert_show_rate&period_type=3&aggr_app_code="+appCode+"&page="+page+"&limit=20");
|
|
|
-// ObjectMapper mapper = new ObjectMapper();
|
|
|
-// mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
|
|
-// try {
|
|
|
-// JsonNode node = mapper.readTree(result);
|
|
|
-// int code = node.get("code").asInt();
|
|
|
-// if (code == 0) {
|
|
|
-// if (node.get("data") != null&&node.get("data").get("materials")!=null) {
|
|
|
-// String dataJson = node.get("data").get("materials").toString();
|
|
|
-// List<HotMaterial> list = mapper.readValue(dataJson,new TypeReference<List<HotMaterial>>() {});
|
|
|
-// if(list != null && list.size() > 0){
|
|
|
-// for (HotMaterial material : list){
|
|
|
-// material.setAppCode(appCode);
|
|
|
-// material.setAppName(appName);
|
|
|
-// material.setVideoUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+material.getVid()+"&line=0"));
|
|
|
-// if(material.getVideoUrl() == null){
|
|
|
-// material.setVideoUrl(getVideoUrl(material.getVid()));
|
|
|
-// }
|
|
|
-//// material.setVideoUrl(getVideoUrl(material.getVid()));
|
|
|
-// material.setCoverImage(getVideoFirstImage(material.getVideoUrl()));
|
|
|
-// if(material.getCoverImage() == null){
|
|
|
-// Thread.sleep(1000);
|
|
|
-// material.setCoverImage(getVideoFirstImage(material.getVideoUrl()));
|
|
|
-// }
|
|
|
-// hotMaterialService.saveOrUpdate(material);
|
|
|
-// }
|
|
|
-// hotMaterialHandler(page+1,appCode,appName);
|
|
|
-// }
|
|
|
-// }
|
|
|
-// }
|
|
|
-// }catch (Exception e){
|
|
|
-// e.printStackTrace();
|
|
|
-// }
|
|
|
-// }
|
|
|
-//
|
|
|
-// public static String getVideoUrl(String vid) throws Exception {
|
|
|
-// String url = "http://i.snssdk.com/video/urls/1/toutiao/mp4/"+vid;
|
|
|
-// String result = HttpUtils2.httpGetRequest(url);
|
|
|
-// ObjectMapper mapper = new ObjectMapper();
|
|
|
-// if(result != null){
|
|
|
-// JsonNode node = mapper.readTree(result);
|
|
|
-// String mainUrl = node.get("data").get("video_list").get("video_1").get("main_url").asText();
|
|
|
-// return Base64Utils.decode(mainUrl);
|
|
|
-// }
|
|
|
-// return null;
|
|
|
-// }
|
|
|
-// private void effectCaseHandler(int page){
|
|
|
-// String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/case/effect_case_list?industry=0&style=0&mode=0&page="+page+"&limit=20");
|
|
|
-// ObjectMapper mapper = new ObjectMapper();
|
|
|
-// mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
|
|
-// try {
|
|
|
-// JsonNode node = mapper.readTree(result);
|
|
|
-// int code = node.get("code").asInt();
|
|
|
-// if(code == 0) {
|
|
|
-// if (node.get("data") != null) {
|
|
|
-// String dataJson = node.get("data").get("effect_cases").toString();
|
|
|
-// List<EffectCase> list = mapper.readValue(dataJson,new TypeReference<List<EffectCase>>() {});
|
|
|
-// if (list != null && list.size() > 0) {
|
|
|
-// for (EffectCase effectCase:list){
|
|
|
-// String resultDetail = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/case/effect_case/"+effectCase.getId());
|
|
|
-// JsonNode detailNode = mapper.readTree(resultDetail);
|
|
|
-// int detailCode = detailNode.get("code").asInt();
|
|
|
-// if(detailCode == 0){
|
|
|
-// if(detailNode.get("data") != null){
|
|
|
-//// String detailJson = node.get("data").get("effect_cases").toString();
|
|
|
-//// EffectCase effectCase1 = mapper.readValue(detailJson,EffectCase.class);
|
|
|
-// effectCase.setAuthor(detailNode.get("data").get("effect_case").get("author").asText());
|
|
|
-// effectCase.setCreateDate(detailNode.get("data").get("effect_case").get("create_date").asText());
|
|
|
-// effectCase.setCreativeId(detailNode.get("data").get("effect_case").get("creative_id").asLong());
|
|
|
-// effectCase.setCreativeScore(detailNode.get("data").get("effect_case").get("creative_score").asText());
|
|
|
-// effectCase.setEffect(detailNode.get("data").get("effect_case").get("effect").toString());
|
|
|
-// effectCase.setMethods(detailNode.get("data").get("effect_case").get("methods").toString());
|
|
|
-// effectCase.setVideoUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+effectCase.getVid()+"&line=0"));
|
|
|
-// if (effectCase.getVideoUrl() == null){
|
|
|
-// effectCase.setVideoUrl(getVideoUrl(effectCase.getVid()));
|
|
|
-// }
|
|
|
-//// effectCase.setVideoUrl(getVideoUrl(effectCase.getVid()));
|
|
|
-// effectCase.setCoverImage(getVideoFirstImage(effectCase.getVideoUrl()));
|
|
|
-// if (effectCase.getCoverImage() == null){
|
|
|
-// Thread.sleep(1000);
|
|
|
-// effectCase.setCoverImage(getVideoFirstImage(effectCase.getVideoUrl()));
|
|
|
-// }
|
|
|
-// effectCase.setCreativeLabel(detailNode.get("data").get("effect_case").get("creative_label").toString());
|
|
|
-// }
|
|
|
-// }
|
|
|
-// effectCaseService.saveOrUpdate(effectCase);
|
|
|
-// }
|
|
|
-// effectCaseHandler(page+1);
|
|
|
-// }
|
|
|
-// }
|
|
|
-// }
|
|
|
-// }catch (Exception e){
|
|
|
-// e.printStackTrace();
|
|
|
-// }
|
|
|
-// }
|
|
|
-//
|
|
|
-// private String getVideoFirstImage(String url){
|
|
|
-// try {
|
|
|
-// FFmpegFrameGrabber ff = new FFmpegFrameGrabber(url);
|
|
|
-// ff.start();
|
|
|
-// int lenght = ff.getLengthInFrames();
|
|
|
-// int i = 0;
|
|
|
-// Frame f = null;
|
|
|
-// while (i < lenght) {
|
|
|
-//// 过滤前5帧,避免出现全黑的图片,依自己情况而定
|
|
|
-// f = ff.grabFrame();
|
|
|
-// if ((i > 5) && (f.image != null)) {
|
|
|
-// break;
|
|
|
-// }
|
|
|
-// i++;
|
|
|
-// }
|
|
|
-//// opencv_core.IplImage img = f.image;
|
|
|
-// int owidth = f.imageWidth;
|
|
|
-// int oheight = f.imageHeight;
|
|
|
-// Java2DFrameConverter converter =new Java2DFrameConverter();
|
|
|
-// BufferedImage fecthedImage =converter.getBufferedImage(f);
|
|
|
-//// 对截取的帧进行等比例缩放
|
|
|
-// int width = 800;
|
|
|
-// int height = (int) (((double) width / owidth) * oheight);
|
|
|
-// BufferedImage bi = new BufferedImage(width, height, BufferedImage.TYPE_3BYTE_BGR);
|
|
|
-// bi.getGraphics().drawImage(fecthedImage.getScaledInstance(width, height, Image.SCALE_SMOOTH),
|
|
|
-// 0, 0, null);
|
|
|
-// ByteArrayOutputStream baos = new ByteArrayOutputStream();//io流
|
|
|
-// ImageIO.write(bi, "jpg", baos);
|
|
|
-// byte[] bytes = baos.toByteArray();//转换成字节
|
|
|
-// String png_base64 = Base64.getEncoder().encodeToString(bytes).trim();//转换成base64串
|
|
|
-// png_base64 = png_base64.replaceAll("\n", "").replaceAll("\r", "");//删除 \r\n
|
|
|
-// // ImageIO.write(bufferedImage, "png", new File("D:/qrcode1.png"));
|
|
|
-// baos.close();
|
|
|
-// ff.close();
|
|
|
-// return "data:image/jpg;base64,"+png_base64;
|
|
|
-// }catch (Exception e){
|
|
|
-// e.printStackTrace();
|
|
|
-// return null;
|
|
|
-// }
|
|
|
-//
|
|
|
-// }
|
|
|
-// private void douyinHotHandler(int page,int listType){
|
|
|
-// String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_video/user_side/list?page="+page+"&limit=20&list_type="+listType+"&aggr_duration_type=-1");
|
|
|
-// ObjectMapper mapper = new ObjectMapper();
|
|
|
-// try {
|
|
|
-// JsonNode node = mapper.readTree(result);
|
|
|
-// int code = node.get("code").asInt();
|
|
|
-// if(code == 0){
|
|
|
-// if(node.get("data")!= null){
|
|
|
-// String dataJson = node.get("data").get("items").toString();
|
|
|
-// List<DouyinHot> list = mapper.readValue(dataJson,new TypeReference<List<DouyinHot>>() {});
|
|
|
-// if (list != null && list.size() > 0){
|
|
|
-// for (DouyinHot douyinHot : list){
|
|
|
-// douyinHot.setUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+douyinHot.getVideoId()+"&line=0"));
|
|
|
-// if(douyinHot.getUrl() == null){
|
|
|
-// douyinHot.setUrl(getVideoUrl(douyinHot.getVideoId()));
|
|
|
-// }
|
|
|
-//// douyinHot.setUrl(getVideoUrl(douyinHot.getVideoId()));
|
|
|
-// douyinHot.setCoverImage(getVideoFirstImage(douyinHot.getUrl()));
|
|
|
-// if(douyinHot.getCoverImage() == null){
|
|
|
-// Thread.sleep(1000);
|
|
|
-// douyinHot.setCoverImage(getVideoFirstImage(douyinHot.getUrl()));
|
|
|
-// }
|
|
|
-// douyinHotService.saveOrUpdate(douyinHot);
|
|
|
-// }
|
|
|
-// douyinHotHandler(page+1,listType);
|
|
|
-// }else {
|
|
|
-// if(listType < 4){
|
|
|
-// douyinHotHandler(1,listType+1);
|
|
|
-// }
|
|
|
-// }
|
|
|
-// }
|
|
|
-// }
|
|
|
-// }catch (Exception e){
|
|
|
-// e.printStackTrace();
|
|
|
-// }
|
|
|
-//
|
|
|
-// }
|
|
|
-// @Override
|
|
|
-// public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
|
|
|
-// String account="3248395570@qq.com";
|
|
|
-// String password = "Ydxq-704127411";
|
|
|
-// login(account,password);
|
|
|
-// douyinHotHandler(1,1);
|
|
|
-// effectCaseHandler(1);
|
|
|
-// hotMaterialHandler(1,1,"西瓜");
|
|
|
-// hotMaterialHandler(1,3,"火山");
|
|
|
-// hotMaterialHandler(1,4,"抖音");
|
|
|
-// hotMaterialHandler(1,8,"头条");
|
|
|
-// hotMaterialHandler(1,9,"穿山甲");
|
|
|
-// log.info("巨量创意抓取完成");
|
|
|
-// }
|
|
|
-//
|
|
|
-// private void login(String account,String password){
|
|
|
-// String url = "https://cc.oceanengine.com/login";
|
|
|
-// System.getProperties().setProperty("webdriver.chrome.driver", chromeDriver);
|
|
|
-// ChromeOptions chromeOptions = new ChromeOptions();
|
|
|
-// chromeOptions.addArguments("--headless");
|
|
|
-// chromeOptions.addArguments("--no-sandbox");
|
|
|
-// chromeOptions.addArguments("--window-size=1920,1080");
|
|
|
-// chromeOptions.addArguments("--user-agent=" + HttpUtils2.USER_AGENT);
|
|
|
-// chromeOptions.setAcceptInsecureCerts(true);
|
|
|
-// WebDriver webDriver = new ChromeDriver(chromeOptions);
|
|
|
-// //清除所有的缓存
|
|
|
-// webDriver.manage().deleteAllCookies();
|
|
|
-// try {
|
|
|
-// //获取登录页面
|
|
|
-// webDriver.get(url);
|
|
|
-// for (Cookie cookie : webDriver.manage().getCookies()) {
|
|
|
-// BasicClientCookie ck = new BasicClientCookie(cookie.getName(), cookie.getValue());
|
|
|
-// ck.setDomain("oceanengine.com");
|
|
|
-// ck.setExpiryDate(cookie.getExpiry());
|
|
|
-// ck.setPath(cookie.getPath());
|
|
|
-// HttpUtils2.cookieStore.addCookie(ck);
|
|
|
-// }
|
|
|
-// Map<String, String> header = new HashMap<>();
|
|
|
-// header.put("Accept", "*/*");
|
|
|
-// header.put("Connection", "keep-alive");
|
|
|
-// header.put("Host", "sso.toutiao.com");
|
|
|
-// header.put("Content-Type", "application/x-www-form-urlencoded");
|
|
|
-// header.put("Origin", "https://cc.oceanengine.com");
|
|
|
-// header.put("Referer", "https://cc.oceanengine.com/login");
|
|
|
-// Map<String, Object> param = new HashMap<>();
|
|
|
-// param.put("mobile", "");
|
|
|
-// param.put("code", "");
|
|
|
-// param.put("account", account);
|
|
|
-// param.put("password", password);
|
|
|
-// param.put("captcha", "hqde");
|
|
|
-// param.put("is_30_days_no_login", "true");
|
|
|
-// param.put("service", "https://cc.oceanengine.com");
|
|
|
-// String res = HttpUtils2.httpPostParamRequest("https://sso.toutiao.com/account_login/", param, header);
|
|
|
-// ObjectMapper mapper = new ObjectMapper();
|
|
|
-// mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
|
|
-// JsonNode jsonNode = mapper.readTree(res);
|
|
|
-// //errorcode == 1102 验证码错误,请重新输入验证码
|
|
|
-// //errorcode == 1101 验证码为空
|
|
|
-//
|
|
|
-// Integer errorCode = jsonNode.get("error_code").asInt();
|
|
|
-// String orderId = null;
|
|
|
-// while (errorCode != 0) {
|
|
|
-// //用户名或者密码错误
|
|
|
-// if (null != errorCode && errorCode == 1009) {
|
|
|
-// return;
|
|
|
-// }
|
|
|
-// FateadmUtil fateadmUtil = new FateadmUtil();
|
|
|
-// fateadmUtil.init();
|
|
|
-// if (orderId != null) {
|
|
|
-// fateadmUtil.justice(orderId);
|
|
|
-// }
|
|
|
-// String captcha = jsonNode.get("captcha").asText();
|
|
|
-// Base64.Decoder decoder = Base64.getDecoder();
|
|
|
-// FateadmHttpUtil.HttpResp resp = fateadmUtil.Predict("30400", decoder.decode(captcha));
|
|
|
-// param.put("captcha", resp.predResl);
|
|
|
-// orderId = resp.reqId;
|
|
|
-// res = HttpUtils2.httpPostParamRequest("https://sso.toutiao.com/account_login/", param, header);
|
|
|
-// jsonNode = mapper.readTree(res);
|
|
|
-// errorCode = jsonNode.get("error_code").asInt();
|
|
|
-// }
|
|
|
-// if (errorCode == 0) {
|
|
|
-// //表示登录成功
|
|
|
-// List<org.apache.http.cookie.Cookie> cookies = HttpUtils2.cookieStore.getCookies();
|
|
|
-// for (org.apache.http.cookie.Cookie ck : cookies) {
|
|
|
-// Cookie cookie = new Cookie(ck.getName(), ck.getValue(), "." + ck.getDomain(), ck.getPath(), ck.getExpiryDate());
|
|
|
-// webDriver.manage().addCookie(cookie);
|
|
|
-// }
|
|
|
-// String redirectUrl = jsonNode.get("redirect_url").asText();
|
|
|
-// webDriver.get(redirectUrl);
|
|
|
-// //登录成功 进入推广页面
|
|
|
-// webDriver.get("https://cc.oceanengine.com/inspiration/douyin-rank-list");
|
|
|
-// Thread.sleep(3000);
|
|
|
-// for (Cookie cookie : webDriver.manage().getCookies()) {
|
|
|
-// BasicClientCookie ck = new BasicClientCookie(cookie.getName(), cookie.getValue());
|
|
|
-// ck.setDomain("cc.oceanengine.com");
|
|
|
-// ck.setExpiryDate(cookie.getExpiry());
|
|
|
-// ck.setPath(cookie.getPath());
|
|
|
-// HttpUtils2.cookieStore.addCookie(ck);
|
|
|
-// }
|
|
|
-//
|
|
|
-// }
|
|
|
-// } catch (Exception e) {
|
|
|
-// e.printStackTrace();
|
|
|
-// } finally {
|
|
|
-// //获得cookie
|
|
|
-// Set<Cookie> coo = webDriver.manage().getCookies();
|
|
|
-// //清除所有的缓存
|
|
|
-// webDriver.manage().deleteAllCookies();
|
|
|
-// webDriver.quit();
|
|
|
-// }
|
|
|
-// }
|
|
|
-//}
|
|
|
+package cn.com.ctop.crawler.modules.oceanengine.service.impl;
|
|
|
+
|
|
|
+import cn.com.ctop.common.module.utils.Base64Utils;
|
|
|
+import cn.com.ctop.common.module.utils.HttpUtils2;
|
|
|
+import cn.com.ctop.crawler.modules.core.util.FateadmHttpUtil;
|
|
|
+import cn.com.ctop.crawler.modules.core.util.FateadmUtil;
|
|
|
+import cn.com.ctop.crawler.modules.oceanengine.entity.DouyinHot;
|
|
|
+import cn.com.ctop.crawler.modules.oceanengine.entity.EffectCase;
|
|
|
+import cn.com.ctop.crawler.modules.oceanengine.entity.HotMaterial;
|
|
|
+import cn.com.ctop.crawler.modules.oceanengine.service.IDouyinHotService;
|
|
|
+import cn.com.ctop.crawler.modules.oceanengine.service.IEffectCaseService;
|
|
|
+import cn.com.ctop.crawler.modules.oceanengine.service.IHotMaterialService;
|
|
|
+import cn.com.ctop.crawler.modules.oceanengine.service.IOceanEngineService;
|
|
|
+import com.fasterxml.jackson.core.type.TypeReference;
|
|
|
+import com.fasterxml.jackson.databind.DeserializationFeature;
|
|
|
+import com.fasterxml.jackson.databind.JsonNode;
|
|
|
+import com.fasterxml.jackson.databind.ObjectMapper;
|
|
|
+import org.apache.http.impl.cookie.BasicClientCookie;
|
|
|
+import org.bytedeco.javacv.FFmpegFrameGrabber;
|
|
|
+import org.bytedeco.javacv.Frame;
|
|
|
+import org.bytedeco.javacv.Java2DFrameConverter;
|
|
|
+import org.openqa.selenium.Cookie;
|
|
|
+import org.openqa.selenium.WebDriver;
|
|
|
+import org.openqa.selenium.chrome.ChromeDriver;
|
|
|
+import org.openqa.selenium.chrome.ChromeOptions;
|
|
|
+import org.springframework.beans.factory.annotation.Autowired;
|
|
|
+import org.springframework.beans.factory.annotation.Value;
|
|
|
+import org.springframework.stereotype.Service;
|
|
|
+
|
|
|
+import javax.imageio.ImageIO;
|
|
|
+import java.awt.*;
|
|
|
+import java.awt.image.BufferedImage;
|
|
|
+import java.io.ByteArrayOutputStream;
|
|
|
+import java.net.HttpURLConnection;
|
|
|
+import java.net.URL;
|
|
|
+import java.util.List;
|
|
|
+import java.util.*;
|
|
|
+
|
|
|
+@Service
|
|
|
+public class OceanEngineServiceImpl implements IOceanEngineService {
|
|
|
+ @Value("${jeecg.path.chrome-driver}")
|
|
|
+ private String chromeDriver;
|
|
|
+ private String getVideoUrlByVid(String vid) {
|
|
|
+ String url = "https://aweme.snssdk.com/aweme/v1/playwm/?video_id=" + vid + "&line=0";
|
|
|
+ return getRealVideoUrl(url);
|
|
|
+ }
|
|
|
+
|
|
|
+ private String getRealVideoUrl(String url) {
|
|
|
+ try {
|
|
|
+ URL serverUrl = new URL(url);
|
|
|
+ HttpURLConnection conn = (HttpURLConnection) serverUrl
|
|
|
+ .openConnection();
|
|
|
+ conn.setRequestMethod("GET");
|
|
|
+ // 必须设置false,否则会自动redirect到Location的地址
|
|
|
+ conn.setInstanceFollowRedirects(false);
|
|
|
+ conn.addRequestProperty("User-Agent",
|
|
|
+ "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.2.8) Firefox/3.6.8");
|
|
|
+ conn.connect();
|
|
|
+ String location = conn.getHeaderField("Location");
|
|
|
+ return location;
|
|
|
+ } catch (Exception e) {
|
|
|
+ e.printStackTrace();
|
|
|
+ }
|
|
|
+ return null;
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public void hotMaterialHandler(int page, int appCode, String appName){
|
|
|
+ String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_material/list?material_type=3&order_by=convert_show_rate&period_type=3&aggr_app_code="+appCode+"&page="+page+"&limit=20");
|
|
|
+ ObjectMapper mapper = new ObjectMapper();
|
|
|
+ mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
|
|
+ try {
|
|
|
+ JsonNode node = mapper.readTree(result);
|
|
|
+ int code = node.get("code").asInt();
|
|
|
+ if (code == 0) {
|
|
|
+ if (node.get("data") != null&&node.get("data").get("materials")!=null) {
|
|
|
+ String dataJson = node.get("data").get("materials").toString();
|
|
|
+ List<HotMaterial> list = mapper.readValue(dataJson,new TypeReference<List<HotMaterial>>() {});
|
|
|
+ if(list != null && list.size() > 0){
|
|
|
+ for (HotMaterial material : list){
|
|
|
+ material.setAppCode(appCode);
|
|
|
+ material.setAppName(appName);
|
|
|
+ material.setVideoUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+material.getVid()+"&line=0"));
|
|
|
+ if(material.getVideoUrl() == null){
|
|
|
+ material.setVideoUrl(getVideoUrl(material.getVid()));
|
|
|
+ }
|
|
|
+// material.setVideoUrl(getVideoUrl(material.getVid()));
|
|
|
+ material.setCoverImage(getVideoFirstImage(material.getVideoUrl()));
|
|
|
+ if(material.getCoverImage() == null){
|
|
|
+ Thread.sleep(1000);
|
|
|
+ material.setCoverImage(getVideoFirstImage(material.getVideoUrl()));
|
|
|
+ }
|
|
|
+ hotMaterialService.saveOrUpdate(material);
|
|
|
+ }
|
|
|
+ hotMaterialHandler(page+1,appCode,appName);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }catch (Exception e){
|
|
|
+ e.printStackTrace();
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ @Autowired
|
|
|
+ private IHotMaterialService hotMaterialService;
|
|
|
+
|
|
|
+ private String getVideoUrl(String vid) throws Exception {
|
|
|
+ String url = "http://i.snssdk.com/video/urls/1/toutiao/mp4/"+vid;
|
|
|
+ String result = HttpUtils2.httpGetRequest(url);
|
|
|
+ ObjectMapper mapper = new ObjectMapper();
|
|
|
+ if(result != null){
|
|
|
+ JsonNode node = mapper.readTree(result);
|
|
|
+ String mainUrl = node.get("data").get("video_list").get("video_1").get("main_url").asText();
|
|
|
+ return Base64Utils.decode(mainUrl);
|
|
|
+ }
|
|
|
+ return null;
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public void effectCaseHandler(int page){
|
|
|
+ String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/case/effect_case_list?industry=0&style=0&mode=0&page="+page+"&limit=20");
|
|
|
+ ObjectMapper mapper = new ObjectMapper();
|
|
|
+ mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
|
|
+ try {
|
|
|
+ JsonNode node = mapper.readTree(result);
|
|
|
+ int code = node.get("code").asInt();
|
|
|
+ if(code == 0) {
|
|
|
+ if (node.get("data") != null) {
|
|
|
+ String dataJson = node.get("data").get("effect_cases").toString();
|
|
|
+ List<EffectCase> list = mapper.readValue(dataJson,new TypeReference<List<EffectCase>>() {});
|
|
|
+ if (list != null && list.size() > 0) {
|
|
|
+ for (EffectCase effectCase:list){
|
|
|
+ String resultDetail = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/case/effect_case/"+effectCase.getId());
|
|
|
+ JsonNode detailNode = mapper.readTree(resultDetail);
|
|
|
+ int detailCode = detailNode.get("code").asInt();
|
|
|
+ if(detailCode == 0){
|
|
|
+ if(detailNode.get("data") != null){
|
|
|
+// String detailJson = node.get("data").get("effect_cases").toString();
|
|
|
+// EffectCase effectCase1 = mapper.readValue(detailJson,EffectCase.class);
|
|
|
+ effectCase.setAuthor(detailNode.get("data").get("effect_case").get("author").asText());
|
|
|
+ effectCase.setCreateDate(detailNode.get("data").get("effect_case").get("create_date").asText());
|
|
|
+ effectCase.setCreativeId(detailNode.get("data").get("effect_case").get("creative_id").asLong());
|
|
|
+ effectCase.setCreativeScore(detailNode.get("data").get("effect_case").get("creative_score").asText());
|
|
|
+ effectCase.setEffect(detailNode.get("data").get("effect_case").get("effect").toString());
|
|
|
+ effectCase.setMethods(detailNode.get("data").get("effect_case").get("methods").toString());
|
|
|
+ effectCase.setVideoUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+effectCase.getVid()+"&line=0"));
|
|
|
+ if (effectCase.getVideoUrl() == null){
|
|
|
+ effectCase.setVideoUrl(getVideoUrl(effectCase.getVid()));
|
|
|
+ }
|
|
|
+// effectCase.setVideoUrl(getVideoUrl(effectCase.getVid()));
|
|
|
+ effectCase.setCoverImage(getVideoFirstImage(effectCase.getVideoUrl()));
|
|
|
+ if (effectCase.getCoverImage() == null){
|
|
|
+ Thread.sleep(1000);
|
|
|
+ effectCase.setCoverImage(getVideoFirstImage(effectCase.getVideoUrl()));
|
|
|
+ }
|
|
|
+ effectCase.setCreativeLabel(detailNode.get("data").get("effect_case").get("creative_label").toString());
|
|
|
+ }
|
|
|
+ }
|
|
|
+ effectCaseService.saveOrUpdate(effectCase);
|
|
|
+ }
|
|
|
+ effectCaseHandler(page+1);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }catch (Exception e){
|
|
|
+ e.printStackTrace();
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ @Autowired
|
|
|
+ private IEffectCaseService effectCaseService;
|
|
|
+
|
|
|
+ public String getVideoFirstImage(String url){
|
|
|
+ try {
|
|
|
+ FFmpegFrameGrabber ff = new FFmpegFrameGrabber(url);
|
|
|
+ ff.start();
|
|
|
+ int lenght = ff.getLengthInFrames();
|
|
|
+ int i = 0;
|
|
|
+ Frame f = null;
|
|
|
+ while (i < lenght) {
|
|
|
+// 过滤前5帧,避免出现全黑的图片,依自己情况而定
|
|
|
+ f = ff.grabFrame();
|
|
|
+ if ((i > 5) && (f.image != null)) {
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ i++;
|
|
|
+ }
|
|
|
+// opencv_core.IplImage img = f.image;
|
|
|
+ int owidth = f.imageWidth;
|
|
|
+ int oheight = f.imageHeight;
|
|
|
+ Java2DFrameConverter converter =new Java2DFrameConverter();
|
|
|
+ BufferedImage fecthedImage =converter.getBufferedImage(f);
|
|
|
+// 对截取的帧进行等比例缩放
|
|
|
+ int width = 800;
|
|
|
+ int height = (int) (((double) width / owidth) * oheight);
|
|
|
+ BufferedImage bi = new BufferedImage(width, height, BufferedImage.TYPE_3BYTE_BGR);
|
|
|
+ bi.getGraphics().drawImage(fecthedImage.getScaledInstance(width, height, Image.SCALE_SMOOTH),
|
|
|
+ 0, 0, null);
|
|
|
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();//io流
|
|
|
+ ImageIO.write(bi, "jpg", baos);
|
|
|
+ byte[] bytes = baos.toByteArray();//转换成字节
|
|
|
+ String png_base64 = Base64.getEncoder().encodeToString(bytes).trim();//转换成base64串
|
|
|
+ png_base64 = png_base64.replaceAll("\n", "").replaceAll("\r", "");//删除 \r\n
|
|
|
+ // ImageIO.write(bufferedImage, "png", new File("D:/qrcode1.png"));
|
|
|
+ baos.close();
|
|
|
+ ff.close();
|
|
|
+ return "data:image/jpg;base64,"+png_base64;
|
|
|
+ }catch (Exception e){
|
|
|
+ e.printStackTrace();
|
|
|
+ return null;
|
|
|
+ }
|
|
|
+
|
|
|
+ }
|
|
|
+
|
|
|
+ @Autowired
|
|
|
+ private IDouyinHotService douyinHotService;
|
|
|
+ @Override
|
|
|
+ public void douyinHotHandler(int page, int listType){
|
|
|
+ String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_video/user_side/list?page="+page+"&limit=20&list_type="+listType+"&aggr_duration_type=-1");
|
|
|
+ ObjectMapper mapper = new ObjectMapper();
|
|
|
+ try {
|
|
|
+ JsonNode node = mapper.readTree(result);
|
|
|
+ int code = node.get("code").asInt();
|
|
|
+ if(code == 0){
|
|
|
+ if(node.get("data")!= null){
|
|
|
+ String dataJson = node.get("data").get("items").toString();
|
|
|
+ List<DouyinHot> list = mapper.readValue(dataJson,new TypeReference<List<DouyinHot>>() {});
|
|
|
+ if (list != null && list.size() > 0){
|
|
|
+ for (DouyinHot douyinHot : list){
|
|
|
+ douyinHot.setUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+douyinHot.getVideoId()+"&line=0"));
|
|
|
+ if(douyinHot.getUrl() == null){
|
|
|
+ douyinHot.setUrl(getVideoUrl(douyinHot.getVideoId()));
|
|
|
+ }
|
|
|
+ douyinHot.setCoverImage(getVideoFirstImage(douyinHot.getUrl()));
|
|
|
+ if(douyinHot.getCoverImage() == null){
|
|
|
+ Thread.sleep(1000);
|
|
|
+ douyinHot.setCoverImage(getVideoFirstImage(douyinHot.getUrl()));
|
|
|
+ }
|
|
|
+ douyinHotService.saveOrUpdate(douyinHot);
|
|
|
+ }
|
|
|
+ douyinHotHandler(page+1,listType);
|
|
|
+ }else {
|
|
|
+ if(listType < 4){
|
|
|
+ douyinHotHandler(1,listType+1);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }catch (Exception e){
|
|
|
+ e.printStackTrace();
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public void login(String account, String password){
|
|
|
+ String url = "https://cc.oceanengine.com/login";
|
|
|
+ System.getProperties().setProperty("webdriver.chrome.driver", chromeDriver);
|
|
|
+ ChromeOptions chromeOptions = new ChromeOptions();
|
|
|
+ chromeOptions.addArguments("--headless");
|
|
|
+ chromeOptions.addArguments("--no-sandbox");
|
|
|
+ chromeOptions.addArguments("--window-size=1920,1080");
|
|
|
+ chromeOptions.addArguments("--user-agent=" + HttpUtils2.USER_AGENT);
|
|
|
+ chromeOptions.setAcceptInsecureCerts(true);
|
|
|
+ WebDriver webDriver = new ChromeDriver(chromeOptions);
|
|
|
+ //清除所有的缓存
|
|
|
+ webDriver.manage().deleteAllCookies();
|
|
|
+ try {
|
|
|
+ //获取登录页面
|
|
|
+ webDriver.get(url);
|
|
|
+ for (Cookie cookie : webDriver.manage().getCookies()) {
|
|
|
+ BasicClientCookie ck = new BasicClientCookie(cookie.getName(), cookie.getValue());
|
|
|
+ ck.setDomain("oceanengine.com");
|
|
|
+ ck.setExpiryDate(cookie.getExpiry());
|
|
|
+ ck.setPath(cookie.getPath());
|
|
|
+ HttpUtils2.cookieStore.addCookie(ck);
|
|
|
+ }
|
|
|
+ Map<String, String> header = new HashMap<>();
|
|
|
+ header.put("Accept", "*/*");
|
|
|
+ header.put("Connection", "keep-alive");
|
|
|
+ header.put("Host", "sso.toutiao.com");
|
|
|
+ header.put("Content-Type", "application/x-www-form-urlencoded");
|
|
|
+ header.put("Origin", "https://cc.oceanengine.com");
|
|
|
+ header.put("Referer", "https://cc.oceanengine.com/login");
|
|
|
+ Map<String, Object> param = new HashMap<>();
|
|
|
+ param.put("mobile", "");
|
|
|
+ param.put("code", "");
|
|
|
+ param.put("account", account);
|
|
|
+ param.put("password", password);
|
|
|
+ param.put("captcha", "hqde");
|
|
|
+ param.put("is_30_days_no_login", "true");
|
|
|
+ param.put("service", "https://cc.oceanengine.com");
|
|
|
+ String res = HttpUtils2.httpPostParamRequest("https://sso.toutiao.com/account_login/", param, header);
|
|
|
+ ObjectMapper mapper = new ObjectMapper();
|
|
|
+ mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
|
|
+ JsonNode jsonNode = mapper.readTree(res);
|
|
|
+ //errorcode == 1102 验证码错误,请重新输入验证码
|
|
|
+ //errorcode == 1101 验证码为空
|
|
|
+
|
|
|
+ Integer errorCode = jsonNode.get("error_code").asInt();
|
|
|
+ String orderId = null;
|
|
|
+ while (errorCode != 0) {
|
|
|
+ //用户名或者密码错误
|
|
|
+ if (null != errorCode && errorCode == 1009) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ FateadmUtil fateadmUtil = new FateadmUtil();
|
|
|
+ fateadmUtil.init();
|
|
|
+ if (orderId != null) {
|
|
|
+ fateadmUtil.justice(orderId);
|
|
|
+ }
|
|
|
+ String captcha = jsonNode.get("captcha").asText();
|
|
|
+ Base64.Decoder decoder = Base64.getDecoder();
|
|
|
+ FateadmHttpUtil.HttpResp resp = fateadmUtil.Predict("30400", decoder.decode(captcha));
|
|
|
+ param.put("captcha", resp.predResl);
|
|
|
+ orderId = resp.reqId;
|
|
|
+ res = HttpUtils2.httpPostParamRequest("https://sso.toutiao.com/account_login/", param, header);
|
|
|
+ jsonNode = mapper.readTree(res);
|
|
|
+ errorCode = jsonNode.get("error_code").asInt();
|
|
|
+ }
|
|
|
+ if (errorCode == 0) {
|
|
|
+ //表示登录成功
|
|
|
+ List<org.apache.http.cookie.Cookie> cookies = HttpUtils2.cookieStore.getCookies();
|
|
|
+ for (org.apache.http.cookie.Cookie ck : cookies) {
|
|
|
+ Cookie cookie = new Cookie(ck.getName(), ck.getValue(), "." + ck.getDomain(), ck.getPath(), ck.getExpiryDate());
|
|
|
+ webDriver.manage().addCookie(cookie);
|
|
|
+ }
|
|
|
+ String redirectUrl = jsonNode.get("redirect_url").asText();
|
|
|
+ webDriver.get(redirectUrl);
|
|
|
+ //登录成功 进入推广页面
|
|
|
+ webDriver.get("https://cc.oceanengine.com/inspiration/douyin-rank-list");
|
|
|
+ Thread.sleep(3000);
|
|
|
+ for (Cookie cookie : webDriver.manage().getCookies()) {
|
|
|
+ BasicClientCookie ck = new BasicClientCookie(cookie.getName(), cookie.getValue());
|
|
|
+ ck.setDomain("cc.oceanengine.com");
|
|
|
+ ck.setExpiryDate(cookie.getExpiry());
|
|
|
+ ck.setPath(cookie.getPath());
|
|
|
+ HttpUtils2.cookieStore.addCookie(ck);
|
|
|
+ }
|
|
|
+
|
|
|
+ }
|
|
|
+ } catch (Exception e) {
|
|
|
+ e.printStackTrace();
|
|
|
+ } finally {
|
|
|
+ //获得cookie
|
|
|
+ Set<Cookie> coo = webDriver.manage().getCookies();
|
|
|
+ //清除所有的缓存
|
|
|
+ webDriver.manage().deleteAllCookies();
|
|
|
+ webDriver.quit();
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|