|
@@ -0,0 +1,251 @@
|
|
|
|
+package org.jeecg.modules.ctop.job;
|
|
|
|
+
|
|
|
|
+import cn.com.ctop.common.module.utils.HttpUtils2;
|
|
|
|
+import cn.com.ctop.common.module.utils.ResultMapUtils;
|
|
|
|
+import cn.com.ctop.common.module.utils.StatusCode;
|
|
|
|
+import cn.com.ctop.crawler.modules.core.util.FateadmHttpUtil;
|
|
|
|
+import cn.com.ctop.crawler.modules.core.util.FateadmUtil;
|
|
|
|
+import cn.com.ctop.crawler.modules.oceanengine.entity.DouyinHot;
|
|
|
|
+import cn.com.ctop.crawler.modules.oceanengine.entity.EffectCase;
|
|
|
|
+import cn.com.ctop.crawler.modules.oceanengine.entity.HotMaterial;
|
|
|
|
+import cn.com.ctop.crawler.modules.oceanengine.service.IDouyinHotService;
|
|
|
|
+import cn.com.ctop.crawler.modules.oceanengine.service.IEffectCaseService;
|
|
|
|
+import cn.com.ctop.crawler.modules.oceanengine.service.IHotMaterialService;
|
|
|
|
+import com.fasterxml.jackson.core.type.TypeReference;
|
|
|
|
+import com.fasterxml.jackson.databind.DeserializationFeature;
|
|
|
|
+import com.fasterxml.jackson.databind.JsonNode;
|
|
|
|
+import com.fasterxml.jackson.databind.ObjectMapper;
|
|
|
|
+import javafx.scene.effect.Effect;
|
|
|
|
+import org.apache.http.impl.cookie.BasicClientCookie;
|
|
|
|
+import org.openqa.selenium.Cookie;
|
|
|
|
+import org.openqa.selenium.WebDriver;
|
|
|
|
+import org.openqa.selenium.chrome.ChromeDriver;
|
|
|
|
+import org.openqa.selenium.chrome.ChromeOptions;
|
|
|
|
+import org.quartz.Job;
|
|
|
|
+import org.quartz.JobExecutionContext;
|
|
|
|
+import org.quartz.JobExecutionException;
|
|
|
|
+import org.springframework.beans.factory.annotation.Autowired;
|
|
|
|
+
|
|
|
|
+import java.util.*;
|
|
|
|
+
|
|
|
|
+public class OceanengineJob implements Job {
|
|
|
|
+ @Autowired
|
|
|
|
+ private IDouyinHotService douyinHotService;
|
|
|
|
+ @Autowired
|
|
|
|
+ private IEffectCaseService effectCaseService;
|
|
|
|
+ @Autowired
|
|
|
|
+ private IHotMaterialService hotMaterialService;
|
|
|
|
+
|
|
|
|
+ private void hotMaterialHandler(int page,int appCode,String appName){
|
|
|
|
+ String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_material/list?material_type=3&order_by=convert_show_rate&period_type=3&aggr_app_code="+appCode+"&page="+page+"&limit=100");
|
|
|
|
+ System.out.println(result);
|
|
|
|
+ ObjectMapper mapper = new ObjectMapper();
|
|
|
|
+ mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
|
|
|
+ try {
|
|
|
|
+ JsonNode node = mapper.readTree(result);
|
|
|
|
+ int code = node.get("code").asInt();
|
|
|
|
+ if (code == 0) {
|
|
|
|
+ if (node.get("data") != null&&node.get("data").get("materials")!=null) {
|
|
|
|
+ String dataJson = node.get("data").get("materials").toString();
|
|
|
|
+ List<HotMaterial> list = mapper.readValue(dataJson,new TypeReference<List<HotMaterial>>() {});
|
|
|
|
+ if(list != null && list.size() > 0){
|
|
|
|
+ for (HotMaterial material : list){
|
|
|
|
+ material.setAppCode(appCode);
|
|
|
|
+ material.setAppName(appName);
|
|
|
|
+ material.setVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+material.getVid()+"&line=0");
|
|
|
|
+ hotMaterialService.saveOrUpdate(material);
|
|
|
|
+ }
|
|
|
|
+ hotMaterialHandler(page+1,appCode,appName);
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+ }catch (Exception e){
|
|
|
|
+ e.printStackTrace();
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+ private void effectCaseHandler(int page){
|
|
|
|
+ String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/case/effect_case_list?industry=0&style=0&mode=0&page="+page+"&limit=100");
|
|
|
|
+ System.out.println(result);
|
|
|
|
+ ObjectMapper mapper = new ObjectMapper();
|
|
|
|
+ mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
|
|
|
+ try {
|
|
|
|
+ JsonNode node = mapper.readTree(result);
|
|
|
|
+ int code = node.get("code").asInt();
|
|
|
|
+ if(code == 0) {
|
|
|
|
+ if (node.get("data") != null) {
|
|
|
|
+ String dataJson = node.get("data").get("effect_cases").toString();
|
|
|
|
+ List<EffectCase> list = mapper.readValue(dataJson,new TypeReference<List<EffectCase>>() {});
|
|
|
|
+ if (list != null && list.size() > 0) {
|
|
|
|
+ for (EffectCase effectCase:list){
|
|
|
|
+ String resultDetail = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/case/effect_case/"+effectCase.getId());
|
|
|
|
+ JsonNode detailNode = mapper.readTree(resultDetail);
|
|
|
|
+ int detailCode = detailNode.get("code").asInt();
|
|
|
|
+ if(detailCode == 0){
|
|
|
|
+ if(detailNode.get("data") != null){
|
|
|
|
+// String detailJson = node.get("data").get("effect_cases").toString();
|
|
|
|
+// EffectCase effectCase1 = mapper.readValue(detailJson,EffectCase.class);
|
|
|
|
+ effectCase.setAuthor(detailNode.get("data").get("effect_case").get("author").asText());
|
|
|
|
+ effectCase.setCreateDate(detailNode.get("data").get("effect_case").get("create_date").asText());
|
|
|
|
+ effectCase.setCreativeId(detailNode.get("data").get("effect_case").get("creative_id").asLong());
|
|
|
|
+ effectCase.setCreativeScore(detailNode.get("data").get("effect_case").get("creative_score").asText());
|
|
|
|
+ effectCase.setEffect(detailNode.get("data").get("effect_case").get("effect").toString());
|
|
|
|
+ effectCase.setMethods(detailNode.get("data").get("effect_case").get("methods").toString());
|
|
|
|
+ effectCase.setVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+effectCase.getVid()+"&line=0");
|
|
|
|
+ effectCase.setCreativeLabel(detailNode.get("data").get("effect_case").get("creative_label").toString());
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+ effectCaseService.saveOrUpdate(effectCase);
|
|
|
|
+ }
|
|
|
|
+ effectCaseHandler(page+1);
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+ }catch (Exception e){
|
|
|
|
+ e.printStackTrace();
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ private void douyinHotHandler(int page,int listType){
|
|
|
|
+ String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_video/user_side/list?page="+page+"&limit=100&list_type="+listType+"&aggr_duration_type=-1");
|
|
|
|
+ System.out.println(result);
|
|
|
|
+ ObjectMapper mapper = new ObjectMapper();
|
|
|
|
+ try {
|
|
|
|
+ JsonNode node = mapper.readTree(result);
|
|
|
|
+ int code = node.get("code").asInt();
|
|
|
|
+ if(code == 0){
|
|
|
|
+ if(node.get("data")!= null){
|
|
|
|
+ String dataJson = node.get("data").get("items").toString();
|
|
|
|
+ List<DouyinHot> list = mapper.readValue(dataJson,new TypeReference<List<DouyinHot>>() {});
|
|
|
|
+ if (list != null && list.size() > 0){
|
|
|
|
+// douyinHotService.replaceBatch(list);
|
|
|
|
+ for (DouyinHot douyinHot : list){
|
|
|
|
+ douyinHot.setUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+douyinHot.getVideoId()+"&line=0");
|
|
|
|
+ douyinHotService.saveOrUpdate(douyinHot);
|
|
|
|
+ }
|
|
|
|
+ douyinHotHandler(page+1,listType);
|
|
|
|
+ }else {
|
|
|
|
+ if(listType < 4){
|
|
|
|
+ douyinHotHandler(1,listType+1);
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+ }catch (Exception e){
|
|
|
|
+ e.printStackTrace();
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ }
|
|
|
|
+ @Override
|
|
|
|
+ public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
|
|
|
|
+ String account="1728681417@qq.com";
|
|
|
|
+ String password = "Yy-704127411";
|
|
|
|
+ login(account,password);
|
|
|
|
+// douyinHotHandler(1,1);
|
|
|
|
+// effectCaseHandler(1);
|
|
|
|
+ hotMaterialHandler(1,1,"西瓜");
|
|
|
|
+ hotMaterialHandler(1,3,"火山");
|
|
|
|
+ hotMaterialHandler(1,4,"抖音");
|
|
|
|
+ hotMaterialHandler(1,8,"头条");
|
|
|
|
+ hotMaterialHandler(1,9,"穿山甲");
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ private void login(String account,String password){
|
|
|
|
+ String url = "https://cc.oceanengine.com/login";
|
|
|
|
+ System.getProperties().setProperty("webdriver.chrome.driver", "D:/chromedriver.exe");
|
|
|
|
+ ChromeOptions chromeOptions = new ChromeOptions();
|
|
|
|
+// chromeOptions.addArguments("--headless");
|
|
|
|
+ chromeOptions.addArguments("--no-sandbox");
|
|
|
|
+ chromeOptions.addArguments("--window-size=1920,1080");
|
|
|
|
+ chromeOptions.addArguments("--user-agent=" + HttpUtils2.USER_AGENT);
|
|
|
|
+ chromeOptions.setAcceptInsecureCerts(true);
|
|
|
|
+ WebDriver webDriver = new ChromeDriver(chromeOptions);
|
|
|
|
+ //清除所有的缓存
|
|
|
|
+ webDriver.manage().deleteAllCookies();
|
|
|
|
+ try {
|
|
|
|
+ //获取登录页面
|
|
|
|
+ webDriver.get(url);
|
|
|
|
+ for (Cookie cookie : webDriver.manage().getCookies()) {
|
|
|
|
+ BasicClientCookie ck = new BasicClientCookie(cookie.getName(), cookie.getValue());
|
|
|
|
+ ck.setDomain("oceanengine.com");
|
|
|
|
+ ck.setExpiryDate(cookie.getExpiry());
|
|
|
|
+ ck.setPath(cookie.getPath());
|
|
|
|
+ HttpUtils2.cookieStore.addCookie(ck);
|
|
|
|
+ }
|
|
|
|
+ Map<String, String> header = new HashMap<>();
|
|
|
|
+ header.put("Accept", "*/*");
|
|
|
|
+ header.put("Connection", "keep-alive");
|
|
|
|
+ header.put("Host", "sso.toutiao.com");
|
|
|
|
+ header.put("Content-Type", "application/x-www-form-urlencoded");
|
|
|
|
+ header.put("Origin", "https://cc.oceanengine.com");
|
|
|
|
+ header.put("Referer", "https://cc.oceanengine.com/login");
|
|
|
|
+ Map<String, Object> param = new HashMap<>();
|
|
|
|
+ param.put("mobile", "");
|
|
|
|
+ param.put("code", "");
|
|
|
|
+ param.put("account", account);
|
|
|
|
+ param.put("password", password);
|
|
|
|
+ param.put("captcha", "hqde");
|
|
|
|
+ param.put("is_30_days_no_login", "true");
|
|
|
|
+ param.put("service", "https://cc.oceanengine.com");
|
|
|
|
+ String res = HttpUtils2.httpPostParamRequest("https://sso.toutiao.com/account_login/", param, header);
|
|
|
|
+ System.out.println(res);
|
|
|
|
+ ObjectMapper mapper = new ObjectMapper();
|
|
|
|
+ mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
|
|
|
+ JsonNode jsonNode = mapper.readTree(res);
|
|
|
|
+ //errorcode == 1102 验证码错误,请重新输入验证码
|
|
|
|
+ //errorcode == 1101 验证码为空
|
|
|
|
+
|
|
|
|
+ Integer errorCode = jsonNode.get("error_code").asInt();
|
|
|
|
+ String orderId = null;
|
|
|
|
+ while (errorCode != 0) {
|
|
|
|
+ //用户名或者密码错误
|
|
|
|
+ if (null != errorCode && errorCode == 1009) {
|
|
|
|
+ return;
|
|
|
|
+ }
|
|
|
|
+ FateadmUtil fateadmUtil = new FateadmUtil();
|
|
|
|
+ fateadmUtil.init();
|
|
|
|
+ if (orderId != null) {
|
|
|
|
+ fateadmUtil.Justice(orderId);
|
|
|
|
+ }
|
|
|
|
+ String captcha = jsonNode.get("captcha").asText();
|
|
|
|
+ Base64.Decoder decoder = Base64.getDecoder();
|
|
|
|
+ FateadmHttpUtil.HttpResp resp = fateadmUtil.Predict("30400", decoder.decode(captcha));
|
|
|
|
+ param.put("captcha", resp.pred_resl);
|
|
|
|
+ orderId = resp.req_id;
|
|
|
|
+ res = HttpUtils2.httpPostParamRequest("https://sso.toutiao.com/account_login/", param, header);
|
|
|
|
+ System.out.println(res);
|
|
|
|
+ jsonNode = mapper.readTree(res);
|
|
|
|
+ errorCode = jsonNode.get("error_code").asInt();
|
|
|
|
+ }
|
|
|
|
+ if (errorCode == 0) {
|
|
|
|
+ //表示登录成功
|
|
|
|
+
|
|
|
|
+ List<org.apache.http.cookie.Cookie> cookies = HttpUtils2.cookieStore.getCookies();
|
|
|
|
+ for (org.apache.http.cookie.Cookie ck : cookies) {
|
|
|
|
+ Cookie cookie = new Cookie(ck.getName(), ck.getValue(), "." + ck.getDomain(), ck.getPath(), ck.getExpiryDate());
|
|
|
|
+ webDriver.manage().addCookie(cookie);
|
|
|
|
+ }
|
|
|
|
+ String redirectUrl = jsonNode.get("redirect_url").asText();
|
|
|
|
+ webDriver.get(redirectUrl);
|
|
|
|
+ //登录成功 进入推广页面
|
|
|
|
+ webDriver.get("https://cc.oceanengine.com/inspiration/douyin-rank-list");
|
|
|
|
+ Thread.sleep(3000);
|
|
|
|
+ for (Cookie cookie : webDriver.manage().getCookies()) {
|
|
|
|
+ BasicClientCookie ck = new BasicClientCookie(cookie.getName(), cookie.getValue());
|
|
|
|
+ ck.setDomain("cc.oceanengine.com");
|
|
|
|
+ ck.setExpiryDate(cookie.getExpiry());
|
|
|
|
+ ck.setPath(cookie.getPath());
|
|
|
|
+ HttpUtils2.cookieStore.addCookie(ck);
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ }
|
|
|
|
+ } catch (Exception e) {
|
|
|
|
+ e.printStackTrace();
|
|
|
|
+ } finally {
|
|
|
|
+ //获得cookie
|
|
|
|
+ Set<Cookie> coo = webDriver.manage().getCookies();
|
|
|
|
+ //清除所有的缓存
|
|
|
|
+ webDriver.manage().deleteAllCookies();
|
|
|
|
+ webDriver.quit();
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+}
|