|
@@ -1,5 +1,6 @@
|
|
package org.jeecg.modules.ctop.job;
|
|
package org.jeecg.modules.ctop.job;
|
|
|
|
|
|
|
|
+import cn.com.ctop.common.module.utils.Base64Utils;
|
|
import cn.com.ctop.common.module.utils.HttpUtils2;
|
|
import cn.com.ctop.common.module.utils.HttpUtils2;
|
|
import cn.com.ctop.common.module.utils.ResultMapUtils;
|
|
import cn.com.ctop.common.module.utils.ResultMapUtils;
|
|
import cn.com.ctop.common.module.utils.StatusCode;
|
|
import cn.com.ctop.common.module.utils.StatusCode;
|
|
@@ -26,6 +27,8 @@ import org.quartz.JobExecutionContext;
|
|
import org.quartz.JobExecutionException;
|
|
import org.quartz.JobExecutionException;
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
|
|
|
|
|
|
+import java.net.HttpURLConnection;
|
|
|
|
+import java.net.URL;
|
|
import java.util.*;
|
|
import java.util.*;
|
|
|
|
|
|
public class OceanengineJob implements Job {
|
|
public class OceanengineJob implements Job {
|
|
@@ -36,8 +39,26 @@ public class OceanengineJob implements Job {
|
|
@Autowired
|
|
@Autowired
|
|
private IHotMaterialService hotMaterialService;
|
|
private IHotMaterialService hotMaterialService;
|
|
|
|
|
|
|
|
+ private String getRealVideoUrl(String url){
|
|
|
|
+ try {
|
|
|
|
+ URL serverUrl = new URL(url);
|
|
|
|
+ HttpURLConnection conn = (HttpURLConnection) serverUrl
|
|
|
|
+ .openConnection();
|
|
|
|
+ conn.setRequestMethod("GET");
|
|
|
|
+ // 必须设置false,否则会自动redirect到Location的地址
|
|
|
|
+ conn.setInstanceFollowRedirects(false);
|
|
|
|
+ conn.addRequestProperty("User-Agent",
|
|
|
|
+ "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.2.8) Firefox/3.6.8");
|
|
|
|
+ conn.connect();
|
|
|
|
+ String location = conn.getHeaderField("Location");
|
|
|
|
+ return location;
|
|
|
|
+ } catch (Exception e) {
|
|
|
|
+ e.printStackTrace();
|
|
|
|
+ }
|
|
|
|
+ return null;
|
|
|
|
+ }
|
|
private void hotMaterialHandler(int page,int appCode,String appName){
|
|
private void hotMaterialHandler(int page,int appCode,String appName){
|
|
- String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_material/list?material_type=3&order_by=convert_show_rate&period_type=3&aggr_app_code="+appCode+"&page="+page+"&limit=100");
|
|
|
|
|
|
+ String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_material/list?material_type=3&order_by=convert_show_rate&period_type=3&aggr_app_code="+appCode+"&page="+page+"&limit=20");
|
|
System.out.println(result);
|
|
System.out.println(result);
|
|
ObjectMapper mapper = new ObjectMapper();
|
|
ObjectMapper mapper = new ObjectMapper();
|
|
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
|
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
|
@@ -52,7 +73,10 @@ public class OceanengineJob implements Job {
|
|
for (HotMaterial material : list){
|
|
for (HotMaterial material : list){
|
|
material.setAppCode(appCode);
|
|
material.setAppCode(appCode);
|
|
material.setAppName(appName);
|
|
material.setAppName(appName);
|
|
- material.setVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+material.getVid()+"&line=0");
|
|
|
|
|
|
+ material.setVideoUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+material.getVid()+"&line=0"));
|
|
|
|
+ if(material.getVideoUrl() == null){
|
|
|
|
+ material.setVideoUrl(getVideoUrl(material.getVid()));
|
|
|
|
+ }
|
|
hotMaterialService.saveOrUpdate(material);
|
|
hotMaterialService.saveOrUpdate(material);
|
|
}
|
|
}
|
|
hotMaterialHandler(page+1,appCode,appName);
|
|
hotMaterialHandler(page+1,appCode,appName);
|
|
@@ -63,8 +87,19 @@ public class OceanengineJob implements Job {
|
|
e.printStackTrace();
|
|
e.printStackTrace();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
+ private String getVideoUrl(String vid) throws Exception{
|
|
|
|
+ String url = "http://i.snssdk.com/video/urls/1/toutiao/mp4/"+vid;
|
|
|
|
+ String result = HttpUtils2.httpGetRequest(url);
|
|
|
|
+ ObjectMapper mapper = new ObjectMapper();
|
|
|
|
+ if(result != null){
|
|
|
|
+ JsonNode node = mapper.readTree(result);
|
|
|
|
+ String mainUrl = node.get("data").get("video_list").get("video_1").get("main_url").asText();
|
|
|
|
+ return Base64Utils.decode(mainUrl);
|
|
|
|
+ }
|
|
|
|
+ return null;
|
|
|
|
+ }
|
|
private void effectCaseHandler(int page){
|
|
private void effectCaseHandler(int page){
|
|
- String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/case/effect_case_list?industry=0&style=0&mode=0&page="+page+"&limit=100");
|
|
|
|
|
|
+ String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/case/effect_case_list?industry=0&style=0&mode=0&page="+page+"&limit=20");
|
|
System.out.println(result);
|
|
System.out.println(result);
|
|
ObjectMapper mapper = new ObjectMapper();
|
|
ObjectMapper mapper = new ObjectMapper();
|
|
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
|
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
|
@@ -90,7 +125,10 @@ public class OceanengineJob implements Job {
|
|
effectCase.setCreativeScore(detailNode.get("data").get("effect_case").get("creative_score").asText());
|
|
effectCase.setCreativeScore(detailNode.get("data").get("effect_case").get("creative_score").asText());
|
|
effectCase.setEffect(detailNode.get("data").get("effect_case").get("effect").toString());
|
|
effectCase.setEffect(detailNode.get("data").get("effect_case").get("effect").toString());
|
|
effectCase.setMethods(detailNode.get("data").get("effect_case").get("methods").toString());
|
|
effectCase.setMethods(detailNode.get("data").get("effect_case").get("methods").toString());
|
|
- effectCase.setVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+effectCase.getVid()+"&line=0");
|
|
|
|
|
|
+ effectCase.setVideoUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+effectCase.getVid()+"&line=0"));
|
|
|
|
+ if (effectCase.getVideoUrl() == null){
|
|
|
|
+ effectCase.setVideoUrl(getVideoUrl(effectCase.getVid()));
|
|
|
|
+ }
|
|
effectCase.setCreativeLabel(detailNode.get("data").get("effect_case").get("creative_label").toString());
|
|
effectCase.setCreativeLabel(detailNode.get("data").get("effect_case").get("creative_label").toString());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
@@ -106,7 +144,7 @@ public class OceanengineJob implements Job {
|
|
}
|
|
}
|
|
|
|
|
|
private void douyinHotHandler(int page,int listType){
|
|
private void douyinHotHandler(int page,int listType){
|
|
- String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_video/user_side/list?page="+page+"&limit=100&list_type="+listType+"&aggr_duration_type=-1");
|
|
|
|
|
|
+ String result = HttpUtils2.httpGetRequest("https://cc.oceanengine.com/creative_center_server/api/hot_video/user_side/list?page="+page+"&limit=20&list_type="+listType+"&aggr_duration_type=-1");
|
|
System.out.println(result);
|
|
System.out.println(result);
|
|
ObjectMapper mapper = new ObjectMapper();
|
|
ObjectMapper mapper = new ObjectMapper();
|
|
try {
|
|
try {
|
|
@@ -119,7 +157,10 @@ public class OceanengineJob implements Job {
|
|
if (list != null && list.size() > 0){
|
|
if (list != null && list.size() > 0){
|
|
// douyinHotService.replaceBatch(list);
|
|
// douyinHotService.replaceBatch(list);
|
|
for (DouyinHot douyinHot : list){
|
|
for (DouyinHot douyinHot : list){
|
|
- douyinHot.setUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+douyinHot.getVideoId()+"&line=0");
|
|
|
|
|
|
+ douyinHot.setUrl(getRealVideoUrl("https://aweme.snssdk.com/aweme/v1/playwm/?video_id="+douyinHot.getVideoId()+"&line=0"));
|
|
|
|
+ if(douyinHot.getVideoId() == null){
|
|
|
|
+ douyinHot.setVideoId(getVideoUrl(douyinHot.getVideoId()));
|
|
|
|
+ }
|
|
douyinHotService.saveOrUpdate(douyinHot);
|
|
douyinHotService.saveOrUpdate(douyinHot);
|
|
}
|
|
}
|
|
douyinHotHandler(page+1,listType);
|
|
douyinHotHandler(page+1,listType);
|
|
@@ -137,11 +178,11 @@ public class OceanengineJob implements Job {
|
|
}
|
|
}
|
|
@Override
|
|
@Override
|
|
public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
|
|
public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
|
|
- String account="1728681417@qq.com";
|
|
|
|
- String password = "Yy-704127411";
|
|
|
|
|
|
+ String account="3248395570@qq.com";
|
|
|
|
+ String password = "Ydxq-704127411";
|
|
login(account,password);
|
|
login(account,password);
|
|
-// douyinHotHandler(1,1);
|
|
|
|
-// effectCaseHandler(1);
|
|
|
|
|
|
+ douyinHotHandler(1,1);
|
|
|
|
+ effectCaseHandler(1);
|
|
hotMaterialHandler(1,1,"西瓜");
|
|
hotMaterialHandler(1,1,"西瓜");
|
|
hotMaterialHandler(1,3,"火山");
|
|
hotMaterialHandler(1,3,"火山");
|
|
hotMaterialHandler(1,4,"抖音");
|
|
hotMaterialHandler(1,4,"抖音");
|
|
@@ -153,7 +194,7 @@ public class OceanengineJob implements Job {
|
|
String url = "https://cc.oceanengine.com/login";
|
|
String url = "https://cc.oceanengine.com/login";
|
|
System.getProperties().setProperty("webdriver.chrome.driver", "D:/chromedriver.exe");
|
|
System.getProperties().setProperty("webdriver.chrome.driver", "D:/chromedriver.exe");
|
|
ChromeOptions chromeOptions = new ChromeOptions();
|
|
ChromeOptions chromeOptions = new ChromeOptions();
|
|
-// chromeOptions.addArguments("--headless");
|
|
|
|
|
|
+ chromeOptions.addArguments("--headless");
|
|
chromeOptions.addArguments("--no-sandbox");
|
|
chromeOptions.addArguments("--no-sandbox");
|
|
chromeOptions.addArguments("--window-size=1920,1080");
|
|
chromeOptions.addArguments("--window-size=1920,1080");
|
|
chromeOptions.addArguments("--user-agent=" + HttpUtils2.USER_AGENT);
|
|
chromeOptions.addArguments("--user-agent=" + HttpUtils2.USER_AGENT);
|