|
@@ -1,16 +1,21 @@
|
|
package org.jeecg;
|
|
package org.jeecg;
|
|
|
|
|
|
import cn.com.ctop.common.module.entity.BindAccountLogin;
|
|
import cn.com.ctop.common.module.entity.BindAccountLogin;
|
|
-import cn.com.ctop.common.module.entity.CtopOauthToken;
|
|
|
|
import cn.com.ctop.common.module.service.IBindAccountLoginService;
|
|
import cn.com.ctop.common.module.service.IBindAccountLoginService;
|
|
import cn.com.ctop.common.module.service.ICtopOauthTokenService;
|
|
import cn.com.ctop.common.module.service.ICtopOauthTokenService;
|
|
import cn.com.ctop.common.module.utils.CtopAdConstant;
|
|
import cn.com.ctop.common.module.utils.CtopAdConstant;
|
|
-import cn.com.ctop.crawler.modules.oceanengine.entity.HotMaterial;
|
|
|
|
-import cn.com.ctop.crawler.modules.oceanengine.service.IHotMaterialService;
|
|
|
|
-import cn.com.ctop.crawler.modules.oceanengine.service.IOceanEngineService;
|
|
|
|
|
|
+import cn.com.ctop.common.module.utils.HttpUtils;
|
|
|
|
+import cn.com.ctop.crawler.modules.pangolin.entity.PangolinApp;
|
|
|
|
+import cn.com.ctop.crawler.modules.pangolin.service.PangolinAppService;
|
|
|
|
+import cn.com.ctop.crawler.modules.pangolin.service.PangolinCrawlerService;
|
|
|
|
+import cn.com.ctop.crawler.modules.pangolin.service.PangolinLoginService;
|
|
import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouWebInterfaceService;
|
|
import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouWebInterfaceService;
|
|
-import cn.com.ctop.toutiao.modules.report.service.IBytedanceFundDailyService;
|
|
|
|
-import com.xxl.job.core.log.XxlJobLogger;
|
|
|
|
|
|
+import cn.com.ctop.toutiao.modules.report.service.IByteDanceVideoReportDailyService;
|
|
|
|
+import cn.com.ctop.toutiao.modules.report.service.IBytedanceReportService;
|
|
|
|
+import cn.com.ctop.toutiao.modules.report.service.IReportService;
|
|
|
|
+import com.alibaba.fastjson.JSONArray;
|
|
|
|
+import com.alibaba.fastjson.JSONObject;
|
|
|
|
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
|
|
import lombok.extern.slf4j.Slf4j;
|
|
import lombok.extern.slf4j.Slf4j;
|
|
import org.jeecg.common.util.DateUtils;
|
|
import org.jeecg.common.util.DateUtils;
|
|
import org.junit.Test;
|
|
import org.junit.Test;
|
|
@@ -21,6 +26,7 @@ import org.springframework.test.context.junit4.SpringRunner;
|
|
|
|
|
|
import java.util.HashMap;
|
|
import java.util.HashMap;
|
|
import java.util.List;
|
|
import java.util.List;
|
|
|
|
+import java.util.Map;
|
|
import java.util.concurrent.CountDownLatch;
|
|
import java.util.concurrent.CountDownLatch;
|
|
import java.util.concurrent.ExecutorService;
|
|
import java.util.concurrent.ExecutorService;
|
|
import java.util.concurrent.Executors;
|
|
import java.util.concurrent.Executors;
|
|
@@ -30,26 +36,53 @@ import java.util.concurrent.Executors;
|
|
@Slf4j
|
|
@Slf4j
|
|
public class SampleTest {
|
|
public class SampleTest {
|
|
@Autowired
|
|
@Autowired
|
|
- private IOceanEngineService oceanEngineService;
|
|
|
|
|
|
+ private IBytedanceReportService bytedanceReportService;
|
|
@Autowired
|
|
@Autowired
|
|
- private IHotMaterialService hotMaterialService;
|
|
|
|
|
|
+ private IByteDanceVideoReportDailyService byteDanceVideoReportDailyService;
|
|
|
|
+ @Autowired
|
|
|
|
+ private ICtopOauthTokenService oauthTokenService;
|
|
|
|
+ @Autowired
|
|
|
|
+ private IReportService reportService;
|
|
|
|
+
|
|
@Test
|
|
@Test
|
|
- public void initImageUrl(){
|
|
|
|
- List<HotMaterial> hotMaterials = hotMaterialService.getId();
|
|
|
|
- if(null!=hotMaterials&&!hotMaterials.isEmpty()){
|
|
|
|
- hotMaterials.forEach(douyinHot -> {
|
|
|
|
- HotMaterial getHot = hotMaterialService.getById(douyinHot.getId());
|
|
|
|
-// String image = getHot.getCoverImage().replace("data:image/jpg;base64,", "");
|
|
|
|
- String destPath = "D:\\data\\effectCase\\";
|
|
|
|
- String fileName = getHot.getId()+".jpg";
|
|
|
|
-// EffectCase.base64ToFile(destPath,image,fileName);
|
|
|
|
- });
|
|
|
|
|
|
+ public void testOceanEngineJob() {
|
|
|
|
+ try {
|
|
|
|
+ String url = "https://ad.e.kuaishou.com/rest/openapi/v1/tool/key_frame";
|
|
|
|
+ JSONArray ptotoArr = new JSONArray();
|
|
|
|
+ ptotoArr.add("5188991225263567823");
|
|
|
|
+
|
|
|
|
+ Map<String, String> headers = new HashMap<>();
|
|
|
|
+ headers.put("Access-Token", "e251db4aa139eb36623818581003bcec");
|
|
|
|
+ headers.put("Content-Type", "application/json");
|
|
|
|
+ JSONObject requestJson = new JSONObject();
|
|
|
|
+ requestJson.put("advertiser_id", 161468);
|
|
|
|
+ requestJson.put("photo_ids", ptotoArr);
|
|
|
|
+ // requestJson.put("type", type);
|
|
|
|
+ String result = HttpUtils.kuaiShouhttpPostRequest(url, requestJson.toJSONString(), headers);
|
|
|
|
+ System.err.println(result);
|
|
|
|
+ // CtopOauthToken byId = oauthTokenService.getById(1654059015242756L);
|
|
|
|
+
|
|
|
|
+ /* SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
|
|
|
|
+ Date start = simpleDateFormat.parse("2020-01-01");
|
|
|
|
+ Date end = simpleDateFormat.parse("2020-06-03");*/
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+ /*String nowDate = "2020-06-03";
|
|
|
|
+ String endDate = "2020-01-01";
|
|
|
|
+ SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
|
|
|
|
+ Date start = simpleDateFormat.parse(nowDate);
|
|
|
|
+ Date end = simpleDateFormat.parse(endDate);
|
|
|
|
+ List<Date> dates = DateUtils.findDates(end, start);
|
|
|
|
+ for (int i = 0; i < dates.size(); i++) {
|
|
|
|
+ String formatDate = simpleDateFormat.format(dates.get(i));
|
|
|
|
+ Date parse = simpleDateFormat.parse(formatDate);
|
|
|
|
+ reportService.getAdvertiserReport(byId, parse, parse, CtopAdConstant.BYTEDANCE_REPORT_TYPE_DAILY);
|
|
|
|
+ }*/
|
|
|
|
+ } catch (Exception e) {
|
|
|
|
+ e.printStackTrace();
|
|
}
|
|
}
|
|
- }
|
|
|
|
- @Test
|
|
|
|
- public void testOceanEngineJob(){
|
|
|
|
- String account="3248395570@qq.com";
|
|
|
|
- String password = "Ydxq-704127411";
|
|
|
|
|
|
+ String account = "3248395570@qq.com";
|
|
|
|
+ /* String password = "Ydxq-704127411";
|
|
oceanEngineService.login(account,password);
|
|
oceanEngineService.login(account,password);
|
|
oceanEngineService.douyinHotHandler(1,1);
|
|
oceanEngineService.douyinHotHandler(1,1);
|
|
oceanEngineService.effectCaseHandler(1);
|
|
oceanEngineService.effectCaseHandler(1);
|
|
@@ -58,45 +91,48 @@ public class SampleTest {
|
|
oceanEngineService.hotMaterialHandler(1,1,"西瓜");
|
|
oceanEngineService.hotMaterialHandler(1,1,"西瓜");
|
|
oceanEngineService.hotMaterialHandler(1,3,"火山");
|
|
oceanEngineService.hotMaterialHandler(1,3,"火山");
|
|
oceanEngineService.hotMaterialHandler(1,9,"穿山甲");
|
|
oceanEngineService.hotMaterialHandler(1,9,"穿山甲");
|
|
- log.info("巨量创意抓取完成");
|
|
|
|
|
|
+ log.info("巨量创意抓取完成");*/
|
|
}
|
|
}
|
|
|
|
|
|
@Autowired
|
|
@Autowired
|
|
private IBindAccountLoginService bindAccountLoginService;
|
|
private IBindAccountLoginService bindAccountLoginService;
|
|
@Autowired
|
|
@Autowired
|
|
private IKuaishouWebInterfaceService kuaishouWebInterfaceService;
|
|
private IKuaishouWebInterfaceService kuaishouWebInterfaceService;
|
|
|
|
+
|
|
@Test
|
|
@Test
|
|
- public void kuaishouLogin(){
|
|
|
|
- Long start =System.currentTimeMillis();
|
|
|
|
- List<BindAccountLogin> list = bindAccountLoginService.getListByParams(CtopAdConstant.PLATFORM_TYPE_KUAISHOU_PY,1);
|
|
|
|
- if(list!=null&&!list.isEmpty()){
|
|
|
|
- for (BindAccountLogin login:list) {
|
|
|
|
- if(null==login.getCookie()||"".equals(login.getCookie().trim())){
|
|
|
|
|
|
+ public void kuaishouLogin() {
|
|
|
|
+ Long start = System.currentTimeMillis();
|
|
|
|
+ List<BindAccountLogin> list = bindAccountLoginService.getListByParams(CtopAdConstant.PLATFORM_TYPE_KUAISHOU_PY, 1);
|
|
|
|
+ if (list != null && !list.isEmpty()) {
|
|
|
|
+ for (BindAccountLogin login : list) {
|
|
|
|
+ if (null == login.getCookie() || "".equals(login.getCookie().trim())) {
|
|
kuaishouWebInterfaceService.getkuaishouWebLoginCookie(login);
|
|
kuaishouWebInterfaceService.getkuaishouWebLoginCookie(login);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
Long end = System.currentTimeMillis();
|
|
Long end = System.currentTimeMillis();
|
|
- log.info("总用时:{}毫秒",end-start);
|
|
|
|
|
|
+ log.info("总用时:{}毫秒", end - start);
|
|
}
|
|
}
|
|
|
|
+
|
|
static ExecutorService executorService = null;
|
|
static ExecutorService executorService = null;
|
|
static CountDownLatch countDownLatch = null;
|
|
static CountDownLatch countDownLatch = null;
|
|
|
|
+
|
|
@Test
|
|
@Test
|
|
- public void kuaishouCommentDelete(){
|
|
|
|
- Long start =System.currentTimeMillis();
|
|
|
|
- List<BindAccountLogin> list = bindAccountLoginService.getListByParams(CtopAdConstant.PLATFORM_TYPE_KUAISHOU_PY,1);
|
|
|
|
- if(list!=null&&!list.isEmpty()){
|
|
|
|
|
|
+ public void kuaishouCommentDelete() {
|
|
|
|
+ Long start = System.currentTimeMillis();
|
|
|
|
+ List<BindAccountLogin> list = bindAccountLoginService.getListByParams(CtopAdConstant.PLATFORM_TYPE_KUAISHOU_PY, 1);
|
|
|
|
+ if (list != null && !list.isEmpty()) {
|
|
executorService = Executors.newFixedThreadPool(10);
|
|
executorService = Executors.newFixedThreadPool(10);
|
|
countDownLatch = new CountDownLatch(list.size());
|
|
countDownLatch = new CountDownLatch(list.size());
|
|
list.forEach(login -> {
|
|
list.forEach(login -> {
|
|
- executorService.submit(()->{
|
|
|
|
|
|
+ executorService.submit(() -> {
|
|
try {
|
|
try {
|
|
- if(null!=login.getCookie()&&!"".equals(login.getCookie().trim())){
|
|
|
|
- kuaishouWebInterfaceService.deleteAllComment(new HashMap<>(),login);
|
|
|
|
|
|
+ if (null != login.getCookie() && !"".equals(login.getCookie().trim())) {
|
|
|
|
+ kuaishouWebInterfaceService.deleteAllComment(new HashMap<>(), login);
|
|
}
|
|
}
|
|
- }catch (Exception e){
|
|
|
|
|
|
+ } catch (Exception e) {
|
|
|
|
|
|
- }finally {
|
|
|
|
|
|
+ } finally {
|
|
countDownLatch.countDown();
|
|
countDownLatch.countDown();
|
|
}
|
|
}
|
|
});
|
|
});
|
|
@@ -108,41 +144,51 @@ public class SampleTest {
|
|
e.printStackTrace();
|
|
e.printStackTrace();
|
|
}
|
|
}
|
|
Long end = System.currentTimeMillis();
|
|
Long end = System.currentTimeMillis();
|
|
- log.info("总用时:{}毫秒",end-start);
|
|
|
|
|
|
+ log.info("总用时:{}毫秒", end - start);
|
|
}
|
|
}
|
|
|
|
+
|
|
|
|
+ @Autowired
|
|
|
|
+ private PangolinLoginService pangolinLoginService;
|
|
@Autowired
|
|
@Autowired
|
|
- private ICtopOauthTokenService tokenService;
|
|
|
|
|
|
+ private PangolinCrawlerService pangolinCrawlerService;
|
|
@Autowired
|
|
@Autowired
|
|
- private IBytedanceFundDailyService bytedanceFundDailyService;
|
|
|
|
|
|
+ private PangolinAppService pangolinAppService;
|
|
|
|
+
|
|
@Test
|
|
@Test
|
|
- public void testBase64(){
|
|
|
|
- List<CtopOauthToken> tokens = tokenService.getTokenListByType(CtopAdConstant.PLATFORM_TYPE_BYTEDANCE);
|
|
|
|
- if (null == tokens || tokens.size() <= 0) {
|
|
|
|
- XxlJobLogger.log("头条账户流水数据异常:未获取到可用的token");
|
|
|
|
- return ;
|
|
|
|
|
|
+ public void testPangolinLogin() {
|
|
|
|
+ QueryWrapper<BindAccountLogin> queryWrapper = new QueryWrapper<>();
|
|
|
|
+ queryWrapper.eq("login_type", "pangolin");
|
|
|
|
+ queryWrapper.eq("status", 1);
|
|
|
|
+ List<BindAccountLogin> list = bindAccountLoginService.list(queryWrapper);
|
|
|
|
+ if (list != null && !list.isEmpty()) {
|
|
|
|
+ for (BindAccountLogin login : list) {
|
|
|
|
+ if (null == login.getCookie() || "".equals(login.getCookie().trim())) {
|
|
|
|
+ pangolinLoginService.pangolinLogin(login.getAccountName(), login.getPassword());
|
|
|
|
+ }
|
|
|
|
+ }
|
|
}
|
|
}
|
|
- executorService = Executors.newFixedThreadPool(4);
|
|
|
|
- countDownLatch = new CountDownLatch(tokens.size());
|
|
|
|
- tokens.forEach(token -> {
|
|
|
|
- executorService.submit(() -> {
|
|
|
|
- try {
|
|
|
|
- for(int i=0;i<12;i++){
|
|
|
|
- String getDate = DateUtils.addDay("2020-05-28",-i);
|
|
|
|
- bytedanceFundDailyService.loadFundDataByPage(token, getDate, getDate,1);
|
|
|
|
- }
|
|
|
|
- }catch (Exception e){
|
|
|
|
|
|
+ }
|
|
|
|
|
|
- }finally {
|
|
|
|
- countDownLatch.countDown();
|
|
|
|
|
|
+ @Test
|
|
|
|
+ public void testPangolinData() {
|
|
|
|
+ QueryWrapper<BindAccountLogin> queryWrapper = new QueryWrapper<>();
|
|
|
|
+ queryWrapper.eq("login_type", "pangolin");
|
|
|
|
+ queryWrapper.eq("status", 1);
|
|
|
|
+ queryWrapper.isNotNull("cookie");
|
|
|
|
+ List<BindAccountLogin> list = bindAccountLoginService.list(queryWrapper);
|
|
|
|
+ if (list != null && !list.isEmpty()) {
|
|
|
|
+ for (BindAccountLogin bindAccountLogin : list) {
|
|
|
|
+ pangolinCrawlerService.getChannelList(bindAccountLogin);
|
|
|
|
+ pangolinCrawlerService.getActivationList(bindAccountLogin, DateUtils.getNowDate("yyyy-MM-dd"), 1);
|
|
|
|
+ pangolinCrawlerService.getCheckList(bindAccountLogin);
|
|
|
|
+ QueryWrapper<PangolinApp> queryWrapper1 = new QueryWrapper<>();
|
|
|
|
+ queryWrapper1.eq("account_name", bindAccountLogin.getAccountName());
|
|
|
|
+ List<PangolinApp> appList = pangolinAppService.list(queryWrapper1);
|
|
|
|
+ if (appList != null && !appList.isEmpty()) {
|
|
|
|
+ appList.forEach(pangolinApp -> pangolinCrawlerService.getRealTimeList(bindAccountLogin, pangolinApp.getId()));
|
|
}
|
|
}
|
|
-
|
|
|
|
- });
|
|
|
|
- });
|
|
|
|
- try {
|
|
|
|
- countDownLatch.await();
|
|
|
|
- } catch (InterruptedException e) {
|
|
|
|
|
|
+ }
|
|
}
|
|
}
|
|
- System.out.println("执行完成");
|
|
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|