Selaa lähdekoodia

修改pangolin爬虫逻辑

syh 5 vuotta sitten
vanhempi
commit
69d140d440

+ 12 - 21
jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java

@@ -5,22 +5,19 @@ import cn.com.ctop.common.module.entity.CtopOauthToken;
 import cn.com.ctop.common.module.service.IBindAccountLoginService;
 import cn.com.ctop.common.module.service.ICtopOauthTokenService;
 import cn.com.ctop.common.module.utils.CtopAdConstant;
-import cn.com.ctop.crawler.modules.pangolin.entity.PangolinApp;
 import cn.com.ctop.crawler.modules.pangolin.service.PangolinAppService;
 import cn.com.ctop.crawler.modules.pangolin.service.PangolinCrawlerService;
 import cn.com.ctop.crawler.modules.pangolin.service.PangolinLoginService;
-import cn.com.ctop.kuaishou.modules.batch.service.IKuaishouInterfaceService;
 import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouWebInterfaceService;
+import cn.com.ctop.toutiao.modules.material.service.IByteDanceAdvertiserDataService;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import lombok.extern.slf4j.Slf4j;
-import org.jeecg.common.util.DateUtils;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.test.context.junit4.SpringRunner;
 
-import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
@@ -34,17 +31,11 @@ public class SampleTest {
     @Autowired
     private ICtopOauthTokenService oauthTokenService;
     @Autowired
-    private IKuaishouInterfaceService kuaishouInterfaceService;
+    private IByteDanceAdvertiserDataService advertiserDataService;
     @Test
     public void testOceanEngineJob() {
-       List<CtopOauthToken> tokens = oauthTokenService.selectKuaiShouToken();
-        Date startDate = DateUtils.addDay(new Date(),-1);
-        Date startDate2 = DateUtils.addDay(new Date(),-2);
-        for(CtopOauthToken token:tokens){
-            kuaishouInterfaceService.getAdvertiserReportDaily(token, startDate, startDate);
-            //1: 获取广告主信息数据
-            kuaishouInterfaceService.getAdvertiserReportDaily(token, startDate2, startDate2);
-       }
+        CtopOauthToken token = oauthTokenService.getTokenByAccountId(1629785592929294L);
+        advertiserDataService.getMaterialList(token);
     }
 
     @Autowired
@@ -132,14 +123,14 @@ public class SampleTest {
         if (list != null && !list.isEmpty()) {
             for (BindAccountLogin bindAccountLogin : list) {
                 pangolinCrawlerService.getChannelList(bindAccountLogin);
-                pangolinCrawlerService.getActivationList(bindAccountLogin, DateUtils.getNowDate("yyyy-MM-dd"), 1);
-                pangolinCrawlerService.getCheckList(bindAccountLogin);
-                QueryWrapper<PangolinApp> queryWrapper1 = new QueryWrapper<>();
-                queryWrapper1.eq("account_name", bindAccountLogin.getAccountName());
-                List<PangolinApp> appList = pangolinAppService.list(queryWrapper1);
-                if (appList != null && !appList.isEmpty()) {
-                    appList.forEach(pangolinApp -> pangolinCrawlerService.getRealTimeList(bindAccountLogin, pangolinApp.getId()));
-                }
+//                pangolinCrawlerService.getActivationList(bindAccountLogin, DateUtils.getNowDate("yyyy-MM-dd"), 1);
+//                pangolinCrawlerService.getCheckList(bindAccountLogin);
+//                QueryWrapper<PangolinApp> queryWrapper1 = new QueryWrapper<>();
+//                queryWrapper1.eq("account_name", bindAccountLogin.getAccountName());
+//                List<PangolinApp> appList = pangolinAppService.list(queryWrapper1);
+//                if (appList != null && !appList.isEmpty()) {
+//                    appList.forEach(pangolinApp -> pangolinCrawlerService.getRealTimeList(bindAccountLogin, pangolinApp.getId()));
+//                }
             }
         }
     }

+ 6 - 0
module-common/src/main/java/cn/com/ctop/common/module/utils/ResultMapUtils.java

@@ -22,4 +22,10 @@ public class ResultMapUtils {
         map.put(MAP_MESSAGE, statusCode.getDesc());
         map.put(MAP_SUCCESS, statusCode.getFlag());
     }
+
+    public static void setResultMap(Map<String, Object> map, int statusCode,String message,boolean flag) {
+        map.put(MAP_CODE, statusCode);
+        map.put(MAP_MESSAGE, message);
+        map.put(MAP_SUCCESS, flag);
+    }
 }

+ 3 - 0
module-common/src/main/java/cn/com/ctop/common/module/utils/StatusCode.java

@@ -51,6 +51,9 @@ public enum StatusCode {
     KUAISHOU_OPTINIZER_PERSONNAL_DATA_IS_NULL("快手媒体优化师个人消耗数据不存在", -3002, false),
     BYTEDANCE_OPTINIZER_PERSONNAL_DATA_IS_NULL("头条媒体优化师个人消耗数据不存在", -3003, false),
     OPTINIZER_PERSONNAL_DATA_IS_NULL("优化师个人消耗数据不存在", -3004, false),
+
+    BYTEDACNE_API_GROUP_CREATE_ERROR("头条API创建广告组异常",-4001,false),
+
     KUAISHOU_CRAWLER_APP_SUCCESS("爬虫运行正常", 0, true),
     KUAISHOU_CRAWLER_APP_TASK_END("爬虫任务运行结束", 1, true),
     KUAISHOU_CRAWLER_APP_JOB_StART("任务开始执行开始", 0, true),

+ 2 - 1
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/entity/PangolinApp.java

@@ -8,8 +8,9 @@ import lombok.Data;
 @Data
 @TableName("ctop_pangolin_app")
 public class PangolinApp {
-    @TableId(value = "id", type = IdType.INPUT)
+    @TableId(value = "id", type = IdType.AUTO)
     private Long id;
+    private Long appId;
     private String name;
     private String accountName;
 }

+ 2 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/PangolinAppService.java

@@ -10,4 +10,6 @@ public interface PangolinAppService extends IService<PangolinApp> {
     Map<String, Object> getListByUserId(String userId, JSONObject data);
 
     Map<String, Object> getMemberList(String userId);
+
+    void saveOrUpdateApp(PangolinApp app);
 }

+ 16 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinAppServiceImpl.java

@@ -10,6 +10,7 @@ import cn.com.ctop.crawler.modules.pangolin.mapper.PangolinAppMapper;
 import cn.com.ctop.crawler.modules.pangolin.service.PangolinAppService;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
@@ -59,4 +60,19 @@ public class PangolinAppServiceImpl extends ServiceImpl<PangolinAppMapper, Pango
         ResultMapUtils.setResultMap(resultMap,StatusCode.COMMON_SUCCESS);
         return resultMap;
     }
+
+    @Override
+    public void saveOrUpdateApp(PangolinApp app) {
+        QueryWrapper<PangolinApp>queryWrapper = new QueryWrapper<>();
+        queryWrapper.eq("account_name",app.getAccountName());
+        queryWrapper.eq("app_id",app.getAppId());
+        queryWrapper.last(" limit 1");
+        PangolinApp getApp = this.getOne(queryWrapper);
+        if(null == getApp){
+            this.save(app);
+        }else{
+            getApp.setName(app.getAccountName());
+            this.updateById(getApp);
+        }
+    }
 }

+ 2 - 2
module-crawler/src/main/java/cn/com/ctop/crawler/modules/pangolin/service/impl/PangolinCrawlerServiceImpl.java

@@ -131,9 +131,9 @@ public class PangolinCrawlerServiceImpl implements PangolinCrawlerService {
                     String name = appNode.get("Name").asText();
                     PangolinApp app = new PangolinApp();
                     app.setAccountName(bindAccountLogin.getAccountName());
-                    app.setId(id);
+                    app.setAppId(id);
                     app.setName(name);
-                    pangolinAppService.saveOrUpdate(app);
+                    pangolinAppService.saveOrUpdateApp(app);
                 }
                 Iterator<JsonNode> channelIterator = resultNode.get("data").get("ChannelNameList").elements();
                 while (channelIterator.hasNext()) {

+ 13 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/batch/service/IBytedanceGroupApiService.java

@@ -0,0 +1,13 @@
+package cn.com.ctop.toutiao.modules.batch.service;
+
+import cn.com.ctop.common.module.entity.CtopOauthToken;
+import cn.com.ctop.toutiao.modules.material.entity.ByteDanceCampaign;
+import com.alibaba.fastjson.JSONArray;
+
+import java.util.List;
+import java.util.Map;
+
+public interface IBytedanceGroupApiService {
+    List<ByteDanceCampaign> getGroupListByParams(CtopOauthToken token, JSONArray ids, String campaignName, String landingType, String status, String campaignCreateTime, Integer page, Integer pageSize);
+    Map<String,Object>createGroup(CtopOauthToken token,String campaignName,String operation,String budgetMode,String budget,String landingType,String uniqueFk);
+}

+ 117 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/batch/service/impl/BytedanceGroupApiServiceImpl.java

@@ -0,0 +1,117 @@
+package cn.com.ctop.toutiao.modules.batch.service.impl;
+
+import cn.com.ctop.common.module.entity.CtopOauthToken;
+import cn.com.ctop.common.module.utils.HttpUtils;
+import cn.com.ctop.common.module.utils.PropertiesUtils;
+import cn.com.ctop.common.module.utils.ResultMapUtils;
+import cn.com.ctop.common.module.utils.StatusCode;
+import cn.com.ctop.toutiao.modules.batch.service.IBytedanceGroupApiService;
+import cn.com.ctop.toutiao.modules.material.entity.ByteDanceCampaign;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@Service
+@Slf4j
+public class BytedanceGroupApiServiceImpl implements IBytedanceGroupApiService {
+
+    @Override
+    /**
+     * 获取广告组数据
+     */
+    public List<ByteDanceCampaign> getGroupListByParams(CtopOauthToken token, JSONArray ids, String campaignName, String landingType, String status, String campaignCreateTime, Integer page, Integer pageSize) {
+        List<ByteDanceCampaign> campaigns = new ArrayList<>();
+        // 请求地址
+        String url = PropertiesUtils.getValue("bytedance_config", "bytedance_api_url") + PropertiesUtils.getValue("bytedance_config", "bytedance_v2_campaign_get");
+        // 请求参数
+        JSONObject filtering = new JSONObject();
+        if(null!=ids&&!ids.isEmpty()){
+            filtering.put("ids",ids);
+        }
+        if(null!=campaignName&&!campaignName.equals("")){
+            filtering.put("campaign_name",campaignName);
+        }
+        if(null!=landingType&&!landingType.equals("")){
+            filtering.put("landing_type",landingType);
+        }
+        if(null!=status&&!status.equals("")){
+            filtering.put("status",status);
+        }
+        if(null!=campaignCreateTime&&!campaignCreateTime.equals("")){
+            filtering.put("campaign_create_time",landingType);
+        }
+
+        JSONObject params = new JSONObject();
+        params.put("advertiser_id", token.getAccountId());
+        params.put("page", page);
+        params.put("page_size", pageSize);
+        params.put("filtering", filtering);
+        JSONObject result = HttpUtils.bytedanceGetRequest(token.getAccessToken(),url,params);
+        if(result == null){
+            log.error("获取广告组数据异常=>param:{}",params.toJSONString());
+            return campaigns;
+        }
+        Integer code = result.getInteger("code");
+        String message = result.getString("message");
+        if(null == code||!code.equals(0)){
+            log.error("获取广告组数据异常=>{},param:{}",message,params.toJSONString());
+            return campaigns;
+        }
+        JSONArray dataList = result.getJSONObject("data").getJSONArray("list");
+        if(null == dataList||dataList.isEmpty()){
+            log.info("获取广告组数据为空=>param:{}",params.toJSONString());
+            return campaigns;
+        }
+        for(int i=0;i<dataList.size();i++){
+            JSONObject data = dataList.getJSONObject(i);
+            ByteDanceCampaign campaign = new ByteDanceCampaign(data,token.getAccountId().toString());
+            campaigns.add(campaign);
+        }
+        return campaigns;
+    }
+
+    @Override
+    public Map<String, Object> createGroup(CtopOauthToken token, String campaignName, String operation, String budgetMode, String budget, String landingType, String uniqueFk) {
+        Map<String,Object>result = new HashMap<>();
+        // 请求地址
+        String url = PropertiesUtils.getValue("bytedance_config", "bytedance_api_url") + PropertiesUtils.getValue("bytedance_config", "bytedance_v2_campaign_create");
+        // 请求参数
+        JSONObject param = new JSONObject();
+        param.put("advertiser_id",token.getAccountId());
+        param.put("campaign_name",campaignName);
+        if(null!=operation&&!operation.trim().equals("")){
+            param.put("operation",operation);
+        }
+        param.put("budget_mode", budgetMode);
+        if(null != budget&&!budget.trim().equals("")){
+            param.put("budget",budget);
+        }
+        param.put("landing_type", landingType);
+        if(null != uniqueFk&&!uniqueFk.trim().equals("")){
+            param.put("unique_fk",uniqueFk);
+        }
+        JSONObject data = HttpUtils.bytedancePostRequest(url,token.getAccessToken(),param);
+        if(data == null){
+            log.error("创建广告组异常=>param:{}",param.toJSONString());
+            ResultMapUtils.setResultMap(result, StatusCode.BYTEDACNE_API_GROUP_CREATE_ERROR);
+            return result;
+        }
+        Integer code = data.getInteger("code");
+        String message = data.getString("message");
+        if(null == code||!code.equals(0)){
+            log.error("获取广告组数据异常=>{},parmas:{}",message,param.toJSONString());
+            ResultMapUtils.setResultMap(result,-1,message,false);
+            return result;
+        }
+        Long campaignId = data.getJSONObject("data").getLong("campaign_id");
+        ResultMapUtils.setResultMap(result,StatusCode.COMMON_SUCCESS);
+        result.put("campaign_id",campaignId);
+        return result;
+    }
+}

+ 4 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/material/entity/ByteDanceCampaign.java

@@ -195,4 +195,8 @@ public class ByteDanceCampaign {
         this.createTime = now;
         this.updateTime = now;
     }
+
+    public ByteDanceCampaign(JSONObject data, CtopOauthToken token) {
+
+    }
 }