Sfoglia il codice sorgente

调整批量数据请求逻辑

yumeng 5 anni fa
parent
commit
01f98662a8

+ 12 - 2
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/controller/BatchController.java

@@ -147,6 +147,16 @@ public class BatchController {
             };
             thread3.start();
 
+            Thread thread4 = new Thread() {
+                @Override
+                public void run() {
+                    kuaiShouGroupService.getGroupList(oauthToken.getAccessToken(), oauthToken.getAccountId());
+
+                }
+            };
+            thread4.start();
+
+
         } catch (Exception e) {
             e.printStackTrace();
             result.setSuccess(false);
@@ -1062,13 +1072,13 @@ public class BatchController {
     public Result<IPage<KuaiShouVideoGet>> getVideoList(KuaiShouVideoGet kuaiShouVideoGet, @RequestParam(name = "pageNo", defaultValue = "1") Integer pageNo,
                                                         @RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize,
                                                         @RequestParam(name = "startDate", defaultValue = "2010-01-01") String startDate,
-                                                        @RequestParam(name = "endDate", defaultValue = "2100-01-01") String endDate,HttpServletRequest req) {
+                                                        @RequestParam(name = "endDate", defaultValue = "2100-01-01") String endDate, HttpServletRequest req) {
         Result<IPage<KuaiShouVideoGet>> result = new Result<>();
         try {
             QueryWrapper<KuaiShouVideoGet> queryWrapper = QueryGenerator.initQueryWrapper(kuaiShouVideoGet, req.getParameterMap());
             startDate = startDate + " 00:00:00";
             endDate = endDate + " 23:59:59";
-            queryWrapper.between("create_time",startDate,endDate);
+            queryWrapper.between("create_time", startDate, endDate);
             queryWrapper.orderByDesc("create_time");
             Page<KuaiShouVideoGet> page = new Page<>(pageNo, pageSize);
             IPage<KuaiShouVideoGet> pageList = kuaiShouVideoGetService.page(page, queryWrapper);

+ 1 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/IKuaiShouGroupService.java

@@ -11,4 +11,5 @@ import com.baomidou.mybatisplus.extension.service.IService;
  */
 public interface IKuaiShouGroupService extends IService<KuaiShouGroup> {
 
+    void getGroupList(String accessToken, Long accountId);
 }

+ 260 - 3
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/impl/KuaiShouGroupServiceImpl.java

@@ -1,12 +1,27 @@
 package cn.com.ctop.kuaishou.modules.batch.service.impl;
 
-import cn.com.ctop.kuaishou.modules.batch.entity.KuaiShouGroup;
+import cn.com.ctop.common.module.utils.Check;
+import cn.com.ctop.common.module.utils.HttpUtils;
+import cn.com.ctop.common.module.utils.KuaishouInterfaceConstant;
+import cn.com.ctop.common.module.utils.PropertiesUtils;
+import cn.com.ctop.kuaishou.modules.batch.entity.*;
+import cn.com.ctop.kuaishou.modules.batch.mapper.KuaiShouAppInfoMapper;
 import cn.com.ctop.kuaishou.modules.batch.mapper.KuaiShouGroupMapper;
+import cn.com.ctop.kuaishou.modules.batch.mapper.KuaiShouGroupTargetMapper;
+import cn.com.ctop.kuaishou.modules.batch.mapper.KuaiShouScheduleMapper;
+import cn.com.ctop.kuaishou.modules.batch.service.IKuaiShouCampaignService;
 import cn.com.ctop.kuaishou.modules.batch.service.IKuaiShouGroupService;
-
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
-import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import java.util.*;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 
 /**
  * @Description: 广告组信息
@@ -14,7 +29,249 @@ import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
  * @Date: 2019-07-23
  * @Version: V1.0
  */
+
+@Slf4j
 @Service
 public class KuaiShouGroupServiceImpl extends ServiceImpl<KuaiShouGroupMapper, KuaiShouGroup> implements IKuaiShouGroupService {
+    @Autowired
+    private IKuaiShouCampaignService kuaiShouCampaignService;
+    @Autowired
+    private KuaiShouGroupMapper groupMapper;
+    @Autowired
+    private KuaiShouAppInfoMapper appInfoMapper;
+    @Autowired
+    private KuaiShouScheduleMapper shouScheduleMapper;
+    @Autowired
+    private KuaiShouGroupTargetMapper groupTargetMapper;
+    private static ExecutorService executorService = Executors.newFixedThreadPool(10);
+
+
+    /**
+     * 获取广告组
+     *
+     * @param accessToken
+     * @param accountId
+     */
+    @Override
+    public void getGroupList(String accessToken, Long accountId) {
+
+        QueryWrapper<KuaiShouCampaign> campaignQueryWrapper = new QueryWrapper<>();
+        campaignQueryWrapper.eq("account_id", accountId);
+        campaignQueryWrapper.eq("status", 4);
+        campaignQueryWrapper.eq("put_status", 1);
+        campaignQueryWrapper.orderByDesc("put_create_time");
+        List<KuaiShouCampaign> list = kuaiShouCampaignService.list(campaignQueryWrapper);
+        if (Check.isNull(list)) {
+            return;
+        }
+        for (KuaiShouCampaign campaign : list) {
+            executorService.submit(new Runnable() {
+                @Override
+                public void run() {
+                    getGroupListByPage(accessToken, accountId, campaign.getCampaignId(), 1);
+                }
+            });
+        }
+
+
+    }
+
+
+    private void getGroupListByPage(String accessToken, Long accountId, Long campaignId, int page) {
+        String url = PropertiesUtils.getConfig("kuaishou_api_url") + KuaishouInterfaceConstant.GROUP_LIST;
+        JSONObject param = new JSONObject();
+        param.put("advertiser_id", accountId);
+        param.put("campaign_id", campaignId);
+        param.put("page_size", 200);
+        param.put("page", page);
+
+        Map<String, String> headers = new HashMap<>();
+        headers.put("Access-Token", accessToken);
+        headers.put("Content-Type", "application/json");
+
+        String result = HttpUtils.kuaiShouhttpPostRequest(url, param.toJSONString(), headers);
+        JSONObject resultJson = JSONObject.parseObject(result);
+        if (Check.isNull(resultJson)) {
+            log.error("获取广告组接口异常,advertiserId:{}", accountId);
+            return;
+        }
+        Integer code = resultJson.getInteger("code");
+        if (null == code || code != 0) {
+            log.error("获取广告组返回结果异常,advertiserId:{},异常信息:{}", accountId, resultJson);
+            return;
+        }
+        JSONArray details = resultJson.getJSONObject("data").getJSONArray("details");
+        if (Check.isNull(details)) {
+            return;
+        }
+        addGroupV2(accountId, details);
+        getGroupListByPage(accessToken, accountId, campaignId, page + 1);
+    }
+
+
+    private void addGroupV2(Long advertiserId, JSONArray details) {
+        if (!Check.isNull(details)) {
+            List<KuaiShouGroup> groups = new ArrayList<>();
+            List<KuaiShouAppInfo> appInfos = new ArrayList<>();
+            List<KuaiShouSchedule> schedules = new ArrayList<>();
+            List<KuaiShouGroupTarget> targets = new ArrayList<>();
+            for (int i = 0; i < details.size(); i++) {
+                JSONObject detail = JSONObject.parseObject(details.get(i).toString());
+                if (!Check.isNull(detail)) {
+                    KuaiShouGroup group = new KuaiShouGroup();
+                    Long unitId = detail.getLong("unit_id");
+                    group.setId("" + advertiserId + unitId);
+                    group.setAccountId(advertiserId);
+                    group.setCampaignId(detail.getLong("campaign_id"));
+                    group.setUnitId(unitId);
+                    group.setUnitName(detail.getString("unit_name"));
+                    group.setStatus(detail.getInteger("status"));
+                    group.setPutStatus(detail.getInteger("put_status"));
+                    group.setCreateChannel(detail.getInteger("create_channel"));
+                    group.setReviewDetail(detail.getString("review_detail"));
+                    group.setBidType(detail.getInteger("bid_type"));
+                    group.setBid(detail.getLong("bid"));
+                    group.setCpaBid(detail.getLong("cpa_bid"));
+                    group.setOcpxActionType(detail.getInteger("ocpx_action_type"));
+                    group.setDeepConversionType(detail.getInteger("deep_conversion_type"));
+                    group.setDeepConversionBid(detail.getLong("deep_conversion_bid"));
+                    group.setDayBudget(detail.getLong("day_budget"));
+                    group.setSpeed(detail.getInteger("speed"));
+                    group.setBeginTime(detail.getString("begin_time"));
+                    group.setEndTime(detail.getString("end_time"));
+                    group.setScheduleTime(detail.getString("schedule_time"));
+                    group.setSceneId(detail.getJSONArray("scene_id") + "");
+                    group.setShowMode(detail.getInteger("show_mode"));
+                    group.setUnitType(detail.getInteger("unit_type"));
+                    group.setUrlType(detail.getInteger("url_type"));
+                    group.setUrl(detail.getString("url"));
+                    group.setAppId(detail.getLong("app_id"));
+                    group.setAppIconUrl(detail.getString("app_icon_url"));
+                    group.setGroupCreateTime(detail.getString("create_time"));
+                    group.setGroupUpdateTime(detail.getString("update_time"));
+                    group.setConvertId(detail.getLong("convert_id"));
+                    group.setUseAppMarket(detail.getInteger("use_app_market"));
+                    group.setCreateTime(new Date());
+                    group.setUpdateTime(new Date());
+                    groups.add(group);
+                    // 添加应用信息
+
+                    JSONObject diverseJson = detail.getJSONObject("diverse_data");
+                    if (!Check.isNull(diverseJson)) {
+                        KuaiShouAppInfo appInfo = new KuaiShouAppInfo();
+                        appInfo.setAccountId(advertiserId);
+                        appInfo.setAppName(diverseJson.getString("app_name"));
+                        appInfo.setAppPackageName(diverseJson.getString("app_package_name"));
+                        appInfo.setDeviceOsType(diverseJson.getInteger("device_os_type"));
+                        appInfo.setUnitId(unitId);
+                        appInfo.setCreateTime(new Date());
+                        appInfo.setUpdateTime(new Date());
+                        appInfos.add(appInfo);
+                    }
+                    //时间段信息
+                    JSONObject scheduleJson = detail.getJSONObject("schedule");
+                    if (!Check.isNull(scheduleJson)) {
+                        KuaiShouSchedule schedule = new KuaiShouSchedule();
+                        schedule.setUnitId(unitId);
+                        schedule.setAccountId(advertiserId);
+                        schedule.setMon(scheduleJson.getJSONArray("mon").toJSONString());
+                        schedule.setTues(scheduleJson.getJSONArray("tues").toJSONString());
+                        schedule.setWed(scheduleJson.getJSONArray("wed").toJSONString());
+                        schedule.setThur(scheduleJson.getJSONArray("thur").toJSONString());
+                        schedule.setFri(scheduleJson.getJSONArray("fri").toJSONString());
+                        schedule.setSat(scheduleJson.getJSONArray("sat").toJSONString());
+                        schedule.setSun(scheduleJson.getJSONArray("sun").toJSONString());
+                        schedule.setCreateTime(new Date());
+                        schedule.setUpdateTime(new Date());
+                        schedules.add(schedule);
+                    }
+
+                    JSONObject targetJson = detail.getJSONObject("target");
+                    if (!Check.isNull(targetJson)) {
+                        KuaiShouGroupTarget groupTarget = new KuaiShouGroupTarget();
+                        groupTarget.setAccountId(advertiserId);
+                        groupTarget.setUnitId(unitId);
+
+                        if (!Check.isNull(targetJson.getJSONArray("region"))) {
+                            groupTarget.setRegion(targetJson.getJSONArray("region").toJSONString());
+                        }
+
+                        JSONObject age = targetJson.getJSONObject("age");
+                        if (!Check.isNull(age)) {
+                            groupTarget.setAgeMin(age.getInteger("min"));
+                            groupTarget.setAgeMax(age.getInteger("max"));
+                        }
+                        if (!Check.isNull(targetJson.getJSONArray("ages_range"))) {
+                            groupTarget.setAgesRange(targetJson.getJSONArray("ages_range").toJSONString());
+                        }
+
+                        groupTarget.setGender(targetJson.getInteger("gender"));
+                        groupTarget.setPlatformOs(targetJson.getInteger("platform_os"));
+                        groupTarget.setAndroidOsv(targetJson.getInteger("android_osv"));
+                        groupTarget.setIosOsv(targetJson.getInteger("ios_osv"));
+                        groupTarget.setNetwork(targetJson.getInteger("network"));
+                        if (!Check.isNull(targetJson.getJSONArray("device_brand"))) {
+                            groupTarget.setDeviceBrand(targetJson.getJSONArray("device_brand").toJSONString());
+                        }
+
+                        if (!Check.isNull(targetJson.getJSONArray("device_price"))) {
+                            groupTarget.setDevicePrice(targetJson.getJSONArray("device_price").toJSONString());
+                        }
+
+                        groupTarget.setBusinessInterestType(targetJson.getInteger("business_interest_type"));
+                        if (!Check.isNull(targetJson.getJSONArray("business_interest"))) {
+                            groupTarget.setBusinessInterest(targetJson.getJSONArray("business_interest").toJSONString());
+                        }
+
+                        if (!Check.isNull(targetJson.getJSONArray("fans_star"))) {
+                            groupTarget.setFansStar(targetJson.getJSONArray("fans_star").toJSONString());
+                        }
+
+                        if (!Check.isNull(targetJson.getJSONArray("interest_video"))) {
+                            groupTarget.setInterestVideo(targetJson.getJSONArray("interest_video").toJSONString());
+                        }
+
+                        if (!Check.isNull(targetJson.getJSONArray("app_interest"))) {
+                            groupTarget.setAppInterest(targetJson.getJSONArray("app_interest").toJSONString());
+                        }
+
+                        if (!Check.isNull(targetJson.getJSONArray("app_ids"))) {
+                            groupTarget.setAppIds(targetJson.getJSONArray("app_ids").toJSONString());
+                        }
+
+                        if (!Check.isNull(targetJson.getJSONArray("population"))) {
+                            groupTarget.setPopulation(targetJson.getJSONArray("population").toJSONString());
+                        }
+
+                        if (!Check.isNull(targetJson.getJSONArray("exclude_population"))) {
+                            groupTarget.setExcludePopulation(targetJson.getJSONArray("exclude_population").toJSONString());
+                        }
+
+                        JSONObject intelliExtend = targetJson.getJSONObject("intelli_extend");
+                        if (!Check.isNull(intelliExtend)) {
+                            groupTarget.setIsOpen(intelliExtend.getInteger("is_open"));
+                            groupTarget.setNoAgeBreak(intelliExtend.getInteger("no_age_break"));
+                            groupTarget.setNoGenderBreak(intelliExtend.getInteger("no_gender_break"));
+                            groupTarget.setNoAreaBreak(intelliExtend.getInteger("no_area_break"));
+
+                        }
+                        targets.add(groupTarget);
+                    }
+                }
+            }
+            if (!Check.isNull(groups)) {
+                groupMapper.replaceBatch(groups);
+            }
+            if (!Check.isNull(schedules)) {
+                shouScheduleMapper.replaceBatch(schedules);
+            }
+            if (!Check.isNull(appInfos)) {
+                appInfoMapper.replaceBatch(appInfos);
+            }
+            if (!Check.isNull(targets)) {
+                groupTargetMapper.replaceBatch(targets);
+            }
+        }
+    }
 
 }

+ 3 - 115
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/impl/KuaishouInterfaceServiceImpl.java

@@ -1244,9 +1244,6 @@ public class KuaishouInterfaceServiceImpl implements IKuaishouInterfaceService {
     private void addGroup(Long advertiserId, JSONArray details) {
         if (!Check.isNull(details)) {
             List<KuaiShouGroup> groups = new ArrayList<>();
-            List<KuaiShouAppInfo> appInfos = new ArrayList<>();
-            List<KuaiShouSchedule> schedules = new ArrayList<>();
-            List<KuaiShouGroupTarget> targets = new ArrayList<>();
             for (int i = 0; i < details.size(); i++) {
                 JSONObject detail = JSONObject.parseObject(details.get(i).toString());
                 if (!Check.isNull(detail)) {
@@ -1287,125 +1284,16 @@ public class KuaishouInterfaceServiceImpl implements IKuaishouInterfaceService {
                     group.setUpdateTime(new Date());
                     groups.add(group);
                     // 添加应用信息
-
-                    JSONObject diverseJson = detail.getJSONObject("diverse_data");
-                    if (!Check.isNull(diverseJson)) {
-                        KuaiShouAppInfo appInfo = new KuaiShouAppInfo();
-                        appInfo.setAccountId(advertiserId);
-                        appInfo.setAppName(diverseJson.getString("app_name"));
-                        appInfo.setAppPackageName(diverseJson.getString("app_package_name"));
-                        appInfo.setDeviceOsType(diverseJson.getInteger("device_os_type"));
-                        appInfo.setUnitId(unitId);
-                        appInfo.setCreateTime(new Date());
-                        appInfo.setUpdateTime(new Date());
-                        appInfos.add(appInfo);
-                    }
-                    //时间段信息
-                    JSONObject scheduleJson = detail.getJSONObject("schedule");
-                    if (!Check.isNull(scheduleJson)) {
-                        KuaiShouSchedule schedule = new KuaiShouSchedule();
-                        schedule.setUnitId(unitId);
-                        schedule.setAccountId(advertiserId);
-                        schedule.setMon(scheduleJson.getJSONArray("mon").toJSONString());
-                        schedule.setTues(scheduleJson.getJSONArray("tues").toJSONString());
-                        schedule.setWed(scheduleJson.getJSONArray("wed").toJSONString());
-                        schedule.setThur(scheduleJson.getJSONArray("thur").toJSONString());
-                        schedule.setFri(scheduleJson.getJSONArray("fri").toJSONString());
-                        schedule.setSat(scheduleJson.getJSONArray("sat").toJSONString());
-                        schedule.setSun(scheduleJson.getJSONArray("sun").toJSONString());
-                        schedule.setCreateTime(new Date());
-                        schedule.setUpdateTime(new Date());
-                        schedules.add(schedule);
-                    }
-
-                    JSONObject targetJson = detail.getJSONObject("target");
-                    if (!Check.isNull(targetJson)) {
-                        KuaiShouGroupTarget groupTarget = new KuaiShouGroupTarget();
-                        groupTarget.setAccountId(advertiserId);
-                        groupTarget.setUnitId(unitId);
-
-                        if (!Check.isNull(targetJson.getJSONArray("region"))) {
-                            groupTarget.setRegion(targetJson.getJSONArray("region").toJSONString());
-                        }
-
-                        JSONObject age = targetJson.getJSONObject("age");
-                        if (!Check.isNull(age)) {
-                            groupTarget.setAgeMin(age.getInteger("min"));
-                            groupTarget.setAgeMax(age.getInteger("max"));
-                        }
-                        if (!Check.isNull(targetJson.getJSONArray("ages_range"))) {
-                            groupTarget.setAgesRange(targetJson.getJSONArray("ages_range").toJSONString());
-                        }
-
-                        groupTarget.setGender(targetJson.getInteger("gender"));
-                        groupTarget.setPlatformOs(targetJson.getInteger("platform_os"));
-                        groupTarget.setAndroidOsv(targetJson.getInteger("android_osv"));
-                        groupTarget.setIosOsv(targetJson.getInteger("ios_osv"));
-                        groupTarget.setNetwork(targetJson.getInteger("network"));
-                        if (!Check.isNull(targetJson.getJSONArray("device_brand"))) {
-                            groupTarget.setDeviceBrand(targetJson.getJSONArray("device_brand").toJSONString());
-                        }
-
-                        if (!Check.isNull(targetJson.getJSONArray("device_price"))) {
-                            groupTarget.setDevicePrice(targetJson.getJSONArray("device_price").toJSONString());
-                        }
-
-                        groupTarget.setBusinessInterestType(targetJson.getInteger("business_interest_type"));
-                        if (!Check.isNull(targetJson.getJSONArray("business_interest"))) {
-                            groupTarget.setBusinessInterest(targetJson.getJSONArray("business_interest").toJSONString());
-                        }
-
-                        if (!Check.isNull(targetJson.getJSONArray("fans_star"))) {
-                            groupTarget.setFansStar(targetJson.getJSONArray("fans_star").toJSONString());
-                        }
-
-                        if (!Check.isNull(targetJson.getJSONArray("interest_video"))) {
-                            groupTarget.setInterestVideo(targetJson.getJSONArray("interest_video").toJSONString());
-                        }
-
-                        if (!Check.isNull(targetJson.getJSONArray("app_interest"))) {
-                            groupTarget.setAppInterest(targetJson.getJSONArray("app_interest").toJSONString());
-                        }
-
-                        if (!Check.isNull(targetJson.getJSONArray("app_ids"))) {
-                            groupTarget.setAppIds(targetJson.getJSONArray("app_ids").toJSONString());
-                        }
-
-                        if (!Check.isNull(targetJson.getJSONArray("population"))) {
-                            groupTarget.setPopulation(targetJson.getJSONArray("population").toJSONString());
-                        }
-
-                        if (!Check.isNull(targetJson.getJSONArray("exclude_population"))) {
-                            groupTarget.setExcludePopulation(targetJson.getJSONArray("exclude_population").toJSONString());
-                        }
-
-                        JSONObject intelliExtend = targetJson.getJSONObject("intelli_extend");
-                        if (!Check.isNull(intelliExtend)) {
-                            groupTarget.setIsOpen(intelliExtend.getInteger("is_open"));
-                            groupTarget.setNoAgeBreak(intelliExtend.getInteger("no_age_break"));
-                            groupTarget.setNoGenderBreak(intelliExtend.getInteger("no_gender_break"));
-                            groupTarget.setNoAreaBreak(intelliExtend.getInteger("no_area_break"));
-
-                        }
-                        targets.add(groupTarget);
-                    }
                 }
             }
             if (!Check.isNull(groups)) {
                 groupMapper.replaceBatch(groups);
             }
-            if (!Check.isNull(schedules)) {
-                shouScheduleMapper.replaceBatch(schedules);
-            }
-            if (!Check.isNull(appInfos)) {
-                appInfoMapper.replaceBatch(appInfos);
-            }
-            if (!Check.isNull(targets)) {
-                groupTargetMapper.replaceBatch(targets);
-            }
         }
     }
 
+
+
     @Override
     public void getCreativeList(CtopOauthToken token, Date startDate, Date endDate) {
         getCreativeListByPage(token.getAccessToken(), token.getAccountId(), startDate, endDate, 1);
@@ -3040,7 +2928,7 @@ public class KuaishouInterfaceServiceImpl implements IKuaishouInterfaceService {
 
     }
 
-    private static ExecutorService executorService = Executors.newFixedThreadPool(5);
+    private static ExecutorService executorService = Executors.newFixedThreadPool(10);
 
     @Override
     public void getAsyncCreativeList(Long accountId, String accessToken, Long campaignId) {