yangzian 3 rokov pred
rodič
commit
d7995e1ef0

+ 11 - 0
job-bytedance/pom.xml

@@ -166,6 +166,17 @@
             <version>1.0.2</version>
         </dependency>
 
+        <!-- Redis -->
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-data-redis</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-pool2</artifactId>
+            <version>2.6.2</version>
+        </dependency>
+
     </dependencies>
     <build>
         <plugins>

+ 26 - 0
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/impl/ByteDanceAdvertiserDataServiceImpl.java

@@ -7,8 +7,10 @@ import cn.com.ctop.job.bytedance.data.mapper.*;
 import cn.com.ctop.job.bytedance.data.service.IByteDanceAdvertiserDataService;
 import cn.com.ctop.job.bytedance.data.service.IBytedanceAgentReportDailyService;
 import cn.com.ctop.job.bytedance.data.service.IBytedanceAgentReportHourlyService;
+import cn.com.ctop.job.bytedance.data.utils.Check;
 import cn.com.ctop.job.bytedance.data.utils.DateUtils;
 import cn.com.ctop.job.bytedance.data.utils.HttpUtils;
+import cn.com.ctop.job.bytedance.utils.RedisUtil;
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
@@ -25,6 +27,7 @@ import org.springframework.stereotype.Service;
 import javax.annotation.Resource;
 import java.text.SimpleDateFormat;
 import java.util.*;
+import java.util.concurrent.TimeUnit;
 
 /**
  * @author zzy
@@ -64,12 +67,19 @@ public class ByteDanceAdvertiserDataServiceImpl implements IByteDanceAdvertiserD
     @Resource
     BytedanceAgentReportHourlyMapper bytedanceAgentReportHourlyMapper;
 
+    @Autowired
+    private RedisUtil redisUtil;
+
+
     /**
      * ------------------------------------------------------------------获取广告计划
      */
 
     public void getAd(OauthToken token, int pageNum, String ids, String date, String updateDate) {
         log.info("开始获取广告计划信息 accountID:{},page:{}", token.getAccountId(), pageNum);
+
+        List<Object> adAccountList = redisUtil.getList("AD"+updateDate,null);
+
         JSONArray getIds = null;
         if (null != ids && !"".equals(ids)) {
             String[] idString = ids.split(",");
@@ -108,10 +118,26 @@ public class ByteDanceAdvertiserDataServiceImpl implements IByteDanceAdvertiserD
                     "获取广告计划信息接口异常==》accountId:{},message:{}",
                     token.getAccountId(),
                     resultObject.getString("message"));
+            //请求过于频繁
+            if (40100 == code){
+                log.info("广告计划请求频繁,code:{},准备重试,accountId:{},错误消息:{}",code,token.getAccountId(),resultObject.getString("message"));
+                //添加重试
+                if (Check.isNull(adAccountList)){
+                    redisUtil.add("AD"+updateDate,Arrays.asList(token.getAccountId()), 1, TimeUnit.HOURS);
+                }else {
+                    adAccountList.add(token.getAccountId());
+                    redisUtil.add("AD"+updateDate,adAccountList, 1, TimeUnit.HOURS);
+                }
+            }
             return;
         }
         JSONArray data = resultObject.getJSONObject("data").getJSONArray("list");
+        if (adAccountList.contains(token.getAccountId())){
+            adAccountList.remove(token.getAccountId());
+            redisUtil.add("AD"+updateDate,adAccountList, 1, TimeUnit.HOURS);
+        }
         if (null == data || data.isEmpty()) {
+            //log.info("广告计划,accountId:{}暂无数据",token.getAccountId());
             return;
         }
         List<BytedanceAdGet> list = new ArrayList<>();

+ 56 - 2
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedancePlanLoadJob.java

@@ -7,9 +7,11 @@ import cn.com.ctop.job.bytedance.data.service.IOauthTokenService;
 import cn.com.ctop.job.bytedance.data.utils.Check;
 import cn.com.ctop.job.bytedance.data.utils.DateUtils;
 import cn.com.ctop.job.bytedance.data.utils.HttpUtils;
+import cn.com.ctop.job.bytedance.utils.RedisUtil;
 import com.xxl.job.core.context.XxlJobHelper;
 import com.xxl.job.core.handler.annotation.XxlJob;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
@@ -17,6 +19,7 @@ import java.text.SimpleDateFormat;
 import java.util.*;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 
 /**
  * 广告计划定时任务
@@ -34,6 +37,57 @@ public class BytedancePlanLoadJob {
     @Autowired
     private IByteDanceAdvertiserDataService advertiserDataService;
 
+    @Autowired
+    private RedisUtil redisUtil;
+
+    /**
+     * 获取广告计划列表信息-遗漏数据补充
+     *
+     * @throws Exception
+     */
+    @XxlJob("bytedancePlanLoadJobSupplement")
+    public void bytedancePlanLoadJobSupplement() throws Exception {
+
+        String nowDate = DateUtils.getNowDate("yyyy-MM-dd");
+        String yesterda = DateUtils.getAnotherDay("yyyy-MM-dd", nowDate, -1);
+        //添加重试
+        List<Object> adAccountList;
+        adAccountList = redisUtil.getList("AD"+yesterda,null);
+        for (Object accountId : adAccountList) {
+
+            log.info("广告计划遗漏数据补充,日期:{},accountId:{}",yesterda,adAccountList.toString());
+            String token = tokenService.getByAccountId(Long.valueOf(String.valueOf(accountId)));
+            if (Check.isNull(token)) {
+                log.error("此账户未获取到相关token,accountId:{}", accountId);
+                return;
+            }
+
+            OauthToken oauthToken = new OauthToken();
+            oauthToken.setAccessToken(token);
+            oauthToken.setAccountId(Long.valueOf(String.valueOf(accountId)));
+            advertiserDataService.getAdvertiserPlan(oauthToken, "", null, yesterda);
+
+            adAccountList = redisUtil.getList("AD"+yesterda,null);
+
+            if (adAccountList.size() == 0){
+                redisUtil.delete("AD"+yesterda);
+            }
+
+            try {
+                Thread.sleep(1000*10);
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+
+        }
+
+
+
+    }
+
+
+
+
     /**
      * 获取广告计划列表信息
      *
@@ -54,12 +108,12 @@ public class BytedancePlanLoadJob {
             }
             String nowDate = DateUtils.getNowDate("yyyy-MM-dd");
             String yesterda = DateUtils.getAnotherDay("yyyy-MM-dd", nowDate, -1);
-            String yesterda2 = DateUtils.getAnotherDay("yyyy-MM-dd", nowDate, -2);
+            //String yesterda2 = DateUtils.getAnotherDay("yyyy-MM-dd", nowDate, -2);
             OauthToken oauthToken = new OauthToken();
             oauthToken.setAccessToken(token);
             oauthToken.setAccountId(Long.parseLong(param));
             advertiserDataService.getAdvertiserPlan(oauthToken, "", null, yesterda);
-            advertiserDataService.getAdvertiserPlan(oauthToken, "", null, yesterda2);
+            //advertiserDataService.getAdvertiserPlan(oauthToken, "", null, yesterda2);
 
         //advertiserDataService.getAdvertiserPlan(oauthToken, "", null, yesterda);
     }

+ 233 - 0
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/utils/RedisUtil.java

@@ -0,0 +1,233 @@
+package cn.com.ctop.job.bytedance.utils;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.ibatis.cursor.Cursor;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.cache.annotation.CachingConfigurerSupport;
+import org.springframework.context.annotation.Bean;
+import org.springframework.data.redis.connection.RedisConnectionFactory;
+import org.springframework.data.redis.core.RedisCallback;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.data.redis.core.ScanOptions;
+import org.springframework.data.redis.core.StringRedisTemplate;
+import org.springframework.data.redis.serializer.RedisSerializer;
+import org.springframework.data.redis.serializer.StringRedisSerializer;
+import org.springframework.stereotype.Component;
+import org.springframework.util.CollectionUtils;
+import org.springframework.util.StringUtils;
+
+import javax.annotation.Resource;
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * redis 工具类
+ * @Author Scott
+ *
+ */
+@Component
+@Slf4j
+public class RedisUtil {
+
+
+	@Autowired
+	private StringRedisTemplate redisTemplate;
+
+
+	private final String DEFAULT_KEY_PREFIX = "";
+	private final int EXPIRE_TIME = 1;
+	private final TimeUnit EXPIRE_TIME_TYPE = TimeUnit.DAYS;
+
+
+	/**
+	 * 数据缓存至redis
+	 *
+	 * @param key
+	 * @param value
+	 * @return
+	 */
+	public <K, V> void add(K key, V value) {
+		try {
+			if (value != null) {
+				redisTemplate
+						.opsForValue()
+						.set(DEFAULT_KEY_PREFIX + key, JSON.toJSONString(value));
+			}
+		} catch (Exception e) {
+			log.error(e.getMessage(), e);
+			throw new RuntimeException("数据缓存至redis失败");
+		}
+	}
+
+	/**
+	 * 数据缓存至redis并设置过期时间
+	 *
+	 * @param key
+	 * @param value
+	 * @return
+	 */
+	public <K, V> void add(K key, V value, long timeout, TimeUnit unit) {
+		try {
+			if (value != null) {
+				redisTemplate
+						.opsForValue()
+						.set(DEFAULT_KEY_PREFIX + key, JSON.toJSONString(value), timeout, unit);
+			}
+		} catch (Exception e) {
+			log.error(e.getMessage(), e);
+			throw new RuntimeException("数据缓存至redis失败");
+		}
+	}
+
+	/**
+	 * 写入 hash-set,已经是key-value的键值,不能再写入为hash-set
+	 *
+	 * @param key    must not be {@literal null}.
+	 * @param subKey must not be {@literal null}.
+	 * @param value  写入的值
+	 */
+	public <K, SK, V> void addHashCache(K key, SK subKey, V value) {
+		redisTemplate.opsForHash().put(DEFAULT_KEY_PREFIX + key, subKey, value);
+	}
+
+	/**
+	 * 写入 hash-set,并设置过期时间
+	 *
+	 * @param key    must not be {@literal null}.
+	 * @param subKey must not be {@literal null}.
+	 * @param value  写入的值
+	 */
+	public <K, SK, V> void addHashCache(K key, SK subKey, V value, long timeout, TimeUnit unit) {
+		redisTemplate.opsForHash().put(DEFAULT_KEY_PREFIX + key, subKey, value);
+		redisTemplate.expire(DEFAULT_KEY_PREFIX + key, timeout, unit);
+	}
+
+	/**
+	 * 获取 hash-setvalue
+	 *
+	 * @param key    must not be {@literal null}.
+	 * @param subKey must not be {@literal null}.
+	 */
+	public <K, SK> Object getHashCache(K key, SK subKey) {
+		return  redisTemplate.opsForHash().get(DEFAULT_KEY_PREFIX + key, subKey);
+	}
+
+
+	/**
+	 * 从redis中获取缓存数据,转成对象
+	 *
+	 * @param key   must not be {@literal null}.
+	 * @param clazz 对象类型
+	 * @return
+	 */
+	public <K, V> V getObject(K key, Class<V> clazz) {
+		String value = this.get(key);
+		V result = null;
+		if (!StringUtils.isEmpty(value)) {
+			result = JSONObject.parseObject(value, clazz);
+		}
+		return result;
+	}
+
+	/**
+	 * 从redis中获取缓存数据,转成list
+	 *
+	 * @param key   must not be {@literal null}.
+	 * @param clazz 对象类型
+	 * @return
+	 */
+	public <K, V> List<V> getList(K key, Class<V> clazz) {
+		String value = this.get(key);
+		List<V> result = Collections.emptyList();
+		if (!StringUtils.isEmpty(value)) {
+			result = JSONArray.parseArray(value, clazz);
+		}
+		return result;
+	}
+
+	/**
+	 * 功能描述:Get the value of {@code key}.
+	 *
+	 * @param key must not be {@literal null}.
+	 * @return java.lang.String
+	 * @date 2021/9/19
+	 **/
+	public <K> String get(K key) {
+		String value;
+		try {
+			value = redisTemplate.opsForValue().get(DEFAULT_KEY_PREFIX + key);
+		} catch (Exception e) {
+			log.error(e.getMessage(), e);
+			throw new RuntimeException("从redis缓存中获取缓存数据失败");
+		}
+		return value;
+	}
+
+	/**
+	 * 删除key
+	 */
+	public void delete(String key) {
+		redisTemplate.delete(key);
+	}
+
+	/**
+	 * 批量删除key
+	 */
+	public void delete(Collection<String> keys) {
+		redisTemplate.delete(keys);
+	}
+
+	/**
+	 * 序列化key
+	 */
+	public byte[] dump(String key) {
+		return redisTemplate.dump(key);
+	}
+
+	/**
+	 * 是否存在key
+	 */
+	public Boolean hasKey(String key) {
+		return redisTemplate.hasKey(key);
+	}
+
+	/**
+	 * 设置过期时间
+	 */
+	public Boolean expire(String key, long timeout, TimeUnit unit) {
+		return redisTemplate.expire(key, timeout, unit);
+	}
+
+	/**
+	 * 设置过期时间
+	 */
+	public Boolean expireAt(String key, Date date) {
+		return redisTemplate.expireAt(key, date);
+	}
+
+
+	/**
+	 * 移除 key 的过期时间,key 将持久保持
+	 */
+	public Boolean persist(String key) {
+		return redisTemplate.persist(key);
+	}
+
+	/**
+	 * 返回 key 的剩余的过期时间
+	 */
+	public Long getExpire(String key, TimeUnit unit) {
+		return redisTemplate.getExpire(key, unit);
+	}
+
+	/**
+	 * 返回 key 的剩余的过期时间
+	 */
+	public Long getExpire(String key) {
+		return redisTemplate.getExpire(key);
+	}
+
+}