Browse Source

视频物料-遗漏数据补充

yangzian 3 years ago
parent
commit
4e25630699

+ 12 - 0
job-kuaishou/pom.xml

@@ -161,6 +161,18 @@
             <version>1.0.2</version>
         </dependency>
 
+
+        <!-- Redis -->
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-data-redis</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-pool2</artifactId>
+            <version>2.6.2</version>
+        </dependency>
+
     </dependencies>
     <build>
         <plugins>

+ 30 - 4
job-kuaishou/src/main/java/cn/com/ctop/job/kuaishou/data/service/impl/KuaishouVideoListServiceImpl.java

@@ -7,6 +7,7 @@ import cn.com.ctop.job.kuaishou.data.mapper.KuaishouVideoListMapper;
 import cn.com.ctop.job.kuaishou.data.service.IKuaishouVideoListService;
 import cn.com.ctop.job.kuaishou.data.utils.Check;
 import cn.com.ctop.job.kuaishou.data.utils.HttpUtils;
+import cn.com.ctop.job.kuaishou.utils.RedisUtil;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
@@ -16,10 +17,8 @@ import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
 import javax.annotation.Resource;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
+import java.util.concurrent.TimeUnit;
 
 /**
  * 快手-素材列表
@@ -37,8 +36,18 @@ public class KuaishouVideoListServiceImpl extends ServiceImpl<KuaishouVideoListM
     @Resource
     private KuaishouVideoListMapper videoListMapper;
 
+
+    @Autowired
+    private RedisUtil redisUtil;
+
+
     @Override
     public void getVideoList(Long advertiserId, String accessToken, String startDate, String endDate, int page) {
+
+        log.info("开始获取所有视频信息 accountID:{},page:{}", advertiserId, page);
+
+        List<Object> adAccountList = redisUtil.getList("kuaiShou:allVideo:list",null);
+
         String url = postUrl + KuaishouConstant.VIDEO_LIST;
         Map<String, String> headers = new HashMap<>();
         headers.put("Access-Token", accessToken);
@@ -60,6 +69,23 @@ public class KuaishouVideoListServiceImpl extends ServiceImpl<KuaishouVideoListM
         String message = resultJson.getString("message");
         if (null == code || code != 0) {
             log.error("获取快手视频列表数据异常:{},accountId:{}", message, advertiserId);
+
+            //请求过于频繁
+            if (400001 == code){
+                log.info("定时任务-->>allVideoList<<<<----获取所有快手视频请求频繁,code:{},准备重试,accountId:{},错误消息:{}",code,advertiserId,message);
+                //添加重试
+                if (Check.isNull(adAccountList)){
+                    redisUtil.add("kuaiShou:allVideo:list", Arrays.asList(advertiserId), 1, TimeUnit.HOURS);
+                }else {
+                    adAccountList.add(advertiserId);
+                    redisUtil.add("kuaiShou:allVideo:list",adAccountList, 1, TimeUnit.HOURS);
+                }
+            }else {
+                if (adAccountList.contains(advertiserId)){
+                    adAccountList.remove(advertiserId);
+                    redisUtil.add("kuaiShou:allVideo:list",advertiserId, 1, TimeUnit.HOURS);
+                }
+            }
             return;
         }
         JSONArray details = resultJson.getJSONObject("data").getJSONArray("details");

+ 60 - 0
job-kuaishou/src/main/java/cn/com/ctop/job/kuaishou/handler/VideoListJob.java

@@ -1,15 +1,20 @@
 package cn.com.ctop.job.kuaishou.handler;
 
+import cn.com.ctop.job.kuaishou.data.entity.OauthToken;
 import cn.com.ctop.job.kuaishou.data.service.IKuaishouVideoListService;
 import cn.com.ctop.job.kuaishou.data.service.IOauthTokenService;
 import cn.com.ctop.job.kuaishou.data.utils.Check;
 import cn.com.ctop.job.kuaishou.data.utils.DateUtils;
+import cn.com.ctop.job.kuaishou.utils.RedisUtil;
 import com.xxl.job.core.context.XxlJobHelper;
 import com.xxl.job.core.handler.annotation.XxlJob;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 
@@ -107,4 +112,59 @@ public class VideoListJob {
     }
 
 
+
+    @Autowired
+    private RedisUtil redisUtil;
+
+    /**
+     * 所有视频物料
+     *
+     * @throws Exception
+     */
+    @XxlJob("allVideoListSupplement")
+    public void allVideoListSupplement() throws Exception {
+        //获取视频物料信息异常的key
+        Set<String> set = redisUtil.kesy("kuaiShou:allVideo:list*");
+        if (Check.isNull(set)){
+            return;
+        }
+        List<String> list = new ArrayList<>(set);
+        for (String key : list) {
+            //添加重试
+            List<Object> accountList;
+            accountList = redisUtil.getList(key,null);
+            //遍历key下的所有信息
+            for (Object accountId : accountList) {
+                log.info("快手视频遗漏数据补充,accountId:{}",accountId);
+                String token = tokenService.getByAccountId(Long.valueOf(String.valueOf(accountId)));
+                if (Check.isNull(token)) {
+                    log.error("此账户未获取到相关token,accountId:{}", accountId);
+                    return;
+                }
+
+                videoListService.getVideoList(Long.valueOf(accountId.toString()), token, null, null, 1);
+                accountList = redisUtil.getList(key,null);
+
+                if (accountList.size() == 0){
+                    redisUtil.delete(key);
+                }
+
+                try {
+                    Thread.sleep(1000*2);
+                } catch (InterruptedException e) {
+                    e.printStackTrace();
+                }
+
+            }
+
+        }
+    }
+
+
+
+
+
+
+
+
 }

+ 232 - 0
job-kuaishou/src/main/java/cn/com/ctop/job/kuaishou/utils/RedisUtil.java

@@ -0,0 +1,232 @@
+package cn.com.ctop.job.kuaishou.utils;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.data.redis.core.StringRedisTemplate;
+import org.springframework.stereotype.Component;
+import org.springframework.util.StringUtils;
+
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * redis 工具类
+ * @Author Scott
+ *
+ */
+@Component
+@Slf4j
+public class RedisUtil {
+
+
+	@Autowired
+	private StringRedisTemplate redisTemplate;
+
+
+	private final String DEFAULT_KEY_PREFIX = "";
+	private final int EXPIRE_TIME = 1;
+	private final TimeUnit EXPIRE_TIME_TYPE = TimeUnit.DAYS;
+
+
+	/**
+	 * 数据缓存至redis
+	 *
+	 * @param key
+	 * @param value
+	 * @return
+	 */
+	public <K, V> void add(K key, V value) {
+		try {
+			if (value != null) {
+				redisTemplate
+						.opsForValue()
+						.set(DEFAULT_KEY_PREFIX + key, JSON.toJSONString(value));
+			}
+		} catch (Exception e) {
+			log.error(e.getMessage(), e);
+			throw new RuntimeException("数据缓存至redis失败");
+		}
+	}
+
+	/**
+	 * 数据缓存至redis并设置过期时间
+	 *
+	 * @param key
+	 * @param value
+	 * @return
+	 */
+	public <K, V> void add(K key, V value, long timeout, TimeUnit unit) {
+		try {
+			if (value != null) {
+				redisTemplate
+						.opsForValue()
+						.set(DEFAULT_KEY_PREFIX + key, JSON.toJSONString(value), timeout, unit);
+			}
+		} catch (Exception e) {
+			log.error(e.getMessage(), e);
+			throw new RuntimeException("数据缓存至redis失败");
+		}
+	}
+
+	/**
+	 * 写入 hash-set,已经是key-value的键值,不能再写入为hash-set
+	 *
+	 * @param key    must not be {@literal null}.
+	 * @param subKey must not be {@literal null}.
+	 * @param value  写入的值
+	 */
+	public <K, SK, V> void addHashCache(K key, SK subKey, V value) {
+		redisTemplate.opsForHash().put(DEFAULT_KEY_PREFIX + key, subKey, value);
+	}
+
+	/**
+	 * 写入 hash-set,并设置过期时间
+	 *
+	 * @param key    must not be {@literal null}.
+	 * @param subKey must not be {@literal null}.
+	 * @param value  写入的值
+	 */
+	public <K, SK, V> void addHashCache(K key, SK subKey, V value, long timeout, TimeUnit unit) {
+		redisTemplate.opsForHash().put(DEFAULT_KEY_PREFIX + key, subKey, value);
+		redisTemplate.expire(DEFAULT_KEY_PREFIX + key, timeout, unit);
+	}
+
+	/**
+	 * 获取 hash-setvalue
+	 *
+	 * @param key    must not be {@literal null}.
+	 * @param subKey must not be {@literal null}.
+	 */
+	public <K, SK> Object getHashCache(K key, SK subKey) {
+		return  redisTemplate.opsForHash().get(DEFAULT_KEY_PREFIX + key, subKey);
+	}
+
+
+	/**
+	 * 从redis中获取缓存数据,转成对象
+	 *
+	 * @param key   must not be {@literal null}.
+	 * @param clazz 对象类型
+	 * @return
+	 */
+	public <K, V> V getObject(K key, Class<V> clazz) {
+		String value = this.get(key);
+		V result = null;
+		if (!StringUtils.isEmpty(value)) {
+			result = JSONObject.parseObject(value, clazz);
+		}
+		return result;
+	}
+
+	/**
+	 * 从redis中获取缓存数据,转成list
+	 *
+	 * @param key   must not be {@literal null}.
+	 * @param clazz 对象类型
+	 * @return
+	 */
+	public <K, V> List<V> getList(K key, Class<V> clazz) {
+		String value = this.get(key);
+		List<V> result = Collections.emptyList();
+		if (!StringUtils.isEmpty(value)) {
+			result = JSONArray.parseArray(value, clazz);
+		}
+		return result;
+	}
+
+	/**
+	 * 功能描述:Get the value of {@code key}.
+	 *
+	 * @param key must not be {@literal null}.
+	 * @return java.lang.String
+	 * @date 2021/9/19
+	 **/
+	public <K> String get(K key) {
+		String value;
+		try {
+			value = redisTemplate.opsForValue().get(DEFAULT_KEY_PREFIX + key);
+		} catch (Exception e) {
+			log.error(e.getMessage(), e);
+			throw new RuntimeException("从redis缓存中获取缓存数据失败");
+		}
+		return value;
+	}
+
+	/**
+	 * 删除key
+	 */
+	public void delete(String key) {
+		redisTemplate.delete(key);
+	}
+
+	/**
+	 * 批量删除key
+	 */
+	public void delete(Collection<String> keys) {
+		redisTemplate.delete(keys);
+	}
+
+	/**
+	 * 序列化key
+	 */
+	public byte[] dump(String key) {
+		return redisTemplate.dump(key);
+	}
+
+	/**
+	 * 是否存在key
+	 */
+	public Boolean hasKey(String key) {
+		return redisTemplate.hasKey(key);
+	}
+
+	/**
+	 * 设置过期时间
+	 */
+	public Boolean expire(String key, long timeout, TimeUnit unit) {
+		return redisTemplate.expire(key, timeout, unit);
+	}
+
+	/**
+	 * 设置过期时间
+	 */
+	public Boolean expireAt(String key, Date date) {
+		return redisTemplate.expireAt(key, date);
+	}
+
+
+	/**
+	 * 移除 key 的过期时间,key 将持久保持
+	 */
+	public Boolean persist(String key) {
+		return redisTemplate.persist(key);
+	}
+
+	/**
+	 * 返回 key 的剩余的过期时间
+	 */
+	public Long getExpire(String key, TimeUnit unit) {
+		return redisTemplate.getExpire(key, unit);
+	}
+
+	/**
+	 * 返回 key 的剩余的过期时间
+	 */
+	public Long getExpire(String key) {
+		return redisTemplate.getExpire(key);
+	}
+
+
+	/**
+	 *  返回 满足前缀 的key
+	 * @param pattern
+	 * @return
+	 */
+	public Set<String> kesy(String pattern){
+		return redisTemplate.keys(pattern);
+	}
+
+}

+ 26 - 1
job-kuaishou/src/main/resources/application.yml

@@ -85,6 +85,30 @@ spring:
           password: hcst@2021
           driver-class-name: com.mysql.jdbc.Driver
 
+  # redis 配置
+  redis:
+    # 地址
+    host: 192.168.0.193
+    # 端口,默认为6379
+    port: 6379
+    # 数据库索引
+    database: 0
+    # 密码
+    password: hcst@2022
+    # 连接超时时间
+    timeout: 1000s
+    lettuce:
+      pool:
+        # 连接池中的最小空闲连接
+        min-idle: 0
+        # 连接池中的最大空闲连接
+        max-idle: 10
+        # 连接池的最大数据库连接数
+        max-active: 10
+        # #连接池最大阻塞等待时间(使用负值表示没有限制)
+        max-wait: -1ms
+
+
 #mybatis plus 设置
 mybatis-plus:
   mapper-locations: classpath*:org/jeecg/modules/**/xml/*Mapper.xml,classpath*:cn/com/ctop/**/xml/*Mapper.xml
@@ -109,7 +133,8 @@ xxl:
   job:
     accessToken:
     executor:
-      appname: module-job-kuaishou
+#      appname: module-job-kuaishou
+      appname: module-job-test
       port: 9997
       ip:
       logpath: /data/applogs/xxl-job/jobhandler