穿山甲资源上传oss

job-bytedance/pom.xml

@@ -187,6 +187,11 @@
             <artifactId>commons-pool2</artifactId>
             <version>2.6.2</version>
         </dependency>
+        <dependency>
+            <groupId>com.aliyun.oss</groupId>
+            <artifactId>aliyun-sdk-oss</artifactId>
+            <version>3.16.0</version>
+        </dependency>
 
     </dependencies>
     <build>

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/controller/FeishuController.java

@@ -1,6 +1,8 @@
 package cn.com.ctop.job.bytedance.controller;
 
 import cn.com.ctop.job.bytedance.data.service.*;
+import cn.com.ctop.job.bytedance.handler.CsjResoureJob;
+import cn.com.ctop.job.bytedance.utils.OssUtil;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import io.swagger.annotations.Api;
@@ -28,14 +30,30 @@ public class FeishuController {
     IBindAppErrorWarnService bindAppErrorWarnService;
     @Autowired
     IBytedanceProductUpdateService bytedanceProductUpdateService;
+    @Autowired
+    IReportsByBusinessPushService reportsByBusinessPushService;
+    @Autowired
+    ICutGroupRankPushService cutGroupRankPushService;
+    @Autowired
+    OssUtil ossUtil;
+    @Autowired
+    CsjResoureJob csjResoureJob;
 
     @Autowired
     IUserLoginRecordService userLoginRecordService;
     @GetMapping("/test")
     public void test() throws Exception {
 //        bytedanceProductUpdateService.initNew();
-        bytedanceProductUpdateService.run();
+//        bytedanceProductUpdateService.run();
+//        reportsAdPushService.test();
+//        totalRankPushService.test();
+//        dateReportsPushService.preSend();
+//        optimizerRankPushService.send();
+//        unbindWarnPushService.send();
+//        cutGroupRankPushService.send();
+//        reportsByBusinessPushService.test();
 
+        csjResoureJob.csjResourceOssJob();
     }
 
 

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/entity/CsjAlbumResourceInfo.java

@@ -0,0 +1,17 @@
+package cn.com.ctop.job.bytedance.data.entity;
+
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+public class CsjAlbumResourceInfo {
+
+    private String resourceId;
+
+    private String albumId;
+
+    private String ossCoverImg;
+    //每个剧集的信息
+    private List<CsjAlbumSeqResourceInfo> seqList;
+}

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/entity/CsjAlbumSeqResourceInfo.java

@@ -0,0 +1,22 @@
+package cn.com.ctop.job.bytedance.data.entity;
+
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@NoArgsConstructor
+@Data
+public class CsjAlbumSeqResourceInfo {
+
+    private String albumId;
+    //第几集
+    private Integer seq;
+    //本集oss封面图
+    private String ossSeqImg;
+    //本集oss视频链接
+    private String ossSeqVideo;
+
+    public CsjAlbumSeqResourceInfo(String albumId, Integer seq) {
+        this.albumId = albumId;
+        this.seq = seq;
+    }
+}

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/mapper/CsjResourceMapper.java

@@ -0,0 +1,19 @@
+package cn.com.ctop.job.bytedance.data.mapper;
+
+import cn.com.ctop.job.bytedance.data.entity.CsjAlbumResourceInfo;
+import cn.com.ctop.job.bytedance.data.entity.CsjAlbumSeqResourceInfo;
+
+import java.util.List;
+
+public interface CsjResourceMapper {
+
+    List<CsjAlbumResourceInfo> getResources();
+
+    String getToken();
+
+    int updateAlbum(CsjAlbumResourceInfo album);
+
+    int updateSeq(CsjAlbumSeqResourceInfo albumSeq);
+
+    int batchUpdateAlbumSeq(List<CsjAlbumSeqResourceInfo> list);
+}

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/mapper/xml/CsjResourceMapper.xml

@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
+<mapper namespace="cn.com.ctop.job.bytedance.data.mapper.CsjResourceMapper">
+
+    <resultMap id="albumInfo" type="cn.com.ctop.job.bytedance.data.entity.CsjAlbumResourceInfo">
+        <id property="albumId" column="album_id"/>
+        <result property="resourceId" column="resource_id"/>
+        <result property="ossCoverImg" column="oss_cover_img"/>
+        <collection property="seqList" ofType="cn.com.ctop.job.bytedance.data.entity.CsjAlbumSeqResourceInfo">
+            <result property="ossSeqImg" column="seq_img"/>
+            <result property="ossSeqVideo" column="seq_video"/>
+            <result property="seq" column="seq"/>
+            <result property="albumId" column="album_id"/>
+        </collection>
+    </resultMap>
+    <update id="updateAlbum">
+        replace INTO thirtpart.csj_album_resource_info
+        (album_id, oss_cover_img)
+        VALUES
+        (#{albumId}, #{ossCoverImg})
+    </update>
+    <update id="batchUpdateAlbumSeq">
+        replace INTO thirtpart.csj_album_seq_resource_info
+        (album_id, seq, oss_cover_img, oss_video_url)
+        VALUES
+        <foreach collection="list" item="item" separator=",">
+            (#{item.albumId}, #{item.seq}, #{item.ossSeqImg}, #{item.ossSeqVideo})
+        </foreach>
+    </update>
+    <update id="updateSeq">
+        replace INTO thirtpart.csj_album_seq_resource_info
+        (album_id, seq, oss_cover_img, oss_video_url)
+        VALUES
+        (#{albumId}, #{seq}, #{ossSeqImg}, #{ossSeqVideo})
+    </update>
+
+    <select id="getResources" resultMap="albumInfo">
+        SELECT tb1.album_id, tb0.resource_id, tb2.oss_cover_img, tb3.seq, tb3.oss_cover_img as seq_img, tb3.oss_video_url as seq_video
+        FROM
+        thirtpart.csj_authorization_list_use tb0
+        LEFT JOIN thirtpart.csj_authorization_detail tb1 on tb0.resource_id = tb1.resource_id
+        LEFT JOIN thirtpart.csj_album_resource_info tb2 on tb1.album_id = tb2.album_id
+        LEFT JOIN thirtpart.csj_album_seq_resource_info tb3 on tb1.album_id = tb3.album_id
+        ORDER BY tb1.create_time DESC
+    </select>
+    <select id="getToken" resultType="java.lang.String">
+        select authorizer_access_token from thirtpart.thirt_app_token where auth_app_id='ttdf35a988403ffce301'
+    </select>
+</mapper>

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/ICsjResourceService.java

@@ -0,0 +1,12 @@
+package cn.com.ctop.job.bytedance.data.service;
+
+import cn.com.ctop.job.bytedance.data.entity.CsjAlbumResourceInfo;
+
+import java.util.List;
+
+public interface ICsjResourceService {
+
+    List<CsjAlbumResourceInfo> albumList();
+
+    String run(CsjAlbumResourceInfo album);
+}

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/impl/CsjResourceServiceImpl.java

@@ -0,0 +1,158 @@
+package cn.com.ctop.job.bytedance.data.service.impl;
+
+import cn.com.ctop.job.bytedance.data.entity.CsjAlbumResourceInfo;
+import cn.com.ctop.job.bytedance.data.entity.CsjAlbumSeqResourceInfo;
+import cn.com.ctop.job.bytedance.data.mapper.CsjResourceMapper;
+import cn.com.ctop.job.bytedance.data.service.ICsjResourceService;
+import cn.com.ctop.job.bytedance.data.utils.Check;
+import cn.com.ctop.job.bytedance.data.utils.HttpUtils;
+import cn.com.ctop.job.bytedance.utils.OssProperties;
+import cn.com.ctop.job.bytedance.utils.OssUtil;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import javax.annotation.Resource;
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Slf4j
+@Service
+public class CsjResourceServiceImpl implements ICsjResourceService {
+
+    @Resource
+    CsjResourceMapper csjResourceMapper;
+    @Autowired
+    OssUtil ossUtil;
+    @Autowired
+    OssProperties ossProperties;
+
+    @Override
+    public List<CsjAlbumResourceInfo> albumList() {
+        return csjResourceMapper.getResources();
+    }
+
+    @Override
+    public String run(CsjAlbumResourceInfo album) {
+        String resourceId = album.getResourceId();
+        String albumId = album.getAlbumId();
+        String token = csjResourceMapper.getToken();
+        if (Check.isNull(token)){
+            return "获取token失败";
+        }
+
+        String csjDownloadUrl = "https://open.douyin.com/api/resource_library/v1/resource/download/";
+        Map<String, String> headers = new HashMap<>();
+        headers.put("access-token", token);
+        headers.put("Content-Type", "application/json");
+        Map<String, Object> params = new HashMap<>();
+        params.put("resource_id", resourceId);
+        String res = HttpUtils.httpPostRequest(csjDownloadUrl, params, headers);
+        if (Check.isNull(res)) {
+            return "请求失败，返回结果为空";
+        }
+        JSONObject resObj = JSONObject.parseObject(res);
+        int err_no = resObj.getInteger("err_no");
+        if (err_no != 0) {
+            return "请求失败，错误码：" + err_no + ", 错误信息：" + resObj.getString("err_msg");
+        }
+        JSONObject data = resObj.getJSONObject("data");
+        JSONArray videoList = data.getJSONArray("video_info_list");
+        JSONArray coverList = data.getJSONArray("cover_info_list");
+
+
+        List<CsjAlbumSeqResourceInfo> seqList = album.getSeqList();
+        if (seqList == null){
+            seqList = new ArrayList<>();
+        }else {
+            seqList.removeIf(seq -> seq.getSeq() == null);
+        }
+        Map<Integer, CsjAlbumSeqResourceInfo> seqMap = seqList.stream().collect(Collectors.toMap(CsjAlbumSeqResourceInfo::getSeq, seq -> seq));
+        if (videoList != null && !videoList.isEmpty()){
+            for (int i = 0; i < videoList.size(); i++) {
+                JSONObject videoInfo = videoList.getJSONObject(i);
+                Integer seqNum = videoInfo.getInteger("seq");
+                String videoUrl = videoInfo.getString("url");
+                if (Check.isNull(videoUrl)) {
+                    log.error("第" + seqNum + "集视频链接为空");
+                }
+                CsjAlbumSeqResourceInfo seq = seqMap.get(seqNum);
+                if (seq == null){
+                    seq = new CsjAlbumSeqResourceInfo(albumId, seqNum);
+                    seqList.add(seq);
+                    seqMap.put(seqNum, seq);
+                }
+                if (StringUtils.isEmpty(seq.getOssSeqVideo())){
+                    String fileName = albumId + "_" + seqNum + "_video.mp4";
+                    String ossVideoUrl = uploadToOss(videoUrl, fileName);
+                    if (StringUtils.isNotEmpty(ossVideoUrl)){
+                        seq.setOssSeqVideo(ossVideoUrl);
+                        csjResourceMapper.updateSeq(seq);
+                    }
+                }
+            }
+        }
+        if (coverList != null && !coverList.isEmpty()){
+            for (int i = 0; i < coverList.size(); i++) {
+                JSONObject coverInfo = coverList.getJSONObject(i);
+                Integer seqNum = coverInfo.getInteger("seq");
+                String coverUrl = coverInfo.getString("url");
+
+                if (seqNum == null || seqNum == 0){
+                    if (StringUtils.isEmpty(album.getOssCoverImg())){
+                        String fileName = albumId + "_cover.jpg";
+                        String ossCoverImg = uploadToOss(coverUrl, fileName);
+                        if (StringUtils.isNotEmpty(ossCoverImg)){
+                            album.setOssCoverImg(ossCoverImg);
+                            csjResourceMapper.updateAlbum(album);
+                        }
+                    }
+                } else {
+                    CsjAlbumSeqResourceInfo seq = seqMap.get(seqNum);
+                    if (seq == null){
+                        seq = new CsjAlbumSeqResourceInfo(albumId, seqNum);
+                        seqList.add(seq);
+                        seqMap.put(seqNum, seq);
+                    }
+                    if (StringUtils.isEmpty(seq.getOssSeqImg())){
+                        String fileName = albumId + "_" + seqNum + "_image.jpg";
+                        String ossSeqImg = uploadToOss(coverUrl, fileName);
+                        if (StringUtils.isNotEmpty(ossSeqImg)){
+                            seq.setOssSeqImg(ossSeqImg);
+                            csjResourceMapper.updateSeq(seq);
+                        }
+                    }
+                }
+            }
+        }
+
+//        //update album, update seq
+//        csjResourceMapper.updateAlbum(album);
+//        if (album.getSeqList() != null && !album.getSeqList().isEmpty()){
+//            csjResourceMapper.batchUpdateAlbumSeq(album.getSeqList());
+//        }
+        log.info("处理资源成功: " + album.getAlbumId() + ", 共处理分集: " + seqList.size());
+
+        return "";
+    }
+
+    private String uploadToOss(String url, String fileName){
+        String objectKey = ossProperties.getCsjFileDir() + fileName;
+        String ossUrl = ossProperties.getCsjDomain() + objectKey;
+        try {
+            System.out.println("文件: "+fileName + ", from: " + url);
+//            return null;
+            ossUtil.uploadFromUrl(objectKey, url);
+            log.info("上传成功: " + ossUrl);
+            return ossUrl;
+        } catch (Exception e) {
+            log.error("上传失败: " + e.getMessage(), e);
+            return null;
+        }
+    }
+
+
+}

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/CsjResoureJob.java

@@ -0,0 +1,80 @@
+package cn.com.ctop.job.bytedance.handler;
+
+import cn.com.ctop.job.bytedance.data.entity.CsjAlbumResourceInfo;
+import cn.com.ctop.job.bytedance.data.entity.CsjAlbumSeqResourceInfo;
+import cn.com.ctop.job.bytedance.data.service.ICsjResourceService;
+import cn.com.ctop.job.bytedance.data.utils.Check;
+import com.xxl.job.core.handler.annotation.XxlJob;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+@Slf4j
+@Component
+public class CsjResoureJob {
+    @Autowired
+    ICsjResourceService csjResourceService;
+
+    private static ExecutorService linkRefreshExecutorService = Executors.newFixedThreadPool(5);
+
+    @XxlJob("csjResourceOssJob")
+    public void csjResourceOssJob(){
+
+        List<CsjAlbumResourceInfo> list = csjResourceService.albumList();
+        if (list == null || list.isEmpty()) {
+            log.info("没有需要处理的资源");
+            return; // 没有需要处理的资源
+        }
+        int count = 0;
+        for (CsjAlbumResourceInfo album : list) {
+            if (isDone(album)) {
+                log.info("资源已被处理，跳过: " + album.getAlbumId());
+                continue;
+            }
+            if (++count > 5){
+                break; //剧目数限制
+            }
+            linkRefreshExecutorService.submit(() -> {
+                try {
+                    String result = csjResourceService.run(album);
+                    if (result != null && !result.isEmpty()) {
+                        log.error("处理资源失败: " + result);
+                    } else {
+                        log.info("处理资源成功: " + album.getAlbumId());
+                    }
+                } catch (Exception e) {
+                    log.error("处理资源时发生异常: " + e.getMessage(), e);
+                }
+            });
+        }
+        log.info("本次上传剧目数：{}", count);
+    }
+
+
+    /**
+     * 是否已经处理完成
+     */
+    private boolean isDone(CsjAlbumResourceInfo album){
+        if (Check.isNull(album.getOssCoverImg())){
+            return false;
+        }
+        if (album.getSeqList() == null){ //todo 没有分集，是否为空
+            log.error("*资源没有分集信息: " + album.getAlbumId());
+            return false;
+        }
+        for (CsjAlbumSeqResourceInfo seq : album.getSeqList()){
+            if (Check.isNull(seq.getOssSeqVideo())){
+                return false;
+            }
+            if (Check.isNull(seq.getOssSeqImg())){
+                return false;
+            }
+        }
+        return true;
+    }
+
+}

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/utils/OssProperties.java

@@ -0,0 +1,24 @@
+package cn.com.ctop.job.bytedance.utils;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.stereotype.Component;
+
+/**
+ * oss 配置
+ */
+@Lazy
+@Setter
+@Getter
+@Component
+@ConfigurationProperties(prefix = "playlet.oss")
+public class OssProperties {
+    private String endpoint;
+    private String accessKey;
+    private String accessSecret;
+    private String bucketName;
+    private String csjDomain;
+    private String csjFileDir;
+}

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/utils/OssUtil.java

@@ -0,0 +1,112 @@
+package cn.com.ctop.job.bytedance.utils;
+import com.aliyun.oss.ClientException;
+import com.aliyun.oss.OSS;
+import com.aliyun.oss.OSSClientBuilder;
+import com.aliyun.oss.OSSException;
+import com.aliyun.oss.model.ObjectMetadata;
+import com.aliyun.oss.model.PutObjectRequest;
+import com.aliyun.oss.model.PutObjectResult;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.stereotype.Component;
+
+import java.io.InputStream;
+import java.net.URL;
+import java.net.HttpURLConnection;
+
+@Lazy
+@Slf4j
+@Component
+@RequiredArgsConstructor
+public class OssUtil {
+    @Getter
+    private final OssProperties ossProperties;
+
+    public void uploadFromUrl(String objectKey, String fileUrl) throws Exception {
+        // 创建OSS客户端
+        OSS ossClient = new OSSClientBuilder().build(ossProperties.getEndpoint(), ossProperties.getAccessKey(), ossProperties.getAccessSecret());
+
+        HttpURLConnection conn = null;
+        InputStream inputStream = null;
+
+        try {
+//            inputStream = new URL(fileUrl).openStream();
+//            // 创建PutObjectRequest对象。
+//            PutObjectRequest putObjectRequest = new PutObjectRequest(ossProperties.getBucketName(), objectKey, inputStream);
+//            // 创建PutObject请求。
+//            PutObjectResult result = ossClient.putObject(putObjectRequest);
+//            // 打开HTTP连接
+            URL url = new URL(fileUrl);
+            conn = (HttpURLConnection) url.openConnection();
+            conn.setRequestMethod("GET");
+            conn.setConnectTimeout(5000);  // 5秒连接超时
+            conn.setReadTimeout(30000);    // 30秒读取超时
+
+            // 获取输入流和元数据
+            inputStream = conn.getInputStream();
+            int contentLength = conn.getContentLength();
+            String contentType = conn.getContentType();
+
+            // 创建OSS元数据
+            ObjectMetadata metadata = new ObjectMetadata();
+            if (contentLength > 0) {
+                metadata.setContentLength(contentLength);
+            }
+            if (contentType != null) {
+                metadata.setContentType(contentType);
+            }
+
+            // 创建上传请求
+            PutObjectRequest putRequest = new PutObjectRequest(
+                    ossProperties.getBucketName(),
+                    objectKey,
+                    inputStream,
+                    metadata
+            );
+
+            // 执行上传
+            PutObjectResult result = ossClient.putObject(putRequest);
+            System.out.println("File uploaded to OSS: " + objectKey);
+            System.out.println(result);
+
+        } catch (OSSException oe) {
+            System.out.println("Caught an OSSException, which means your request made it to OSS, "
+                    + "but was rejected with an error response for some reason.");
+            System.out.println("Error Message:" + oe.getErrorMessage());
+            System.out.println("Error Code:" + oe.getErrorCode());
+            System.out.println("Request ID:" + oe.getRequestId());
+            System.out.println("Host ID:" + oe.getHostId());
+            throw oe;
+        } catch (ClientException ce) {
+            System.out.println("Caught an ClientException, which means the client encountered "
+                    + "a serious internal problem while trying to communicate with OSS, "
+                    + "such as not being able to access the network.");
+            System.out.println("Error Message:" + ce.getMessage());
+            throw ce;
+        } catch (Exception e){
+            throw e;
+        } finally {
+            // 关闭资源
+            if (inputStream != null) inputStream.close();
+            if (conn != null) conn.disconnect();
+            if (ossClient != null) ossClient.shutdown();
+        }
+    }
+
+    public void test() {
+        try {
+            String fileUrl = "https://p3-developer-sign.bytemaimg.com/tos-cn-i-o9yqv5mzf9/63af95e356f4f348b809b5b4a8f7462b~tplv-ke512zj2cu-jpg.image?lk3s=6ecb03b0&x-expires=1755601981&x-signature=L7y5AuCHlmkrtYyMybynp558wM8%3D";
+            String objectKey = "csj/test.jpg"; // OSS存储路径
+
+            uploadFromUrl(objectKey, fileUrl);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+}

job-bytedance/src/main/resources/application.yml

@@ -180,5 +180,14 @@ api:
   bytedance:
     api-prefix: https://ad.oceanengine.com/open_api
 
+playlet:
+  oss:
+    accessKey: LTAI5tAZjbTv3PtTMZHFAhkb
+    accessSecret: DK3uoycrIiUxJSol3gMNWmcOyzJqU8
+    endpoint: oss-cn-beijing-internal.aliyuncs.com
+#    endpoint: oss-cn-beijing.aliyuncs.com
+    bucketName: short-theatre-hr
+    csjDomain: https://oss.artlinkworld.cn/
+    csjFileDir: csj/