yumeng 5 dias atrás
pai
commit
bea53aebfb
16 arquivos alterados com 5 adições e 477 exclusões
  1. 0 20
      job-adq/src/main/java/cn/com/ctop/job/adq/data/service/impl/AdqAccountReportServiceImpl.java
  2. 0 7
      job-adq/src/main/java/cn/com/ctop/job/adq/data/service/impl/AdqAdIncomeServiceImpl.java
  3. 0 18
      job-adq/src/main/java/cn/com/ctop/job/adq/data/service/impl/AdqCreativeCreativeReportServiceImpl.java
  4. 0 6
      job-adq/src/main/java/cn/com/ctop/job/adq/data/service/impl/AdqFlowDetailsReportServiceImpl.java
  5. 0 4
      job-adq/src/main/java/cn/com/ctop/job/adq/data/service/impl/AdqMaterialReportServiceImpl.java
  6. 0 23
      job-adq/src/main/java/cn/com/ctop/job/adq/data/service/impl/AdqRetryReportServiceImpl.java
  7. 0 49
      job-adq/src/main/java/cn/com/ctop/job/adq/utils/DorisConfig.java
  8. 0 16
      job-adq/src/main/java/cn/com/ctop/job/adq/utils/doris/sink/DorisBufferConfig.java
  9. 0 51
      job-adq/src/main/java/cn/com/ctop/job/adq/utils/doris/sink/DorisMultiTableStreamLoadManager.java
  10. 0 7
      job-adq/src/main/java/cn/com/ctop/job/adq/utils/doris/sink/DorisSinkClient.java
  11. 0 17
      job-adq/src/main/java/cn/com/ctop/job/adq/utils/doris/sink/RecordWithMeta.java
  12. 0 94
      job-adq/src/main/java/cn/com/ctop/job/adq/utils/doris/sink/SimpleHttpDorisSinkClient.java
  13. 0 38
      job-adq/src/main/java/cn/com/ctop/job/adq/utils/doris/sink/StrictRedirectStrategy.java
  14. 0 96
      job-adq/src/main/java/cn/com/ctop/job/adq/utils/doris/sink/TableBufferManager.java
  15. 0 26
      job-adq/src/main/java/cn/com/ctop/job/adq/utils/doris/sink/TableKey.java
  16. 5 5
      job-adq/src/main/resources/application.yml

+ 0 - 20
job-adq/src/main/java/cn/com/ctop/job/adq/data/service/impl/AdqAccountReportServiceImpl.java

@@ -7,30 +7,20 @@ import cn.com.ctop.job.adq.data.service.IAdqAccountReportService;
 import cn.com.ctop.job.adq.data.utils.Check;
 import cn.com.ctop.job.adq.data.utils.FieldsConfig;
 import cn.com.ctop.job.adq.data.utils.HttpUtils;
-import cn.com.ctop.job.adq.data.utils.JsonUtils;
 import cn.com.ctop.job.adq.utils.RandomStringGenerator;
-import cn.com.ctop.job.adq.utils.doris.sink.DorisMultiTableStreamLoadManager;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.scheduling.annotation.EnableScheduling;
-import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Service;
 
-import javax.annotation.PreDestroy;
-import java.text.DecimalFormat;
 import java.time.Instant;
 import java.util.*;
 
 @Service
 @Slf4j
-@EnableScheduling
 public class AdqAccountReportServiceImpl implements IAdqAccountReportService {
     @Autowired
-    private DorisMultiTableStreamLoadManager dorisLoadManager;
-
-    @Autowired
     private AdqErrorLogMapper adqErrorLogMapper;
     @Autowired
     private AdqAccountReportMapper accountReportMapper;
@@ -77,11 +67,6 @@ public class AdqAccountReportServiceImpl implements IAdqAccountReportService {
                                 hourReport.setAdvertiserId(advertiserId);
                                 hourReport.setStatTimeDay(Long.valueOf(startDate.replace("-", "")));
                                 hourList.add(hourReport);
-
-                                String filterJSON = JsonUtils.toFilterJSON(hourReport);
-                                JSONObject json = JSONObject.parseObject(filterJSON);
-                                json.put("video_outer_play100_count",hourReport.getVideoOuterPlay100Count());
-                                dorisLoadManager.add("media_api", "adq_advertiser_report_hour", json);
                             }
                         }
                     }
@@ -152,11 +137,6 @@ public class AdqAccountReportServiceImpl implements IAdqAccountReportService {
                                 dailyReport.setAdvertiserId(advertiserId);
                                 dailyReport.setStatTimeDay(Long.valueOf(dataJson.getString("date").replace("-", "")));
                                 dailyList.add(dailyReport);
-
-                                String filterJSON = JsonUtils.toFilterJSON(dailyReport);
-                                JSONObject json = JSONObject.parseObject(filterJSON);
-                                json.put("video_outer_play100_count",dailyReport.getVideoOuterPlay100Count());
-                                dorisLoadManager.add("media_api", "adq_advertiser_report_daily", json);
                             }
                         }
                     }

+ 0 - 7
job-adq/src/main/java/cn/com/ctop/job/adq/data/service/impl/AdqAdIncomeServiceImpl.java

@@ -9,7 +9,6 @@ import cn.com.ctop.job.adq.data.utils.AlertUtils;
 import cn.com.ctop.job.adq.data.utils.Check;
 import cn.com.ctop.job.adq.data.utils.DateUtils;
 import cn.com.ctop.job.adq.data.utils.HttpUtils;
-import cn.com.ctop.job.adq.utils.doris.sink.DorisMultiTableStreamLoadManager;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.google.gson.Gson;
@@ -31,8 +30,6 @@ import java.util.Map;
 @Service
 @Slf4j
 public class AdqAdIncomeServiceImpl implements IAdqAdIncomeService {
-    @Autowired
-    private DorisMultiTableStreamLoadManager dorisLoadManager;
     @Resource
     private RedisTemplate redis2Template_db5;
     @Resource
@@ -112,8 +109,6 @@ public class AdqAdIncomeServiceImpl implements IAdqAdIncomeService {
                             report.setAdSlotName(adSlotMap.get(report.getAdSlot()));
                             //System.out.println(report);
                             reports.add(report);
-
-                            dorisLoadManager.add("media_api", "adq_publisher_adunit_general", report);
                         }
                     }
 
@@ -193,8 +188,6 @@ public class AdqAdIncomeServiceImpl implements IAdqAdIncomeService {
                             report.setAdSlotName(adSlotMap.get(report.getAdSlot()));
                             //System.out.println(report);
                             reports.add(report);
-
-                            dorisLoadManager.add("media_api", "adq_publisher_adunit_general", report);
                         }
                     }
 

+ 0 - 18
job-adq/src/main/java/cn/com/ctop/job/adq/data/service/impl/AdqCreativeCreativeReportServiceImpl.java

@@ -9,25 +9,18 @@ import cn.com.ctop.job.adq.data.service.IAdqCreativeReportService;
 import cn.com.ctop.job.adq.data.utils.Check;
 import cn.com.ctop.job.adq.data.utils.FieldsConfig;
 import cn.com.ctop.job.adq.data.utils.HttpUtils;
-import cn.com.ctop.job.adq.data.utils.JsonUtils;
 import cn.com.ctop.job.adq.utils.RandomStringGenerator;
-import cn.com.ctop.job.adq.utils.doris.sink.DorisMultiTableStreamLoadManager;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.scheduling.annotation.EnableScheduling;
-import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Service;
 
-import javax.annotation.PreDestroy;
-import java.text.DecimalFormat;
 import java.time.Instant;
 import java.util.*;
 
 @Service
 @Slf4j
-@EnableScheduling
 public class AdqCreativeCreativeReportServiceImpl implements IAdqCreativeReportService {
     @Autowired
     private AdqCreativeReportMapper adqReportMapper;
@@ -35,9 +28,6 @@ public class AdqCreativeCreativeReportServiceImpl implements IAdqCreativeReportS
     @Autowired
     private AdqErrorLogMapper adqErrorLogMapper;
 
-    @Autowired
-    private DorisMultiTableStreamLoadManager dorisLoadManager;
-
 
     @Override
     public void getAdqCreativeHour(Long advertiserId, String accessToken, String startDate, String endDate, Integer page, Integer isLog) {
@@ -80,10 +70,6 @@ public class AdqCreativeCreativeReportServiceImpl implements IAdqCreativeReportS
                                 hourReport.setAdvertiserId(advertiserId);
                                 hourReport.setStatTimeDay(Long.valueOf(startDate.replace("-", "")));
                                 hourList.add(hourReport);
-
-                                JSONObject filterJson = JSONObject.parseObject(JsonUtils.toFilterJSON(hourReport));
-                                filterJson.put("video_outer_play100_count",hourReport.getVideoOuterPlay100Count());
-                                dorisLoadManager.add("media_api","adq_creative_report_hour", filterJson);
                             }
                         }
                     }
@@ -165,10 +151,6 @@ public class AdqCreativeCreativeReportServiceImpl implements IAdqCreativeReportS
                                 hourReport.setStatTimeDay(Long.valueOf(dataJson.getString("date").replace("-", "")));
                                 dailies.add(hourReport);
 
-                                JSONObject filterJson = JSONObject.parseObject(JsonUtils.toFilterJSON(hourReport));
-                                filterJson.put("video_outer_play100_count",hourReport.getVideoOuterPlay100Count());
-                                dorisLoadManager.add("media_api","adq_creative_report_daily", filterJson);
-
                             }
                         }
                     }

+ 0 - 6
job-adq/src/main/java/cn/com/ctop/job/adq/data/service/impl/AdqFlowDetailsReportServiceImpl.java

@@ -10,7 +10,6 @@ import cn.com.ctop.job.adq.data.utils.Check;
 import cn.com.ctop.job.adq.data.utils.DateUtils;
 import cn.com.ctop.job.adq.data.utils.HttpUtils;
 import cn.com.ctop.job.adq.utils.RandomStringGenerator;
-import cn.com.ctop.job.adq.utils.doris.sink.DorisMultiTableStreamLoadManager;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import lombok.extern.slf4j.Slf4j;
@@ -32,9 +31,6 @@ import java.util.stream.Collectors;
 @Service
 @Slf4j
 public class AdqFlowDetailsReportServiceImpl implements IAdqFlowDetailsReportService {
-    @Autowired
-    private DorisMultiTableStreamLoadManager dorisLoadManager;
-
     @Resource
     private AdqFlowDetailsReportDailyMapper adqFlowDetailsReportDailyMapper;
 
@@ -117,8 +113,6 @@ public class AdqFlowDetailsReportServiceImpl implements IAdqFlowDetailsReportSer
                             dailyList = buildFlowDetailsReportDaily(dailyList);
 
                             adqFlowDetailsReportDailyMapper.replaceFlowDetailsReportDailies(dailyList);
-
-                            dorisLoadManager.add("media_api", "adq_flow_details_report_daily", dailyList);
                             //adqFlowDetailsReportDailyMapper.replaceFlowDetailsReportDailiesSD(dailyList);
                         }
                     }

+ 0 - 4
job-adq/src/main/java/cn/com/ctop/job/adq/data/service/impl/AdqMaterialReportServiceImpl.java

@@ -8,7 +8,6 @@ import cn.com.ctop.job.adq.data.utils.Check;
 import cn.com.ctop.job.adq.data.utils.FieldsConfig;
 import cn.com.ctop.job.adq.data.utils.HttpUtils;
 import cn.com.ctop.job.adq.utils.RandomStringGenerator;
-import cn.com.ctop.job.adq.utils.doris.sink.DorisMultiTableStreamLoadManager;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import lombok.extern.slf4j.Slf4j;
@@ -23,8 +22,6 @@ import java.util.Map;
 @Service
 public class AdqMaterialReportServiceImpl implements IAdqMaterialReportService {
     @Autowired
-    private DorisMultiTableStreamLoadManager dorisLoadManager;
-    @Autowired
     private AdqErrorLogMapper adqErrorLogMapper;
 
     @Override
@@ -100,7 +97,6 @@ public class AdqMaterialReportServiceImpl implements IAdqMaterialReportService {
                                     dataJson.put("material_id", dataJson.getString("video_id"));
                                     dataJson.put("material_type", "VIDEO");
                                 }
-                                dorisLoadManager.add("media_api", "adq_material_report_daily", dataJson);
                             }
                         }
                     }

+ 0 - 23
job-adq/src/main/java/cn/com/ctop/job/adq/data/service/impl/AdqRetryReportServiceImpl.java

@@ -7,7 +7,6 @@ import cn.com.ctop.job.adq.data.mapper.AdqErrorLogMapper;
 import cn.com.ctop.job.adq.data.service.IAdqRetryReportService;
 import cn.com.ctop.job.adq.data.utils.*;
 import cn.com.ctop.job.adq.utils.RandomStringGenerator;
-import cn.com.ctop.job.adq.utils.doris.sink.DorisMultiTableStreamLoadManager;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import lombok.extern.slf4j.Slf4j;
@@ -19,15 +18,11 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
 
 @Slf4j
 @Service
 public class AdqRetryReportServiceImpl implements IAdqRetryReportService {
     @Autowired
-    private DorisMultiTableStreamLoadManager dorisLoadManager;
-    @Autowired
     private AdqCreativeReportMapper adqReportMapper;
     @Autowired
     private AdqAccountReportMapper accountReportMapper;
@@ -74,10 +69,6 @@ public class AdqRetryReportServiceImpl implements IAdqRetryReportService {
                             hourReport.setAdvertiserId(advertiserId);
                             hourReport.setStatTimeDay(Long.valueOf(startDate.replace("-", "")));
                             hourList.add(hourReport);
-
-                            JSONObject filterJson = JSONObject.parseObject(JsonUtils.toFilterJSON(hourReport));
-                            filterJson.put("video_outer_play100_count",hourReport.getVideoOuterPlay100Count());
-                            dorisLoadManager.add("media_api","adq_creative_report_hour",filterJson);
                         }
                     }
                 }
@@ -144,10 +135,6 @@ public class AdqRetryReportServiceImpl implements IAdqRetryReportService {
                                 hourReport.setStatTimeDay(Long.valueOf(dataJson.getString("date").replace("-", "")));
                                 dailies.add(hourReport);
 
-                                JSONObject filterJson = JSONObject.parseObject(JsonUtils.toFilterJSON(hourReport));
-                                filterJson.put("video_outer_play100_count",hourReport.getVideoOuterPlay100Count());
-                                dorisLoadManager.add("media_api","adq_creative_report_daily", filterJson);
-
                             }
                         }
                     }
@@ -214,11 +201,6 @@ public class AdqRetryReportServiceImpl implements IAdqRetryReportService {
                                 hourReport.setAdvertiserId(advertiserId);
                                 hourReport.setStatTimeDay(Long.valueOf(startDate.replace("-", "")));
                                 hourList.add(hourReport);
-
-                                String filterJSON = JsonUtils.toFilterJSON(hourReport);
-                                JSONObject json = JSONObject.parseObject(filterJSON);
-                                json.put("video_outer_play100_count",hourReport.getVideoOuterPlay100Count());
-                                dorisLoadManager.add("media_api", "adq_advertiser_report_hour", json);
                             }
                         }
                     }
@@ -284,11 +266,6 @@ public class AdqRetryReportServiceImpl implements IAdqRetryReportService {
                                 dailyReport.setAdvertiserId(advertiserId);
                                 dailyReport.setStatTimeDay(Long.valueOf(dataJson.getString("date").replace("-", "")));
                                 dailyList.add(dailyReport);
-
-                                String filterJSON = JsonUtils.toFilterJSON(dailyReport);
-                                JSONObject json = JSONObject.parseObject(filterJSON);
-                                json.put("video_outer_play100_count",dailyReport.getVideoOuterPlay100Count());
-                                dorisLoadManager.add("media_api", "adq_advertiser_report_daily", json);
                             }
                         }
                     }

+ 0 - 49
job-adq/src/main/java/cn/com/ctop/job/adq/utils/DorisConfig.java

@@ -1,49 +0,0 @@
-package cn.com.ctop.job.adq.utils;
-
-import cn.com.ctop.job.adq.utils.doris.sink.DorisBufferConfig;
-import cn.com.ctop.job.adq.utils.doris.sink.DorisMultiTableStreamLoadManager;
-import cn.com.ctop.job.adq.utils.doris.sink.DorisSinkClient;
-import cn.com.ctop.job.adq.utils.doris.sink.SimpleHttpDorisSinkClient;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-
-@Configuration
-public class DorisConfig {
-    @Value("${spring.doris.fe-host}")
-    private String feHost;
-
-    @Value("${spring.doris.user}")
-    private String userName;
-
-    @Value("${spring.doris.password}")
-    private String password;
-
-    @Value("${spring.doris.label-prefix}")
-    private String labelPrefix;
-
-    @Value("${spring.doris.sink-threads}")
-    private int uploaderThreads;
-
-    @Value("${spring.doris.sink-batch-rows}")
-    private int flushMaxRows;
-
-    @Value("${spring.doris.sink-batch-mb}")
-    private long flushMaxMb;
-
-    @Value("${spring.doris.sink-batch-seconds}")
-    private long flushIntervalSeconds;
-
-    @Bean
-    public DorisMultiTableStreamLoadManager dorisStreamLoadManager() {
-        DorisBufferConfig defaultConfig = DorisBufferConfig.builder()
-                .bufferFlushMaxRows(flushMaxRows)
-                .bufferFlushMaxBytes(flushMaxMb * 1024 * 1024)
-                .bufferFlushIntervalMs(flushIntervalSeconds * 1000)
-                .build();
-
-        DorisSinkClient sinkClient = new SimpleHttpDorisSinkClient(feHost, userName, password, labelPrefix);
-
-        return new DorisMultiTableStreamLoadManager(uploaderThreads, defaultConfig, sinkClient);
-    }
-}

+ 0 - 16
job-adq/src/main/java/cn/com/ctop/job/adq/utils/doris/sink/DorisBufferConfig.java

@@ -1,16 +0,0 @@
-package cn.com.ctop.job.adq.utils.doris.sink;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class DorisBufferConfig {
-    private long bufferFlushMaxRows;
-    private long bufferFlushMaxBytes;
-    private long bufferFlushIntervalMs;
-}

+ 0 - 51
job-adq/src/main/java/cn/com/ctop/job/adq/utils/doris/sink/DorisMultiTableStreamLoadManager.java

@@ -1,51 +0,0 @@
-package cn.com.ctop.job.adq.utils.doris.sink;
-
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.scheduling.concurrent.CustomizableThreadFactory;
-
-import javax.annotation.PreDestroy;
-import java.util.Collection;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-@Slf4j
-public class DorisMultiTableStreamLoadManager {
-    private final Map<TableKey, TableBufferManager> bufferManagers = new ConcurrentHashMap<>();
-    private final ExecutorService uploaderExecutor;
-    private final DorisSinkClient sinkClient;
-    private final DorisBufferConfig defaultBufferConfig;
-
-    public DorisMultiTableStreamLoadManager(int uploaderThreads,
-                                            DorisBufferConfig defaultBufferConfig,
-                                            DorisSinkClient sinkClient) {
-        log.info("init DorisMultiTableStreamLoadManager");
-        this.defaultBufferConfig = defaultBufferConfig;
-        this.sinkClient = sinkClient;
-        this.uploaderExecutor = Executors.newFixedThreadPool(uploaderThreads,
-                new CustomizableThreadFactory("doris-stream-uploader-"));
-    }
-
-    public void add(String database, String table, Object obj) {
-        TableKey key = new TableKey(database, table);
-        if (obj instanceof Collection) {
-            Collection<?> collection = (Collection<?>) obj;
-            for (Object o : collection) {
-                bufferManagers.computeIfAbsent(key, k -> new TableBufferManager(k, defaultBufferConfig, sinkClient, uploaderExecutor))
-                        .add(new RecordWithMeta(database, table, o));
-            }
-        } else {
-            bufferManagers.computeIfAbsent(key, k -> new TableBufferManager(k, defaultBufferConfig, sinkClient, uploaderExecutor))
-                    .add(new RecordWithMeta(database, table, obj));
-        }
-    }
-
-
-    @PreDestroy
-    public void shutdown() {
-        log.info("Shutting down DorisMultiTableStreamLoadManager");
-        bufferManagers.values().forEach(TableBufferManager::flushNow);
-        uploaderExecutor.shutdown();
-    }
-}

+ 0 - 7
job-adq/src/main/java/cn/com/ctop/job/adq/utils/doris/sink/DorisSinkClient.java

@@ -1,7 +0,0 @@
-package cn.com.ctop.job.adq.utils.doris.sink;
-
-import java.util.List;
-@FunctionalInterface
-public interface DorisSinkClient {
-    void streamLoad(List<RecordWithMeta> batch) throws Exception;
-}

+ 0 - 17
job-adq/src/main/java/cn/com/ctop/job/adq/utils/doris/sink/RecordWithMeta.java

@@ -1,17 +0,0 @@
-package cn.com.ctop.job.adq.utils.doris.sink;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-public class RecordWithMeta {
-    private String database;
-    private String table;
-    private Object data;
-}

+ 0 - 94
job-adq/src/main/java/cn/com/ctop/job/adq/utils/doris/sink/SimpleHttpDorisSinkClient.java

@@ -1,94 +0,0 @@
-package cn.com.ctop.job.adq.utils.doris.sink;
-
-import cn.com.ctop.job.adq.data.utils.JsonUtils;
-import com.alibaba.fastjson.JSONObject;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.http.HttpEntity;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpPut;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.util.EntityUtils;
-
-import java.nio.charset.StandardCharsets;
-import java.util.Base64;
-import java.util.List;
-import java.util.UUID;
-import java.util.stream.Collectors;
-
-@Slf4j
-public class SimpleHttpDorisSinkClient implements DorisSinkClient {
-    private final String feHost;      // http://fe_host:8030
-    private final String user;        // doris 用户名
-    private final String password;    // doris 密码
-    private final String format;      // json/csv, 推荐 json
-    private final String labelPrefix; // stream load label 前缀(可选)
-
-    public SimpleHttpDorisSinkClient(String feHost, String user, String password, String labelPrefix) {
-        this.feHost = feHost;
-        this.user = user;
-        this.password = password;
-        this.format = "json";
-        this.labelPrefix = labelPrefix;
-    }
-
-    @Override
-    public void streamLoad(List<RecordWithMeta> records) throws Exception {
-        if (records == null || records.isEmpty()) return;
-
-        RecordWithMeta sample = records.get(0);
-        String db = sample.getDatabase();
-        String table = sample.getTable();
-
-        String label = labelPrefix + "_" + db + "_" + table + "_" + UUID.randomUUID();
-        String url = String.format("%s/api/%s/%s/_stream_load", feHost, db, table);
-
-        // 提取 data
-        List<String> dataList = records.stream()
-                .map(d->{
-                    Object data = d.getData();
-                    if (data instanceof JSONObject) {
-                        return data.toString();
-                    }else {
-                        return JsonUtils.toFilterJSON(data);
-                    }
-                })
-                .collect(Collectors.toList());
-
-        String jsonBody = dataList.toString();
-        //log.info("jsonBody:{}", jsonBody);
-
-        HttpPut put = new HttpPut(url);
-        put.setHeader("Expect", "100-continue");
-        put.setHeader("label", label);
-        put.setHeader("Content-Type", "application/json");
-        if (user != null && password != null) {
-            String auth = Base64.getEncoder().encodeToString((user + ":" + password).getBytes(StandardCharsets.UTF_8));
-            put.setHeader("Authorization", "Basic " + auth);
-        }
-
-        put.setHeader("format", format); // json
-        put.setHeader("strip_outer_array", "true"); // json array
-        put.setHeader("fuzzy_parse", "true"); // json array
-
-        put.setEntity(new StringEntity(jsonBody, StandardCharsets.UTF_8));
-
-        try (CloseableHttpClient httpClient = HttpClients.custom()
-                .setRedirectStrategy(new StrictRedirectStrategy())  // 支持 307/301 重定向
-                .build();
-             CloseableHttpResponse response = httpClient.execute(put)) {
-            int code = response.getStatusLine().getStatusCode();
-            HttpEntity entity = response.getEntity();
-            String responseBody = EntityUtils.toString(entity, "UTF-8");
-            if (code != 200 && code != 201) {
-                throw new RuntimeException("Doris stream load failed. Status: " + code + ", response: " + responseBody);
-            }
-            // 可选: 解析返回内容判断 success
-            if (!responseBody.contains("\"Status\": \"Success\"")) {
-                throw new RuntimeException("Stream load returned non-success: " + responseBody);
-            }
-            log.info("Doris stream load success: {}", responseBody);
-        }
-    }
-}

+ 0 - 38
job-adq/src/main/java/cn/com/ctop/job/adq/utils/doris/sink/StrictRedirectStrategy.java

@@ -1,38 +0,0 @@
-package cn.com.ctop.job.adq.utils.doris.sink;
-
-import org.apache.http.HttpEntityEnclosingRequest;
-import org.apache.http.HttpRequest;
-import org.apache.http.HttpResponse;
-import org.apache.http.ProtocolException;
-import org.apache.http.client.methods.HttpPut;
-import org.apache.http.client.methods.HttpUriRequest;
-import org.apache.http.impl.client.DefaultRedirectStrategy;
-import org.apache.http.protocol.HttpContext;
-
-import java.net.URI;
-
-public class StrictRedirectStrategy extends DefaultRedirectStrategy {
-
-    @Override
-    protected boolean isRedirectable(String method) {
-        // 允许 GET, POST, PUT 等都可重定向
-        return true;
-    }
-
-    @Override
-    public HttpUriRequest getRedirect(HttpRequest request, HttpResponse response, HttpContext context)
-            throws ProtocolException {
-        URI uri = getLocationURI(request, response, context);
-
-        String method = request.getRequestLine().getMethod();
-        if (method.equalsIgnoreCase(HttpPut.METHOD_NAME)) {
-            HttpPut put = new HttpPut(uri);
-            put.setHeaders(request.getAllHeaders());
-            if (request instanceof HttpEntityEnclosingRequest) {
-                put.setEntity(((HttpEntityEnclosingRequest) request).getEntity());
-            }
-            return put;
-        }
-        return super.getRedirect(request, response, context);
-    }
-}

+ 0 - 96
job-adq/src/main/java/cn/com/ctop/job/adq/utils/doris/sink/TableBufferManager.java

@@ -1,96 +0,0 @@
-package cn.com.ctop.job.adq.utils.doris.sink;
-
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-import org.springframework.scheduling.concurrent.CustomizableThreadFactory;
-
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.UUID;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-
-@Slf4j
-public class TableBufferManager {
-
-    private final TableKey tableKey;
-    private final DorisBufferConfig config;
-    private final DorisSinkClient sinkClient;
-    private final ExecutorService uploaderExecutor;
-
-    private final List<RecordWithMeta> buffer = new ArrayList<>();
-    private final Object lock = new Object();
-    private final AtomicLong currentBytes = new AtomicLong(0);
-
-    private final ScheduledExecutorService scheduler =
-            Executors.newSingleThreadScheduledExecutor(new CustomizableThreadFactory("flusher-" + UUID.randomUUID()));
-
-    public TableBufferManager(TableKey tableKey,
-                              DorisBufferConfig config,
-                              DorisSinkClient sinkClient,
-                              ExecutorService uploaderExecutor) {
-        this.tableKey = tableKey;
-        this.config = config;
-        this.sinkClient = sinkClient;
-        this.uploaderExecutor = uploaderExecutor;
-
-        scheduler.scheduleAtFixedRate(this::tryFlush,
-                config.getBufferFlushIntervalMs(),
-                config.getBufferFlushIntervalMs(),
-                TimeUnit.MILLISECONDS);
-    }
-
-    public void add(RecordWithMeta record) {
-        if (record == null || StringUtils.isBlank(record.getDatabase()) || StringUtils.isBlank(record.getTable())) {
-            return;
-        }
-        synchronized (lock) {
-            buffer.add(record);
-            currentBytes.addAndGet(estimateSize(record));
-
-            if (buffer.size() >= config.getBufferFlushMaxRows() ||
-                    currentBytes.get() >= config.getBufferFlushMaxBytes()) {
-                flush();
-            }
-        }
-    }
-
-    public void tryFlush() {
-        synchronized (lock) {
-            if (!buffer.isEmpty()) {
-                flush();
-            }
-        }
-    }
-
-    public void flushNow() {
-        synchronized (lock) {
-            flush();
-        }
-        scheduler.shutdownNow();
-    }
-
-    private void flush() {
-        if (buffer.isEmpty()) return;
-
-        List<RecordWithMeta> toSend = new ArrayList<>(buffer);
-        buffer.clear();
-        currentBytes.set(0);
-
-        uploaderExecutor.submit(() -> {
-            try {
-                sinkClient.streamLoad(toSend);
-            } catch (Exception e) {
-                log.error("Stream load failed for {}.{}: {}", tableKey.getDatabase(), tableKey.getTable(), e.getMessage(), e);
-            }
-        });
-    }
-
-    private long estimateSize(RecordWithMeta record) {
-        return record.toString().getBytes(StandardCharsets.UTF_8).length;
-    }
-}

+ 0 - 26
job-adq/src/main/java/cn/com/ctop/job/adq/utils/doris/sink/TableKey.java

@@ -1,26 +0,0 @@
-package cn.com.ctop.job.adq.utils.doris.sink;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-
-import java.util.Objects;
-
-@Data
-@AllArgsConstructor
-public class TableKey {
-    private String database;
-    private String table;
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(database, table);
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        if (this == obj) return true;
-        if (!(obj instanceof TableKey)) return false;
-        TableKey other = (TableKey) obj;
-        return Objects.equals(database, other.database) && Objects.equals(table, other.table);
-    }
-}

+ 5 - 5
job-adq/src/main/resources/application.yml

@@ -83,10 +83,10 @@ spring:
         connectionProperties: druid.stat.mergeSql\=true;druid.stat.slowSqlMillis\=5000
       datasource:
         master:
-          url: jdbc:mysql://172.16.0.117:3390/media_api?characterEncoding=UTF-8&useUnicode=true&useSSL=false
-#          url: jdbc:mysql://8.147.135.80:3390/media_api?characterEncoding=UTF-8&useUnicode=true&useSSL=false
-          username: data
-          password: hr@2025..
+          #url: jdbc:mysql://192.168.0.249:3390/media_api?characterEncoding=UTF-8&useUnicode=true&useSSL=false
+          url: jdbc:mysql://114.242.47.94:3394/media_api?characterEncoding=UTF-8&useUnicode=true&useSSL=false
+          username: hcst
+          password: hcst@2024
           driver-class-name: com.mysql.jdbc.Driver
 
 
@@ -153,7 +153,7 @@ xxl:
       port: 9007
       ip:
       includeIpPrefix: 172
-      logpath: /data/webapp/hr_job-adq/log
+      logpath: /home/webapp/job-adq/log
       logretentiondays: 3
     admin:
       addresses: http://172.16.0.108:9003/xxl-job-admin