Преглед изворни кода

Merge #36 into master from pingkang

Merge branch 'master' of codeup.aliyun.com:60c06368dbbed34fe0f0c263/job/hr_job-...

* pingkang: (3 commits)
  selectdb batch stream load
  selectdb batch stream load
  Merge branch 'master' of codeup.aliyun.com:60c06368dbbed34fe0f0c263/job/hr_job-...

Signed-off-by: 1129979389@qq.com <accounts_6361eaad7b63b44dd7833e63@mail.teambition.com>
Merged-by: 1129979389@qq.com <accounts_6361eaad7b63b44dd7833e63@mail.teambition.com>

CR-link: https://codeup.aliyun.com/60c06368dbbed34fe0f0c263/job/hr_job-bytedance/change/36
huangpingkang пре 1 година
родитељ
комит
783b0d963b
14 измењених фајлова са 457 додато и 17 уклоњено
  1. 9 4
      job-bytedance/src/main/java/cn/com/ctop/job/bytedance/controller/BytedanceController.java
  2. 9 11
      job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/impl/BytedanceAccountReportServiceImpl.java
  3. 56 0
      job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/utils/JsonUtils.java
  4. 2 2
      job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedanceReportRetryJob.java
  5. 16 0
      job-bytedance/src/main/java/cn/com/ctop/job/bytedance/utils/doris_sink/DorisBufferConfig.java
  6. 45 0
      job-bytedance/src/main/java/cn/com/ctop/job/bytedance/utils/doris_sink/DorisConfiguration.java
  7. 41 0
      job-bytedance/src/main/java/cn/com/ctop/job/bytedance/utils/doris_sink/DorisMultiTableStreamLoadManager.java
  8. 7 0
      job-bytedance/src/main/java/cn/com/ctop/job/bytedance/utils/doris_sink/DorisSinkClient.java
  9. 17 0
      job-bytedance/src/main/java/cn/com/ctop/job/bytedance/utils/doris_sink/RecordWithMeta.java
  10. 86 0
      job-bytedance/src/main/java/cn/com/ctop/job/bytedance/utils/doris_sink/SimpleHttpDorisSinkClient.java
  11. 38 0
      job-bytedance/src/main/java/cn/com/ctop/job/bytedance/utils/doris_sink/StrictRedirectStrategy.java
  12. 96 0
      job-bytedance/src/main/java/cn/com/ctop/job/bytedance/utils/doris_sink/TableBufferManager.java
  13. 26 0
      job-bytedance/src/main/java/cn/com/ctop/job/bytedance/utils/doris_sink/TableKey.java
  14. 9 0
      job-bytedance/src/main/resources/application.yml

+ 9 - 4
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/controller/BytedanceController.java

@@ -192,7 +192,7 @@ public class BytedanceController {
     }
 
     @RequestMapping("/adq/creatives")
-    public void adqCreatives(Long accountId,String startDate,String endDate) {
+    public void adqCreatives(Long accountId, String startDate, String endDate) {
         cdpProjectService.getAdqCreatives(accountId, startDate, endDate);
     }
 
@@ -203,15 +203,20 @@ public class BytedanceController {
     }
 
     @GetMapping("/adqWechatPages")
-    public void adqWechatPages(){
+    public void adqWechatPages() {
         List<JSONObject> iids = cdpProjectService.queryPageAdvertiserIds();
         if (!Check.isNull(iids)) {
             for (JSONObject iid : iids) {
                 Long pageId = iid.getLong("page_id");
                 Long advertiserId = iid.getLong("advertiser_id");
-                dailyRetrExecutorService.submit(() -> cdpProjectService.getAdqWechatPages(advertiserId,pageId));
+                dailyRetrExecutorService.submit(() -> cdpProjectService.getAdqWechatPages(advertiserId, pageId));
             }
         }
     }
 
-}
+
+    @GetMapping("/accountHour")
+    public void bytedanceAccountHour(Long advertiserId, String accessToken, String date) {
+        accountReportService.getBytedanceAccountHour(advertiserId, accessToken, date, date, 1, 0);
+    }
+}

+ 9 - 11
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/impl/BytedanceAccountReportServiceImpl.java

@@ -7,6 +7,8 @@ import cn.com.ctop.job.bytedance.data.utils.AlertUtils;
 import cn.com.ctop.job.bytedance.data.utils.Check;
 import cn.com.ctop.job.bytedance.data.utils.FieldsConfig;
 import cn.com.ctop.job.bytedance.data.utils.HttpUtils;
+import cn.com.ctop.job.bytedance.utils.doris_sink.DorisMultiTableStreamLoadManager;
+import cn.com.ctop.job.bytedance.utils.doris_sink.RecordWithMeta;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import lombok.extern.slf4j.Slf4j;
@@ -24,15 +26,10 @@ import java.util.*;
 @EnableScheduling
 public class BytedanceAccountReportServiceImpl implements IBytedanceAccountReportService {
     @Autowired
-    private BytedanceAccountReportMapper accountReportMapper;
-
-    /*private static final int MAX_BATCH_SIZE = 5000;
-    private static final long MAX_WAIT_TIME_MS = 2 * 60 * 1000;
+    private DorisMultiTableStreamLoadManager dorisLoadManager;
 
-    private final List<BytedanceAdvertiserReportHour> advertiserReportHourBuffer = Collections.synchronizedList(new ArrayList<>());
-    private final List<BytedanceAdvertiserReportDaily> advertiserReportDailyBuffer = Collections.synchronizedList(new ArrayList<>());
-    private long advertiserReportHourLastFlushTime = System.currentTimeMillis();
-    private long advertiserReportDailyLastFlushTime = System.currentTimeMillis();*/
+    @Autowired
+    private BytedanceAccountReportMapper accountReportMapper;
 
 
     @Override
@@ -231,6 +228,8 @@ public class BytedanceAccountReportServiceImpl implements IBytedanceAccountRepor
                             advertiserReportHour.setStatTimeDay(statTimeDay);
                             advertiserReportHour.setHour(hour);
                             hours.add(advertiserReportHour);
+
+                            dorisLoadManager.add(new RecordWithMeta("media_api","bytedance_advertiser_report_hour",advertiserReportHour));
                         }
                     } else {
                         log.info("账户id:{},日期:{}广告时报(账户粒度)请求结果为空", advertiserId, startDate);
@@ -253,8 +252,7 @@ public class BytedanceAccountReportServiceImpl implements IBytedanceAccountRepor
             }
             if (!Check.isNull(hours)) {
                 accountReportMapper.replaceAccountReportHour(hours);
-                accountReportMapper.replaceAccountReportHourSD(hours);
-                //advertiserReportHourBuffer.addAll(hours);
+                //accountReportMapper.replaceAccountReportHourSD(hours);
             }
             if (hours.size() >= 100) {
                 try {
@@ -350,7 +348,7 @@ public class BytedanceAccountReportServiceImpl implements IBytedanceAccountRepor
                 } catch (InterruptedException e) {
                     throw new RuntimeException(e);
                 }
-                getBytedanceAccountHour(advertiserId, accessToken, startDate, endDate, page + 1, isLog);
+                getBytedanceAccountDaily(advertiserId, accessToken, startDate, endDate, page + 1, isLog);
             } else {
                 log.info("账户ID:{},日期:{},广告日报(账户粒度)数据请求成功", advertiserId, startDate);
             }

+ 56 - 0
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/utils/JsonUtils.java

@@ -0,0 +1,56 @@
+package cn.com.ctop.job.bytedance.data.utils;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.PropertyNamingStrategy;
+import com.alibaba.fastjson.serializer.NameFilter;
+import com.alibaba.fastjson.serializer.SerializeConfig;
+import com.alibaba.fastjson.serializer.SerializerFeature;
+
+import java.util.regex.Pattern;
+
+public class JsonUtils {
+
+    // 正则预编译
+    private static final Pattern INVALID_PREFIX = Pattern.compile("^[^a-zA-Z0-9]+");
+    private static final Pattern INVALID_SUFFIX = Pattern.compile("[^a-zA-Z0-9]+$");
+    private static final Pattern LETTER_NUMBER = Pattern.compile("([a-zA-Z])([0-9])");
+    //private static final Pattern NUMBER_LETTER = Pattern.compile("([0-9])([a-zA-Z])");
+    private static final Pattern CAMEL_CASE = Pattern.compile("([a-z])([A-Z])");
+
+    // 全局 SerializeConfig
+    private static final SerializeConfig SNAKE_CASE_CONFIG = new SerializeConfig();
+
+    static {
+        SNAKE_CASE_CONFIG.propertyNamingStrategy = PropertyNamingStrategy.SnakeCase;
+    }
+
+    // NameFilter
+    private static final NameFilter CAMEL_TO_SNAKE_FILTER = (object, name, value) -> {
+        // 去除非法首尾字符
+        name = INVALID_PREFIX.matcher(name).replaceAll("");
+        name = INVALID_SUFFIX.matcher(name).replaceAll("");
+
+        // a3 -> a_3,3a -> 3_a
+        name = LETTER_NUMBER.matcher(name).replaceAll("$1_$2");
+        //name = NUMBER_LETTER.matcher(name).replaceAll("$1_$2");
+
+        // camelCase -> camel_case
+        name = CAMEL_CASE.matcher(name).replaceAll("$1_$2");
+
+        return name.toLowerCase();
+    };
+
+    /**
+     * 将对象转为 JSON 字符串,字段名转为下划线命名。
+     */
+    public static String toSnakeCaseJsonIgnoreNull(Object obj) {
+        return JSON.toJSONString(obj, SNAKE_CASE_CONFIG);
+    }
+
+    /**
+     * 将对象转为 JSON 字符串,字段名转为下划线命名
+     */
+    public static String toFilterJSON(Object obj) {
+        return JSON.toJSONString(obj, CAMEL_TO_SNAKE_FILTER, SerializerFeature.WriteMapNullValue, SerializerFeature.MapSortField);
+    }
+}

+ 2 - 2
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedanceReportRetryJob.java

@@ -204,8 +204,6 @@ public class BytedanceReportRetryJob {
                 String dataDimension = jsonObject.getString("dataDimension");
                 Integer isSend = jsonObject.getInteger("isSend");
                 String accessToken = jsonObject.getString("accessToken");
-                Long taskId = jsonObject.getJSONObject("msg").getLong("taskId");
-
                 if ("xingtu_task_list".equalsIgnoreCase(dataDimension)) {
                     xingTuExecutorService.submit(new Runnable() {
                         @Override
@@ -214,6 +212,7 @@ public class BytedanceReportRetryJob {
                         }
                     });
                 } else if ("xingtu_task_info".equalsIgnoreCase(dataDimension)) {
+                    Long taskId = jsonObject.getJSONObject("msg").getLong("taskId");
                     xingTuExecutorService.submit(new Runnable() {
                         @Override
                         public void run() {
@@ -221,6 +220,7 @@ public class BytedanceReportRetryJob {
                         }
                     });
                 } else if ("xingtu_task_report".equalsIgnoreCase(dataDimension)) {
+                    Long taskId = jsonObject.getJSONObject("msg").getLong("taskId");
                     xingTuExecutorService.submit(new Runnable() {
                         @Override
                         public void run() {

+ 16 - 0
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/utils/doris_sink/DorisBufferConfig.java

@@ -0,0 +1,16 @@
+package cn.com.ctop.job.bytedance.utils.doris_sink;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class DorisBufferConfig {
+    private long bufferFlushMaxRows;
+    private long bufferFlushMaxBytes;
+    private long bufferFlushIntervalMs;
+}

+ 45 - 0
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/utils/doris_sink/DorisConfiguration.java

@@ -0,0 +1,45 @@
+package cn.com.ctop.job.bytedance.utils.doris_sink;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class DorisConfiguration {
+    @Value("${spring.doris.fe-host}")
+    private String feHost;
+
+    @Value("${spring.doris.user}")
+    private String userName;
+
+    @Value("${spring.doris.password}")
+    private String password;
+
+    @Value("${spring.doris.label-prefix}")
+    private String labelPrefix;
+
+    @Value("${spring.doris.sink-threads}")
+    private int uploaderThreads;
+
+    @Value("${spring.doris.sink-batch-rows}")
+    private int flushMaxRows;
+
+    @Value("${spring.doris.sink-batch-mb}")
+    private long flushMaxMb;
+
+    @Value("${spring.doris.sink-batch-seconds}")
+    private long flushIntervalSeconds;
+
+    @Bean
+    public DorisMultiTableStreamLoadManager dorisStreamLoadManager() {
+        DorisBufferConfig defaultConfig = DorisBufferConfig.builder()
+                .bufferFlushMaxRows(flushMaxRows)
+                .bufferFlushMaxBytes(flushMaxMb * 1024 * 1024)
+                .bufferFlushIntervalMs(flushIntervalSeconds * 1000)
+                .build();
+
+        DorisSinkClient sinkClient = new SimpleHttpDorisSinkClient(feHost, userName, password, labelPrefix);
+
+        return new DorisMultiTableStreamLoadManager(uploaderThreads, defaultConfig, sinkClient);
+    }
+}

+ 41 - 0
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/utils/doris_sink/DorisMultiTableStreamLoadManager.java

@@ -0,0 +1,41 @@
+package cn.com.ctop.job.bytedance.utils.doris_sink;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.scheduling.concurrent.CustomizableThreadFactory;
+
+import javax.annotation.PreDestroy;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+@Slf4j
+public class DorisMultiTableStreamLoadManager {
+    private final Map<TableKey, TableBufferManager> bufferManagers = new ConcurrentHashMap<>();
+    private final ExecutorService uploaderExecutor;
+    private final DorisSinkClient sinkClient;
+    private final DorisBufferConfig defaultBufferConfig;
+
+    public DorisMultiTableStreamLoadManager(int uploaderThreads,
+                                             DorisBufferConfig defaultBufferConfig,
+                                             DorisSinkClient sinkClient) {
+        log.info("init DorisMultiTableStreamLoadManager");
+        this.defaultBufferConfig = defaultBufferConfig;
+        this.sinkClient = sinkClient;
+        this.uploaderExecutor = Executors.newFixedThreadPool(uploaderThreads,
+                new CustomizableThreadFactory("doris-stream-uploader-"));
+    }
+
+    public void add(RecordWithMeta record) {
+        TableKey key = new TableKey(record.getDatabase(), record.getTable());
+        bufferManagers.computeIfAbsent(key, k -> new TableBufferManager(k, defaultBufferConfig, sinkClient, uploaderExecutor))
+                .add(record);
+    }
+
+    @PreDestroy
+    public void shutdown() {
+        log.info("Shutting down DorisMultiTableStreamLoadManager");
+        bufferManagers.values().forEach(TableBufferManager::flushNow);
+        uploaderExecutor.shutdown();
+    }
+}

+ 7 - 0
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/utils/doris_sink/DorisSinkClient.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.job.bytedance.utils.doris_sink;
+
+import java.util.List;
+@FunctionalInterface
+public interface DorisSinkClient {
+    void streamLoad(List<RecordWithMeta> batch) throws Exception;
+}

+ 17 - 0
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/utils/doris_sink/RecordWithMeta.java

@@ -0,0 +1,17 @@
+package cn.com.ctop.job.bytedance.utils.doris_sink;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+public class RecordWithMeta {
+    private String database;
+    private String table;
+    private Object data;
+}

+ 86 - 0
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/utils/doris_sink/SimpleHttpDorisSinkClient.java

@@ -0,0 +1,86 @@
+package cn.com.ctop.job.bytedance.utils.doris_sink;
+
+import cn.com.ctop.job.bytedance.data.utils.JsonUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.http.HttpEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.List;
+import java.util.UUID;
+import java.util.stream.Collectors;
+
+@Slf4j
+public class SimpleHttpDorisSinkClient implements DorisSinkClient {
+    private final String feHost;      // http://fe_host:8030
+    private final String user;        // doris 用户名
+    private final String password;    // doris 密码
+    private final String format;      // json/csv, 推荐 json
+    private final String labelPrefix; // stream load label 前缀(可选)
+
+    public SimpleHttpDorisSinkClient(String feHost, String user, String password, String labelPrefix) {
+        this.feHost = feHost;
+        this.user = user;
+        this.password = password;
+        this.format = "json";
+        this.labelPrefix = labelPrefix;
+    }
+
+    @Override
+    public void streamLoad(List<RecordWithMeta> records) throws Exception {
+        if (records == null || records.isEmpty()) return;
+
+        RecordWithMeta sample = records.get(0);
+        String db = sample.getDatabase();
+        String table = sample.getTable();
+
+        String label = labelPrefix + db + "_" + table + "_" + UUID.randomUUID();
+        String url = String.format("%s/api/%s/%s/_stream_load", feHost, db, table);
+
+        // 提取 data
+        List<Object> dataList = records.stream()
+                .map(RecordWithMeta::getData)
+                .collect(Collectors.toList());
+
+        String jsonBody = JsonUtils.toFilterJSON(dataList);
+        //log.info("jsonBody:{}", jsonBody);
+
+        HttpPut put = new HttpPut(url);
+        put.setHeader("Expect", "100-continue");
+        put.setHeader("label", label);
+        put.setHeader("Content-Type", "application/json");
+        if (user != null && password != null) {
+            String auth = Base64.getEncoder().encodeToString((user + ":" + password).getBytes(StandardCharsets.UTF_8));
+            put.setHeader("Authorization", "Basic " + auth);
+        }
+
+        put.setHeader("format", format); // json
+        put.setHeader("strip_outer_array", "true"); // json array
+        put.setHeader("fuzzy_parse", "true"); // json array
+
+        put.setEntity(new StringEntity(jsonBody, StandardCharsets.UTF_8));
+
+        try (CloseableHttpClient httpClient = HttpClients.custom()
+                .setRedirectStrategy(new StrictRedirectStrategy())  // 支持 307/301 重定向
+                .build();
+             CloseableHttpResponse response = httpClient.execute(put)) {
+            int code = response.getStatusLine().getStatusCode();
+            HttpEntity entity = response.getEntity();
+            String responseBody = EntityUtils.toString(entity, "UTF-8");
+            if (code != 200 && code != 201) {
+                throw new RuntimeException("Doris stream load failed. Status: " + code + ", response: " + responseBody);
+            }
+            // 可选: 解析返回内容判断 success
+            if (!responseBody.contains("\"Status\": \"Success\"")) {
+                throw new RuntimeException("Stream load returned non-success: " + responseBody);
+            }
+            log.info("Doris stream load success: {}", responseBody);
+        }
+    }
+}

+ 38 - 0
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/utils/doris_sink/StrictRedirectStrategy.java

@@ -0,0 +1,38 @@
+package cn.com.ctop.job.bytedance.utils.doris_sink;
+
+import org.apache.http.HttpEntityEnclosingRequest;
+import org.apache.http.HttpRequest;
+import org.apache.http.HttpResponse;
+import org.apache.http.ProtocolException;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.impl.client.DefaultRedirectStrategy;
+import org.apache.http.protocol.HttpContext;
+
+import java.net.URI;
+
+public class StrictRedirectStrategy extends DefaultRedirectStrategy {
+
+    @Override
+    protected boolean isRedirectable(String method) {
+        // 允许 GET, POST, PUT 等都可重定向
+        return true;
+    }
+
+    @Override
+    public HttpUriRequest getRedirect(HttpRequest request, HttpResponse response, HttpContext context)
+            throws ProtocolException {
+        URI uri = getLocationURI(request, response, context);
+
+        String method = request.getRequestLine().getMethod();
+        if (method.equalsIgnoreCase(HttpPut.METHOD_NAME)) {
+            HttpPut put = new HttpPut(uri);
+            put.setHeaders(request.getAllHeaders());
+            if (request instanceof HttpEntityEnclosingRequest) {
+                put.setEntity(((HttpEntityEnclosingRequest) request).getEntity());
+            }
+            return put;
+        }
+        return super.getRedirect(request, response, context);
+    }
+}

+ 96 - 0
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/utils/doris_sink/TableBufferManager.java

@@ -0,0 +1,96 @@
+package cn.com.ctop.job.bytedance.utils.doris_sink;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.scheduling.concurrent.CustomizableThreadFactory;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+@Slf4j
+public class TableBufferManager {
+
+    private final TableKey tableKey;
+    private final DorisBufferConfig config;
+    private final DorisSinkClient sinkClient;
+    private final ExecutorService uploaderExecutor;
+
+    private final List<RecordWithMeta> buffer = new ArrayList<>();
+    private final Object lock = new Object();
+    private final AtomicLong currentBytes = new AtomicLong(0);
+
+    private final ScheduledExecutorService scheduler =
+            Executors.newSingleThreadScheduledExecutor(new CustomizableThreadFactory("flusher-" + UUID.randomUUID()));
+
+    public TableBufferManager(TableKey tableKey,
+                              DorisBufferConfig config,
+                              DorisSinkClient sinkClient,
+                              ExecutorService uploaderExecutor) {
+        this.tableKey = tableKey;
+        this.config = config;
+        this.sinkClient = sinkClient;
+        this.uploaderExecutor = uploaderExecutor;
+
+        scheduler.scheduleAtFixedRate(this::tryFlush,
+                config.getBufferFlushIntervalMs(),
+                config.getBufferFlushIntervalMs(),
+                TimeUnit.MILLISECONDS);
+    }
+
+    public void add(RecordWithMeta record) {
+        if (record == null || StringUtils.isBlank(record.getDatabase()) || StringUtils.isBlank(record.getTable())) {
+            return;
+        }
+        synchronized (lock) {
+            buffer.add(record);
+            currentBytes.addAndGet(estimateSize(record));
+
+            if (buffer.size() >= config.getBufferFlushMaxRows() ||
+                    currentBytes.get() >= config.getBufferFlushMaxBytes()) {
+                flush();
+            }
+        }
+    }
+
+    public void tryFlush() {
+        synchronized (lock) {
+            if (!buffer.isEmpty()) {
+                flush();
+            }
+        }
+    }
+
+    public void flushNow() {
+        synchronized (lock) {
+            flush();
+        }
+        scheduler.shutdownNow();
+    }
+
+    private void flush() {
+        if (buffer.isEmpty()) return;
+
+        List<RecordWithMeta> toSend = new ArrayList<>(buffer);
+        buffer.clear();
+        currentBytes.set(0);
+
+        uploaderExecutor.submit(() -> {
+            try {
+                sinkClient.streamLoad(toSend);
+            } catch (Exception e) {
+                log.error("Stream load failed for {}.{}: {}", tableKey.getDatabase(), tableKey.getTable(), e.getMessage(), e);
+            }
+        });
+    }
+
+    private long estimateSize(RecordWithMeta record) {
+        return record.toString().getBytes(StandardCharsets.UTF_8).length;
+    }
+}

+ 26 - 0
job-bytedance/src/main/java/cn/com/ctop/job/bytedance/utils/doris_sink/TableKey.java

@@ -0,0 +1,26 @@
+package cn.com.ctop.job.bytedance.utils.doris_sink;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+import java.util.Objects;
+
+@Data
+@AllArgsConstructor
+public class TableKey {
+    private String database;
+    private String table;
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(database, table);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (!(obj instanceof TableKey)) return false;
+        TableKey other = (TableKey) obj;
+        return Objects.equals(database, other.database) && Objects.equals(table, other.table);
+    }
+}

+ 9 - 0
job-bytedance/src/main/resources/application.yml

@@ -134,6 +134,15 @@ spring:
       min-idle: 8
     port: 6379
 
+  doris:
+    fe-host: http://selectdb-cn-vym4at1qw04.selectdbfe.rds.aliyuncs.com:8080
+    user: data
+    password: hr@2025..
+    label-prefix: spring-bytedance
+    sink-threads: 3
+    sink-batch-rows: 50000
+    sink-batch-mb: 20
+    sink-batch-seconds: 20
 
 #mybatis plus 设置
 mybatis-plus: