数据清理

README.md

@@ -1,5 +1,55 @@
 # OPPO 渠道接入说明
 
+### jlt68 荣耀激活 OAID 日导出
+
+`scripts/export_honor_jlt68_oaids.py` 会从 `tencent_baidu_conversions` 导出上海时区前一日的
+`tag_id=jlt68`、`media=honor`、`act=1` 数据。输出是 UTF-8、**无表头**的 CSV，每行一个去重后的
+`device_id`（OAID 原值或 MD5 值），可以直接上传。脚本不会改变 OAID 的大小写；若原始数据中存在
+大小写不同但数据库排序规则视为相同的值，数据库会将其去重。
+
+脚本允许导出任意非负数量的 OAID（包括空文件），但会拒绝发布多于 5,000 万条的文件，也会在失败时删除临时文件；成功时通过原子替换
+生成文件，避免上传到半写入的 CSV。默认输出目录为
+`/data/webapp/tencent-baidu-tracking/oaid-export`，可通过 `OAID_EXPORT_DIR` 修改。数据库连接只从以下
+环境变量读取，避免密码出现在 crontab 或 Shell 历史中：
+
+```text
+OAID_EXPORT_MYSQL_HOST
+OAID_EXPORT_MYSQL_PORT       # 可选，默认 3390
+OAID_EXPORT_MYSQL_USER
+OAID_EXPORT_MYSQL_PASSWORD
+OAID_EXPORT_MYSQL_DB         # 可选，默认 adx_tencent
+OAID_EXPORT_DIR              # 可选
+```
+
+安装依赖后可先补跑指定日期验证：
+
+```bash
+python3 -m pip install PyMySQL
+export OAID_EXPORT_MYSQL_HOST='your-tidb-host'
+export OAID_EXPORT_MYSQL_USER='readonly-user'
+export OAID_EXPORT_MYSQL_PASSWORD='***'
+python3 scripts/export_honor_jlt68_oaids.py --date 20261101
+```
+
+生产环境建议为运行服务的账号创建一个仅有 `SELECT` 权限的数据库账号，并将上述变量放入权限为 `600`
+的环境文件（例如 `/etc/tencent-baidu-tracking/oaid-export.env`）。每天 08:10（上海时区）执行的 crontab
+示例：
+
+```cron
+CRON_TZ=Asia/Shanghai
+10 8 * * * set -a; . /etc/tencent-baidu-tracking/oaid-export.env; set +a; /usr/bin/python3 /data/webapp/tencent-baidu-tracking/scripts/export_honor_jlt68_oaids.py >> /data/webapp/tencent-baidu-tracking/log/oaid-export.log 2>&1
+```
+
+首次上线前，在变更窗口执行 [reports/add-honor-jlt68-oaid-export-index.sql](reports/add-honor-jlt68-oaid-export-index.sql)。该复合索引避免每日全表扫描；若表很大，应由 DBA 使用在线 DDL 流程创建。
+
+业务人员可通过浏览器实时下载，不需要服务器目录或数据库账号：
+
+```text
+GET /admin/exports/honor/jlt68/oaids?date=20261101
+```
+
+`date` 可省略，省略时按上海时区取前一日。接口固定导出 `jlt68`、`honor`、`act=1`，不接受广告位、媒体或 SQL 条件；响应以附件形式下载 CSV，并通过 `X-OAID-Count` 返回导出前计算的去重 ID 数。因设备标识属于敏感数据，必须仅在已有后台 SSO、内网 VPN 或网关鉴权之后向业务开放 `/admin/exports/honor/jlt68/oaids`；当前应用本身未提供管理员登录功能。
+
 OPPO APP 数据回传协议使用 `POST https://api.ads.heytapmobi.com/api/uploadActiveData`。
 配置 `adx.oppo-conversion-salt`、`adx.oppo-aes-base64-key` 和可选的
 `adx.oppo-owner-token` 后，通过 `OppoClient` 发送回传：`imei`、`mac`、`ouId`

src/main/java/com/adx/tencent/httpapi/AdminController.java

@@ -10,11 +10,21 @@ import com.adx.tencent.conversionsync.RetryService;
 import org.springframework.data.redis.core.Cursor;
 import org.springframework.data.redis.core.ScanOptions;
 import org.springframework.data.redis.core.StringRedisTemplate;
+import org.springframework.http.ContentDisposition;
+import org.springframework.http.HttpHeaders;
 import org.springframework.http.HttpStatus;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
 import org.springframework.lang.Nullable;
 import org.springframework.web.bind.annotation.*;
 import org.springframework.web.server.ResponseStatusException;
+import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;
 
+import java.nio.charset.StandardCharsets;
+import java.time.LocalDate;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
 import java.time.Duration;
 import java.util.ArrayList;
 import java.util.List;
@@ -37,6 +47,7 @@ public class AdminController {
     private final RetryService retryService;
     private final ManualTencentCallbackService manualTencentCallbackService;
     private final StringRedisTemplate redisTemplate;
+    private final HonorOaidExportService honorOaidExportService;
 
     public AdminController(ConversionClient conversionClient,
                            @Nullable ConversionSyncRunner conversionSyncRunner,
@@ -44,7 +55,8 @@ public class AdminController {
                            @Nullable ConversionSyncService conversionSyncer,
                            @Nullable RetryService retryService,
                            @Nullable ManualTencentCallbackService manualTencentCallbackService,
-                           @Nullable StringRedisTemplate redisTemplate) {
+                           @Nullable StringRedisTemplate redisTemplate,
+                           HonorOaidExportService honorOaidExportService) {
         this.conversionClient = conversionClient;
         this.conversionSyncRunner = conversionSyncRunner;
         this.backfillJobService = backfillJobService;
@@ -52,6 +64,39 @@ public class AdminController {
         this.retryService = retryService;
         this.manualTencentCallbackService = manualTencentCallbackService;
         this.redisTemplate = redisTemplate;
+        this.honorOaidExportService = honorOaidExportService;
+    }
+
+    /**
+     * Browser download endpoint for the fixed jlt68/Honor/activation OAID file.
+     * Only the data date is configurable, so this endpoint cannot be used as a
+     * general-purpose conversion-table export.
+     */
+    @GetMapping(value = "/exports/honor/jlt68/oaids", produces = "text/csv")
+    public ResponseEntity<StreamingResponseBody> exportHonorJlt68Oaids(
+            @RequestParam(value = "date", required = false) String date) {
+        LocalDate exportDate = parseExportDate(date);
+        long count = honorOaidExportService.countExportRows(exportDate);
+        String filename = "jlt68_honor_act1_oaid_" + DateTimeFormatter.BASIC_ISO_DATE.format(exportDate) + ".csv";
+        StreamingResponseBody body = outputStream -> honorOaidExportService.writeCsv(exportDate, outputStream);
+
+        return ResponseEntity.ok()
+                .contentType(new MediaType("text", "csv", StandardCharsets.UTF_8))
+                .header(HttpHeaders.CONTENT_DISPOSITION,
+                        ContentDisposition.attachment().filename(filename, StandardCharsets.UTF_8).build().toString())
+                .header("X-OAID-Count", Long.toString(count))
+                .body(body);
+    }
+
+    private static LocalDate parseExportDate(String value) {
+        if (value == null || value.isBlank()) {
+            return LocalDate.now(ZoneId.of("Asia/Shanghai")).minusDays(1);
+        }
+        try {
+            return LocalDate.parse(value, DateTimeFormatter.BASIC_ISO_DATE);
+        } catch (DateTimeParseException e) {
+            throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "date must be YYYYMMDD");
+        }
     }
 
     @PostMapping("/conversions/query")

src/main/java/com/adx/tencent/httpapi/HonorOaidExportService.java

@@ -0,0 +1,129 @@
+package com.adx.tencent.httpapi;
+
+import org.springframework.http.HttpStatus;
+import org.springframework.lang.Nullable;
+import org.springframework.stereotype.Service;
+import org.springframework.web.server.ResponseStatusException;
+
+import javax.sql.DataSource;
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
+
+/** Streams the fixed Honor/jlt68 activation OAID export to an HTTP response. */
+@Service
+public class HonorOaidExportService {
+
+    public static final long MAXIMUM_OAIDS = 50_000_000L;
+
+    private static final String TAG_ID = "jlt68";
+    private static final String MEDIA = "honor";
+    private static final int ACTIVATE_ACT = 1;
+    private static final DateTimeFormatter DATE_VALUE_FORMAT = DateTimeFormatter.BASIC_ISO_DATE;
+
+    private static final String COUNT_SQL = """
+            SELECT COUNT(*)
+            FROM (
+                SELECT DISTINCT TRIM(device_id) AS device_id
+                FROM tencent_baidu_conversions
+                WHERE date_value = ?
+                  AND tag_id = ?
+                  AND act = ?
+                  AND media = ?
+                  AND device_id IS NOT NULL
+                  AND TRIM(device_id) <> ''
+            ) export_oaids
+            """;
+
+    private static final String EXPORT_SQL = """
+            SELECT DISTINCT TRIM(device_id) AS device_id
+            FROM tencent_baidu_conversions
+            WHERE date_value = ?
+              AND tag_id = ?
+              AND act = ?
+              AND media = ?
+              AND device_id IS NOT NULL
+              AND TRIM(device_id) <> ''
+            """;
+
+    @Nullable
+    private final DataSource dataSource;
+
+    public HonorOaidExportService(@Nullable DataSource dataSource) {
+        this.dataSource = dataSource;
+    }
+
+    /** Validates the platform maximum before the servlet starts downloading bytes. */
+    public long countExportRows(LocalDate exportDate) {
+        DataSource source = requireDataSource();
+        try (Connection connection = source.getConnection();
+             PreparedStatement statement = connection.prepareStatement(COUNT_SQL)) {
+            bindExportFilters(statement, exportDate);
+            try (ResultSet result = statement.executeQuery()) {
+                if (!result.next()) {
+                    return 0L;
+                }
+                long count = result.getLong(1);
+                if (count > MAXIMUM_OAIDS) {
+                    throw new ResponseStatusException(HttpStatus.UNPROCESSABLE_ENTITY,
+                            "OAID count " + count + " exceeds the upload limit of " + MAXIMUM_OAIDS);
+                }
+                return count;
+            }
+        } catch (SQLException e) {
+            throw new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE,
+                    "Unable to query OAID export data", e);
+        }
+    }
+
+    /**
+     * Writes a headerless UTF-8 CSV. The MySQL streaming fetch mode prevents a
+     * large daily export from being loaded into application memory.
+     */
+    public void writeCsv(LocalDate exportDate, OutputStream outputStream) throws IOException {
+        DataSource source = requireDataSource();
+        try (Connection connection = source.getConnection();
+             PreparedStatement statement = connection.prepareStatement(EXPORT_SQL,
+                     ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY)) {
+            bindExportFilters(statement, exportDate);
+            statement.setFetchSize(Integer.MIN_VALUE);
+            try (ResultSet result = statement.executeQuery();
+                 BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(outputStream, StandardCharsets.UTF_8))) {
+                while (result.next()) {
+                    String deviceId = result.getString(1);
+                    if (deviceId == null || deviceId.indexOf('\r') >= 0 || deviceId.indexOf('\n') >= 0) {
+                        throw new IOException("Invalid OAID value in export data");
+                    }
+                    writer.write(deviceId);
+                    writer.newLine();
+                }
+                writer.flush();
+            }
+        } catch (SQLException e) {
+            throw new IOException("Unable to stream OAID export data", e);
+        }
+    }
+
+    private static void bindExportFilters(PreparedStatement statement, LocalDate exportDate) throws SQLException {
+        statement.setString(1, DATE_VALUE_FORMAT.format(exportDate));
+        statement.setString(2, TAG_ID);
+        statement.setInt(3, ACTIVATE_ACT);
+        statement.setString(4, MEDIA);
+    }
+
+    private DataSource requireDataSource() {
+        if (dataSource == null) {
+            throw new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE,
+                    "TiDB datasource is not configured");
+        }
+        return dataSource;
+    }
+}