|  | @@ -1,29 +1,43 @@
 | 
	
		
			
				|  |  | -import datetime
 | 
	
		
			
				|  |  |  import hashlib
 | 
	
		
			
				|  |  | +import traceback
 | 
	
		
			
				|  |  |  import uuid
 | 
	
		
			
				|  |  |  from concurrent.futures import ThreadPoolExecutor
 | 
	
		
			
				|  |  | +from datetime import date
 | 
	
		
			
				|  |  | +from datetime import timedelta
 | 
	
		
			
				|  |  |  from io import BytesIO
 | 
	
		
			
				|  |  |  from typing import Optional, List
 | 
	
		
			
				|  |  |  from urllib.parse import quote
 | 
	
		
			
				|  |  | -import pymysql
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  |  import pandas as pd
 | 
	
		
			
				|  |  |  import uvicorn
 | 
	
		
			
				|  |  |  import yaml
 | 
	
		
			
				|  |  |  from fastapi import FastAPI
 | 
	
		
			
				|  |  |  from fastapi.middleware.cors import CORSMiddleware
 | 
	
		
			
				|  |  |  from fastapi.responses import StreamingResponse
 | 
	
		
			
				|  |  | +from loguru import logger
 | 
	
		
			
				|  |  |  from pydantic import BaseModel, Field
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  |  from asr_client import send_asr_request, send_task_request
 | 
	
		
			
				|  |  | -from common_func import get_db_engine, mysql_replace_into
 | 
	
		
			
				|  |  | +from common_func import get_db_engine
 | 
	
		
			
				|  |  |  from config.url import toutiao_static_video_url
 | 
	
		
			
				|  |  |  from database import insert, update, query, Task
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | +logger.add("logs/loguru.{time:YYYY-MM-DD}.log",
 | 
	
		
			
				|  |  | +           rotation="00:00",
 | 
	
		
			
				|  |  | +           format="{time:YYYY-MM-DD HH:mm:ss,SSS} [{process}] [{thread}] {level} {file} {line} - {message}",
 | 
	
		
			
				|  |  | +           level="INFO")
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  |  with open('/data/pythonProject/video_to_word/config/config.yaml', mode='r', encoding='utf-8') as f:
 | 
	
		
			
				|  |  |      config = yaml.load(f.read(), Loader=yaml.FullLoader)
 | 
	
		
			
				|  |  |      source_name_map = config['source_name_map']
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -ai_word_engine = get_db_engine(config['ai_word_dev_db'])
 | 
	
		
			
				|  |  | +# 数据库连接引擎,依据开发、测试环境/生产环境 进行切换
 | 
	
		
			
				|  |  | +    mac = uuid.UUID(int=uuid.getnode()).hex[-12:]
 | 
	
		
			
				|  |  | +    if mac in ['5254003fa716', '52540003f5dd']:
 | 
	
		
			
				|  |  | +        ai_word_engine = get_db_engine(config['ai_word_dev_db'])
 | 
	
		
			
				|  |  | +    else:
 | 
	
		
			
				|  |  | +        ai_word_engine = get_db_engine(config['ai_word_product_db'])
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  |  threadPool = ThreadPoolExecutor(max_workers=4)
 | 
	
		
			
				|  |  |  app = FastAPI()
 | 
	
	
		
			
				|  | @@ -54,12 +68,12 @@ class QueryItem():
 | 
	
		
			
				|  |  |      url: Optional[str] = None
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -@app.get('/')
 | 
	
		
			
				|  |  | +@app.get('/', tags=['back-end task'])
 | 
	
		
			
				|  |  |  def index():
 | 
	
		
			
				|  |  |      return {'message': '你已经正确创建 FastApi 服务!'}
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -@app.post('/asr/task/submit')
 | 
	
		
			
				|  |  | +@app.post('/asr/task/submit', tags=['back-end task'])
 | 
	
		
			
				|  |  |  def task_submit(signature: str, url: str):
 | 
	
		
			
				|  |  |      json = send_asr_request(url)
 | 
	
		
			
				|  |  |      task = Task(signature=signature, task_id=json.Data.TaskId, task_result=json.to_json_string(), task_status=1)
 | 
	
	
		
			
				|  | @@ -67,7 +81,7 @@ def task_submit(signature: str, url: str):
 | 
	
		
			
				|  |  |      return {'code': 0, 'taskId': json.Data.TaskId}
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -@app.post('/asr/task/result')
 | 
	
		
			
				|  |  | +@app.post('/asr/task/result', tags=['back-end task'])
 | 
	
		
			
				|  |  |  def task_submit(task_id: int):
 | 
	
		
			
				|  |  |      json = send_task_request(task_id)
 | 
	
		
			
				|  |  |      task = query(None, None, task_id)[0]
 | 
	
	
		
			
				|  | @@ -85,150 +99,269 @@ def task_submit(task_id: int):
 | 
	
		
			
				|  |  |      return {'code': 0, 'status': json.Data.StatusStr}
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -@app.post('/asr/task/list')
 | 
	
		
			
				|  |  | +@app.post('/asr/task/list', tags=['back-end task'])
 | 
	
		
			
				|  |  |  def task_submit(task_status: int):
 | 
	
		
			
				|  |  |      task = query(None, task_status, None)
 | 
	
		
			
				|  |  |      return {'code': 0, 'data': task}
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -class QueryWordItem(BaseModel):
 | 
	
		
			
				|  |  | -    query_word: str = Field(..., description="查询词", min_length=1)
 | 
	
		
			
				|  |  | -    stat_date: str = Field(..., description="日期", min_length=10, max_length=10)
 | 
	
		
			
				|  |  | -    source: int = Field(..., description="来源,")
 | 
	
		
			
				|  |  | +class BaseResponse(BaseModel):
 | 
	
		
			
				|  |  | +    message: str = Field(..., description='消息')
 | 
	
		
			
				|  |  | +    success: bool = Field(..., description='true or false')
 | 
	
		
			
				|  |  | +    code: int = Field(..., description='')
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -@app.post('/export_excel/')
 | 
	
		
			
				|  |  | -def export_excel(item: List[QueryWordItem]):
 | 
	
		
			
				|  |  | -    video_df = pd.DataFrame()
 | 
	
		
			
				|  |  | -    if len(item) == 1:
 | 
	
		
			
				|  |  | -        # 单个条目,直接导出
 | 
	
		
			
				|  |  | -        pass
 | 
	
		
			
				|  |  | -    else:
 | 
	
		
			
				|  |  | -        # 1 从数据库获取视频数据
 | 
	
		
			
				|  |  | -        # 多个条目,如果同一个素材有多个查询词,则合并打上这多个查询词
 | 
	
		
			
				|  |  | -        for obj in item:
 | 
	
		
			
				|  |  | -            query_word = obj.query_word
 | 
	
		
			
				|  |  | -            stat_date = obj.stat_date
 | 
	
		
			
				|  |  | -            source = obj.source
 | 
	
		
			
				|  |  | -            sql = f"select signature, video_url, query_word, stat_date, {source} source from {source_name_map[source]['table']} " \
 | 
	
		
			
				|  |  | -                  f"where query_word = '{query_word}' " \
 | 
	
		
			
				|  |  | -                  f"and stat_date = '{stat_date}'"
 | 
	
		
			
				|  |  | -            df = pd.read_sql(sql, ai_word_engine)
 | 
	
		
			
				|  |  | -            video_df = video_df.append(df)
 | 
	
		
			
				|  |  | +class TaskDetail(BaseModel):
 | 
	
		
			
				|  |  | +    source_name: str = Field('内部创意', description='数据来源名称')
 | 
	
		
			
				|  |  | +    query_word: str = Field('红包', description='关键词')
 | 
	
		
			
				|  |  | +    stat_date: str = Field('2021-11-11', description='日期')
 | 
	
		
			
				|  |  | +    script_num: int = Field(180, description='脚本数量')
 | 
	
		
			
				|  |  | +    task_status: str = Field('执行成功', description='状态')
 | 
	
		
			
				|  |  | +    number: int = Field(0, description='序号')
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -        # 按 'signature' + 'query_word' + 'stat_date' 进行去重
 | 
	
		
			
				|  |  | -        video_df.drop_duplicates(['signature', 'query_word', 'stat_date', 'source'], keep='last', inplace=True)
 | 
	
		
			
				|  |  | -        g = video_df.groupby('signature')
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -        query_word_lst_df = g.apply(lambda x: x['query_word'].unique())
 | 
	
		
			
				|  |  | -        query_word_lst_df.name = 'query_word_lst'
 | 
	
		
			
				|  |  | +class ConfigDetail(BaseModel):
 | 
	
		
			
				|  |  | +    config_id: str = Field(..., description="脚本配置id")
 | 
	
		
			
				|  |  | +    query_word_lst: List[str] = Field(..., description="关键词")
 | 
	
		
			
				|  |  | +    create_time: str = Field(..., description="创建时间")
 | 
	
		
			
				|  |  | +    operator: str = Field(..., description="创建人")
 | 
	
		
			
				|  |  | +    number: int = Field(..., description="序号")
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -        url_df = g.apply(lambda x: x['video_url'].values[0])
 | 
	
		
			
				|  |  | -        url_df.name = 'video_url'
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -        source_df = g.apply(lambda x: x['source'].values[0])
 | 
	
		
			
				|  |  | -        source_df.name = 'source'
 | 
	
		
			
				|  |  | +class TaskResponse(BaseResponse):
 | 
	
		
			
				|  |  | +    total_num: int = Field(0, description="总个数")
 | 
	
		
			
				|  |  | +    page_num: int = Field(1, description="第几页")
 | 
	
		
			
				|  |  | +    page_size: int = Field(10, description="每页个数")
 | 
	
		
			
				|  |  | +    config_id: str = Field('', description="脚本配置id")
 | 
	
		
			
				|  |  | +    result: List[TaskDetail] = Field(..., description="结果详情")
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -        video_query_word_df = pd.concat([query_word_lst_df, url_df, source_df], axis=1)
 | 
	
		
			
				|  |  | -        video_query_word_df.reset_index(inplace=True, drop=False)
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -        video_query_word_df['video_url'] = video_query_word_df.apply(
 | 
	
		
			
				|  |  | -            lambda row: toutiao_static_video_url + row['signature'] if row.get('source') == 2 else row['video_url'], axis=1)
 | 
	
		
			
				|  |  | +class ConfigResponse(BaseResponse):
 | 
	
		
			
				|  |  | +    total_num: int = Field(0, description="总个数")
 | 
	
		
			
				|  |  | +    page_num: int = Field(1, description="第几页")
 | 
	
		
			
				|  |  | +    page_size: int = Field(10, description="每页个数")
 | 
	
		
			
				|  |  | +    result: List[ConfigDetail] = Field(..., description="结果详情")
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -        # 2 根据第一步的视频数据获取脚本
 | 
	
		
			
				|  |  | -        if not video_query_word_df.empty:
 | 
	
		
			
				|  |  | -            sql = f"select signature, word_text from tb_asr_result where signature in " \
 | 
	
		
			
				|  |  | -                  f"{tuple(video_query_word_df.signature.values) if len(video_query_word_df.signature.values) > 1 else tuple(list(video_query_word_df.signature.values) * 2)} " \
 | 
	
		
			
				|  |  | -                  f"and task_status = 2"
 | 
	
		
			
				|  |  | -            script_df = pd.read_sql(sql, ai_word_engine)
 | 
	
		
			
				|  |  | -            out_df = video_query_word_df.merge(script_df, on='signature', how='inner')
 | 
	
		
			
				|  |  | -        else:
 | 
	
		
			
				|  |  | -            pass
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -        # 3 返回流数据
 | 
	
		
			
				|  |  | -        if not out_df.empty:
 | 
	
		
			
				|  |  | -            bio = BytesIO()
 | 
	
		
			
				|  |  | -            writer = pd.ExcelWriter(bio, engine='xlsxwriter')
 | 
	
		
			
				|  |  | -            out_df[['signature', 'query_word_lst', 'word_text', 'video_url']].to_excel(writer, index=False, encoding='utf8mb4')
 | 
	
		
			
				|  |  | -            writer.save()
 | 
	
		
			
				|  |  | -            bio.seek(0)
 | 
	
		
			
				|  |  | +class QueryWordItem(BaseModel):
 | 
	
		
			
				|  |  | +    query_word: str = Field("红包", description="查询词", min_length=1)
 | 
	
		
			
				|  |  | +    stat_date: str = Field("2021-11-16", description="日期", min_length=10, max_length=10)
 | 
	
		
			
				|  |  | +    source_code: int = Field(2, description="数据来源编码{1:'内部创意', 2:'巨量创意', 3:'开眼快创'}")
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -            # 组装header
 | 
	
		
			
				|  |  | -            now_date = datetime.date.today().strftime('%Y-%m-%d')
 | 
	
		
			
				|  |  | -            headers = {"content-type": "application/vnd.ms-excel",
 | 
	
		
			
				|  |  | -                       "content-disposition": f"attachment;filename={quote('优质素材脚本_')}{now_date}.xlsx"
 | 
	
		
			
				|  |  | -                       }
 | 
	
		
			
				|  |  | +class ScriptConfigLst(BaseModel):
 | 
	
		
			
				|  |  | +    start_date: Optional[date] = Field(date.today() + timedelta(days=-6), description="开始日期-用于查询")
 | 
	
		
			
				|  |  | +    end_date: Optional[date] = Field(date.today(), description="结束日期-用于查询")
 | 
	
		
			
				|  |  | +    search_word: Optional[str] = Field('', description="关键词-用于查询")
 | 
	
		
			
				|  |  | +    page_num: int = Field(1, description="第几页")
 | 
	
		
			
				|  |  | +    page_size: int = Field(10, description="每页的大小")
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -            return StreamingResponse(bio, media_type='xlsx', headers=headers)
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -    return None
 | 
	
		
			
				|  |  | +class QueryWordTaskInfoLst(BaseModel):
 | 
	
		
			
				|  |  | +    start_date: Optional[date] = Field(date.today() + timedelta(days=-30), description="开始日期-用于查询")
 | 
	
		
			
				|  |  | +    end_date: Optional[date] = Field(date.today(), description="结束日期-用于查询")
 | 
	
		
			
				|  |  | +    search_word: Optional[str] = Field('', description="关键词-用于查询")
 | 
	
		
			
				|  |  | +    page_num: int = Field(1, description="第几页")
 | 
	
		
			
				|  |  | +    page_size: int = Field(10, description="每页的大小")
 | 
	
		
			
				|  |  | +    config_id: Optional[str] = Field('', description="脚本配置id")
 | 
	
		
			
				|  |  | +    source_code: Optional[List[int]] = Field([0], description="数据来源编码{1:'内部创意', 2:'巨量创意', 3:'开眼快创'}")
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -class ScriptConfig(BaseModel):
 | 
	
		
			
				|  |  | -    query_word_lst: List = Field(..., description="关键词组")
 | 
	
		
			
				|  |  | +class AddScriptConfig(BaseModel):
 | 
	
		
			
				|  |  | +    query_word_lst: List[str] = Field(..., description="关键词组")
 | 
	
		
			
				|  |  |      operator: str = Field(..., description="操作者")
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | +    class Config:
 | 
	
		
			
				|  |  | +        schema_extra = {
 | 
	
		
			
				|  |  | +            "example": {
 | 
	
		
			
				|  |  | +                "query_word_lst": ["红包", "淘特"],
 | 
	
		
			
				|  |  | +                "operator": "龙猫"
 | 
	
		
			
				|  |  | +            }
 | 
	
		
			
				|  |  | +        }
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +@logger.catch
 | 
	
		
			
				|  |  | +@app.post('/export_script_file/', tags=['front-end interactive'],
 | 
	
		
			
				|  |  | +          description="导出文件",
 | 
	
		
			
				|  |  | +          summary='导出文件'
 | 
	
		
			
				|  |  | +          )
 | 
	
		
			
				|  |  | +def export_script_file(item: List[QueryWordItem]):
 | 
	
		
			
				|  |  | +    video_df = pd.DataFrame()
 | 
	
		
			
				|  |  | +    # 1 从数据库获取视频数据
 | 
	
		
			
				|  |  | +    # 如果同一个素材有多个查询词,则合并打上这多个查询词
 | 
	
		
			
				|  |  | +    for obj in item:
 | 
	
		
			
				|  |  | +        query_word = obj.query_word
 | 
	
		
			
				|  |  | +        stat_date = obj.stat_date
 | 
	
		
			
				|  |  | +        source_code = obj.source_code
 | 
	
		
			
				|  |  | +        sql = f"select signature, video_url, query_word, stat_date, {source_code} source_code from {source_name_map[source_code]['table']} " \
 | 
	
		
			
				|  |  | +              f"where query_word = '{query_word}' " \
 | 
	
		
			
				|  |  | +              f"and stat_date = '{stat_date}'"
 | 
	
		
			
				|  |  | +        df = pd.read_sql(sql, ai_word_engine)
 | 
	
		
			
				|  |  | +        video_df = video_df.append(df)
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +    # 按 'signature' + 'query_word' + 'stat_date' 进行去重
 | 
	
		
			
				|  |  | +    video_df.drop_duplicates(['signature', 'query_word', 'stat_date', 'source_code'], keep='last', inplace=True)
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +    video_query_word_df = video_df.groupby('signature').apply(lambda x: pd.Series({'query_word_lst': x['query_word'].unique(),
 | 
	
		
			
				|  |  | +                                                                                   'video_url': x['video_url'].values[0],
 | 
	
		
			
				|  |  | +                                                                                   'source_code': x['source_code'].values[0]}))
 | 
	
		
			
				|  |  | +    video_query_word_df.reset_index(inplace=True, drop=False)
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +    # 如果来源==2 (头条巨量引擎),把视频链接替换为永久链接
 | 
	
		
			
				|  |  | +    video_query_word_df['video_url'] = video_query_word_df.apply(
 | 
	
		
			
				|  |  | +        lambda row: toutiao_static_video_url + row['signature'] if row.get('source_code') == 2 else row['video_url'], axis=1)
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +    # 2 根据第一步的视频数据获取脚本
 | 
	
		
			
				|  |  | +    if not video_query_word_df.empty:
 | 
	
		
			
				|  |  | +        signature_lst = list(video_query_word_df.signature.values) if len(video_query_word_df.signature.values) > 1 \
 | 
	
		
			
				|  |  | +            else list(video_query_word_df.signature.values) * 2
 | 
	
		
			
				|  |  | +        sql = f"select signature, word_text from tb_asr_result where signature in {tuple(signature_lst)}" \
 | 
	
		
			
				|  |  | +              f"and word_text is not null"
 | 
	
		
			
				|  |  | +        script_df = pd.read_sql(sql, ai_word_engine)
 | 
	
		
			
				|  |  | +        out_df = video_query_word_df.merge(script_df, on='signature', how='inner')
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +    # 3 返回流数据
 | 
	
		
			
				|  |  | +    if not out_df.empty:
 | 
	
		
			
				|  |  | +        bio = BytesIO()
 | 
	
		
			
				|  |  | +        writer = pd.ExcelWriter(bio, engine='xlsxwriter')
 | 
	
		
			
				|  |  | +        out_df[['signature', 'query_word_lst', 'word_text', 'video_url']].to_excel(writer, index=False, encoding='utf8mb4')
 | 
	
		
			
				|  |  | +        writer.save()
 | 
	
		
			
				|  |  | +        bio.seek(0)
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +        # 组装header
 | 
	
		
			
				|  |  | +        now_date = date.today().strftime('%Y-%m-%d')
 | 
	
		
			
				|  |  | +        headers = {"content-type": "application/vnd.ms-excel",
 | 
	
		
			
				|  |  | +                   "content-disposition": f"attachment;filename={quote('优质素材脚本_')}{now_date}.xlsx"
 | 
	
		
			
				|  |  | +                   }
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +        return StreamingResponse(bio, media_type='xlsx', headers=headers)
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -@app.post('/get_script_config_lst/')
 | 
	
		
			
				|  |  | -def get_script_config_lst():
 | 
	
		
			
				|  |  | -    pass
 | 
	
		
			
				|  |  | -
 | 
	
		
			
				|  |  | -
 | 
	
		
			
				|  |  | -@app.post('/add_script_config/')
 | 
	
		
			
				|  |  | -def add_script_config(item: ScriptConfig):
 | 
	
		
			
				|  |  | -    config_id = str(uuid.uuid4())
 | 
	
		
			
				|  |  | -    config_lst = []
 | 
	
		
			
				|  |  | -    for query_word in item.query_word_lst:
 | 
	
		
			
				|  |  | -        sql = f"select * from ctop_ai_query_word where query_word = '{query_word}'"
 | 
	
		
			
				|  |  | -        query_word_df = pd.read_sql(sql, ai_word_engine)
 | 
	
		
			
				|  |  | -        if not query_word_df.empty:
 | 
	
		
			
				|  |  | -            # 更新 ctop_ai_query_word
 | 
	
		
			
				|  |  | -            query_word_id = query_word_df.query_word_id.values[0]
 | 
	
		
			
				|  |  | -            script_config_conn_num = query_word_df.script_config_conn_num.values[0] + 1
 | 
	
		
			
				|  |  | -            db_con = pymysql.connect(**config['ai_word_dev_db'])
 | 
	
		
			
				|  |  | -            db_cur = db_con.cursor()
 | 
	
		
			
				|  |  | -            sql = f"update ctop_ai_query_word set script_config_conn_num = {script_config_conn_num} where query_word_id = '{query_word_id}'"
 | 
	
		
			
				|  |  | -            db_cur.execute(sql)
 | 
	
		
			
				|  |  | -            db_con.commit()
 | 
	
		
			
				|  |  | -            db_con.close()
 | 
	
		
			
				|  |  | -            # update_query_word_df = pd.DataFrame([{"query_word_id": query_word_id,
 | 
	
		
			
				|  |  | -            #                                       "query_word": query_word,
 | 
	
		
			
				|  |  | -            #                                       "script_conn_num": script_conn_num}])
 | 
	
		
			
				|  |  | -            # update_query_word_df.to_sql(name="ctop_ai_query_word",
 | 
	
		
			
				|  |  | -            #                             con=ai_word_engine,
 | 
	
		
			
				|  |  | -            #                             if_exists="append",
 | 
	
		
			
				|  |  | -            #                             method=mysql_replace_into,
 | 
	
		
			
				|  |  | -            #                             index=False)
 | 
	
		
			
				|  |  | +    return None
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -        else:
 | 
	
		
			
				|  |  | -            query_word_id = str(uuid.uuid4())
 | 
	
		
			
				|  |  | -            new_query_word_df = pd.DataFrame([{"query_word_id": query_word_id, "query_word": query_word, "script_config_conn_num": 1}])
 | 
	
		
			
				|  |  | -            new_query_word_df.to_sql(name="ctop_ai_query_word",
 | 
	
		
			
				|  |  | -                                     con=ai_word_engine,
 | 
	
		
			
				|  |  | -                                     if_exists="append",
 | 
	
		
			
				|  |  | -                                     index=False)
 | 
	
		
			
				|  |  | -
 | 
	
		
			
				|  |  | -        config_lst.append({"config_id": config_id, "query_word_id": query_word_id})
 | 
	
		
			
				|  |  | -
 | 
	
		
			
				|  |  | -    # 新增配置记录插入到 ctop_ai_script_query_word_config
 | 
	
		
			
				|  |  | -    config_df = pd.DataFrame(config_lst)
 | 
	
		
			
				|  |  | -    config_df['operator'] = item.operator
 | 
	
		
			
				|  |  | -    config_df['operate_type'] = 1
 | 
	
		
			
				|  |  | -    config_df.to_sql(name="ctop_ai_script_query_word_config",
 | 
	
		
			
				|  |  | -                     con=ai_word_engine,
 | 
	
		
			
				|  |  | -                     if_exists='append',
 | 
	
		
			
				|  |  | -                     index=False)
 | 
	
		
			
				|  |  | -    return {"code": 0, "message": "success"}
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | +@logger.catch
 | 
	
		
			
				|  |  | +@app.post('/get_script_config_lst/', tags=['front-end interactive'], response_model=ConfigResponse,
 | 
	
		
			
				|  |  | +          description="脚本配置列表",
 | 
	
		
			
				|  |  | +          summary='脚本配置列表'
 | 
	
		
			
				|  |  | +          )
 | 
	
		
			
				|  |  | +def get_script_config_lst(item: ScriptConfigLst):
 | 
	
		
			
				|  |  | +    try:
 | 
	
		
			
				|  |  | +        end_date = item.end_date + timedelta(days=1)
 | 
	
		
			
				|  |  | +        sql = f"select * from ctop_ai_script_query_word_config " \
 | 
	
		
			
				|  |  | +              f"where start_time >= '{item.start_date}' " \
 | 
	
		
			
				|  |  | +              f"and start_time < '{end_date}' " \
 | 
	
		
			
				|  |  | +              f"and ('{item.search_word}' = '' or query_word like '%%{item.search_word}%%')"
 | 
	
		
			
				|  |  | +        org_df = pd.read_sql(sql, ai_word_engine)
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +        g_df = org_df.groupby('config_id').apply(lambda x: pd.Series({'query_word_lst': list(x['query_word'].unique()),
 | 
	
		
			
				|  |  | +                                                                      'operator': x['operator'].min(),
 | 
	
		
			
				|  |  | +                                                                      'create_time': str(x['start_time'].min())}))
 | 
	
		
			
				|  |  | +        g_df.reset_index(drop=False, inplace=True)
 | 
	
		
			
				|  |  | +        g_df.sort_values(by='create_time', ascending=False, inplace=True)
 | 
	
		
			
				|  |  | +        g_df['number'] = list(range(1, len(g_df) + 1))
 | 
	
		
			
				|  |  | +        total_num = g_df.shape[0]
 | 
	
		
			
				|  |  | +        detail = g_df.iloc[(item.page_num - 1) * item.page_size: item.page_num * item.page_size].to_dict('records')
 | 
	
		
			
				|  |  | +        response = {'code': 0,
 | 
	
		
			
				|  |  | +                    "message": "查询成功",
 | 
	
		
			
				|  |  | +                    "success": True,
 | 
	
		
			
				|  |  | +                    "result": detail,
 | 
	
		
			
				|  |  | +                    "total_num": total_num,
 | 
	
		
			
				|  |  | +                    "page_num": item.page_num,
 | 
	
		
			
				|  |  | +                    "page_size": item.page_size}
 | 
	
		
			
				|  |  | +        logger.info(f"request body: {item}, response body: {response}")
 | 
	
		
			
				|  |  | +        return response
 | 
	
		
			
				|  |  | +    except:
 | 
	
		
			
				|  |  | +        response = {"code": -1,
 | 
	
		
			
				|  |  | +                    "message": traceback.format_exc(),
 | 
	
		
			
				|  |  | +                    "success": False,
 | 
	
		
			
				|  |  | +                    "result": None}
 | 
	
		
			
				|  |  | +        logger.error(f"request body: {item}, response body: {response}")
 | 
	
		
			
				|  |  | +        return response
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +@logger.catch
 | 
	
		
			
				|  |  | +@app.post('/get_query_word_task_info_lst/', tags=['front-end interactive'], response_model=TaskResponse,
 | 
	
		
			
				|  |  | +          description="脚本数据导出列表",
 | 
	
		
			
				|  |  | +          summary='脚本数据导出列表'
 | 
	
		
			
				|  |  | +          )
 | 
	
		
			
				|  |  | +def get_query_word_task_info_lst(item: QueryWordTaskInfoLst):
 | 
	
		
			
				|  |  | +    try:
 | 
	
		
			
				|  |  | +        end_date = item.end_date + timedelta(days=1)
 | 
	
		
			
				|  |  | +        source_code_lst = item.source_code * 2 if len(item.source_code) == 1 else item.source_code
 | 
	
		
			
				|  |  | +        if item.config_id != '':
 | 
	
		
			
				|  |  | +            sql = f"select distinct(query_word) query_word from ctop_ai_script_query_word_config where config_id = {item.config_id}"
 | 
	
		
			
				|  |  | +            query_word_lst = list(pd.read_sql(sql, ai_word_engine).query_word.values)
 | 
	
		
			
				|  |  | +            if len(query_word_lst) > 0:
 | 
	
		
			
				|  |  | +                query_word_lst = query_word_lst * 2 if len(query_word_lst) == 1 else query_word_lst
 | 
	
		
			
				|  |  | +                sql = f"select * from ctop_ai_query_word_task_record where query_word in {tuple(query_word_lst)}" \
 | 
	
		
			
				|  |  | +                      f"and stat_date >= '{item.start_date}' and stat_date < '{end_date}' " \
 | 
	
		
			
				|  |  | +                      f"and ('{item.source_code}' = '[0]' or source_code in {tuple(source_code_lst)}) " \
 | 
	
		
			
				|  |  | +                      f"and ('{item.search_word}' = '' or query_word = '{item.search_word}')"
 | 
	
		
			
				|  |  | +                df = pd.read_sql(sql, ai_word_engine)
 | 
	
		
			
				|  |  | +        else:
 | 
	
		
			
				|  |  | +            sql = f"select * from ctop_ai_query_word_task_record where " \
 | 
	
		
			
				|  |  | +                  f"stat_date >= '{item.start_date}' and stat_date < '{end_date}' " \
 | 
	
		
			
				|  |  | +                  f"and ('{item.source_code}' = '[0]' or source_code in {tuple(source_code_lst)}) " \
 | 
	
		
			
				|  |  | +                  f"and ('{item.search_word}' = '' or query_word = '{item.search_word}')"
 | 
	
		
			
				|  |  | +            df = pd.read_sql(sql, ai_word_engine)
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -if __name__ == '__main__':
 | 
	
		
			
				|  |  | -    # 1 读取配置文件
 | 
	
		
			
				|  |  | +        df['source_name'] = df['source_code'].apply(lambda x: source_name_map[x]['name'])
 | 
	
		
			
				|  |  | +        df = df[['source_name', 'query_word', 'stat_date', 'script_num', 'task_status']]
 | 
	
		
			
				|  |  | +        df.sort_values(['stat_date', 'source_name', 'query_word'], ascending=False, inplace=True)
 | 
	
		
			
				|  |  | +        df['number'] = list(range(1, len(df) + 1))
 | 
	
		
			
				|  |  | +        total_num = df.shape[0]
 | 
	
		
			
				|  |  | +        detail = df.iloc[(item.page_num - 1) * item.page_size: item.page_num * item.page_size].to_dict('records')
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +        response = {'code': 0,
 | 
	
		
			
				|  |  | +                    "message": "查询成功",
 | 
	
		
			
				|  |  | +                    "success": True,
 | 
	
		
			
				|  |  | +                    "result": detail,
 | 
	
		
			
				|  |  | +                    "total_num": total_num,
 | 
	
		
			
				|  |  | +                    "page_num": item.page_num,
 | 
	
		
			
				|  |  | +                    "page_size": item.page_size,
 | 
	
		
			
				|  |  | +                    "config_id": item.config_id}
 | 
	
		
			
				|  |  | +        logger.info(f"request body: {item}, response body: {response}")
 | 
	
		
			
				|  |  | +        return response
 | 
	
		
			
				|  |  | +    except:
 | 
	
		
			
				|  |  | +        response = {"code": -1,
 | 
	
		
			
				|  |  | +                    "message": traceback.format_exc(),
 | 
	
		
			
				|  |  | +                    "success": False,
 | 
	
		
			
				|  |  | +                    "result": None}
 | 
	
		
			
				|  |  | +        logger.error(f"request body: {item}, response body: {response}")
 | 
	
		
			
				|  |  | +        return response
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +@logger.catch
 | 
	
		
			
				|  |  | +@app.post('/add_script_config/',
 | 
	
		
			
				|  |  | +          tags=['front-end interactive'],
 | 
	
		
			
				|  |  | +          description="新增脚本配置",
 | 
	
		
			
				|  |  | +          summary='新增脚本配置',
 | 
	
		
			
				|  |  | +          response_model=BaseResponse)
 | 
	
		
			
				|  |  | +def add_script_config(item: AddScriptConfig):
 | 
	
		
			
				|  |  | +    try:
 | 
	
		
			
				|  |  | +        # 按查询词拆分配置记录
 | 
	
		
			
				|  |  | +        config_id = str(uuid.uuid4())
 | 
	
		
			
				|  |  | +        config_df = pd.DataFrame(data=item.query_word_lst, columns=['query_word'])
 | 
	
		
			
				|  |  | +        config_df['config_id'] = config_id
 | 
	
		
			
				|  |  | +        config_df['operator'] = item.operator
 | 
	
		
			
				|  |  | +        config_df['operate_type'] = 1
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +        # 新增配置记录插入到 ctop_ai_script_query_word_config
 | 
	
		
			
				|  |  | +        config_df.to_sql(name="ctop_ai_script_query_word_config",
 | 
	
		
			
				|  |  | +                         con=ai_word_engine,
 | 
	
		
			
				|  |  | +                         if_exists='append',
 | 
	
		
			
				|  |  | +                         index=False)
 | 
	
		
			
				|  |  | +        logger.info(f"request body: {item}, code:0, message: add_script_config success")
 | 
	
		
			
				|  |  | +        return {"code": 0,
 | 
	
		
			
				|  |  | +                "message": "add success",
 | 
	
		
			
				|  |  | +                "success": True}
 | 
	
		
			
				|  |  | +    except:
 | 
	
		
			
				|  |  | +        logger.error(f"request body: {item}, code:-1, message: add_script_config fail {traceback.format_exc()}")
 | 
	
		
			
				|  |  | +        return {"code": -1,
 | 
	
		
			
				|  |  | +                "message": traceback.format_exc(),
 | 
	
		
			
				|  |  | +                "success": False}
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -    # test_items = [{'query_word': '红包', 'stat_date': '2021-10-28', 'source': 2},
 | 
	
		
			
				|  |  | -    #               {'query_word': '红包', 'stat_date': '2021-10-28', 'source': 3},
 | 
	
		
			
				|  |  | -    #               {'query_word': '赚钱', 'stat_date': '2021-10-28', 'source': 2},
 | 
	
		
			
				|  |  | -    #               {'query_word': '赚钱', 'stat_date': '2021-10-28', 'source': 3}]
 | 
	
		
			
				|  |  | -    # export_excel(test_items)
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | +if __name__ == '__main__':
 | 
	
		
			
				|  |  |      uvicorn.run(app='main:app', host="0.0.0.0", port=31013, reload=True, debug=True)
 | 
	
		
			
				|  |  |  # gunicorn main:app -w 4 -k uvicorn.workers.UvicornWorker #线上启动命令
 |