liyuyi@c-top.com.cn 4 years ago
parent
commit
789006720b
2 changed files with 95 additions and 17 deletions
  1. 72 0
      get_data.py
  2. 23 17
      main.py

+ 72 - 0
get_data.py

@@ -0,0 +1,72 @@
+from concurrent.futures import ThreadPoolExecutor
+import requests
+import datetime
+import json
+import pandas as pd
+import uuid
+import traceback
+from sqlalchemy import create_engine
+from urllib import parse
+from urllib.parse import urlencode
+
+if __name__ == '__main__':
+    online_db_con_str = "mysql+pymysql://%s:%s@%s:%d/%s" % ("readonly", parse.quote_plus("hcst@2021"), "139.186.27.96", 3390, "jeecg-boot")
+    online_engine = create_engine(online_db_con_str, connect_args={'charset': 'utf8'})
+
+    test_db_con_str = "mysql+pymysql://%s:%s@%s:%d/%s" % ("hcst", parse.quote_plus("hcst@2020"), "139.186.165.84", 3306, "db_ai_word")
+    test_engine = create_engine(test_db_con_str, connect_args={'charset': 'utf8'})
+
+    task_submit_url = 'http://139.186.165.84:31012/asr/task/submit'
+    task_result_url = 'http://139.186.165.84:31012/asr/task/result'
+
+    # 1、获取素材信息
+    get_material_info_sql = """
+    select t2.signature, t2.url
+    from
+    (select signature, account_id from ctop_kuaishou_report_daily_material 
+           where account_id in (select account_id from ctop_user_allocation where project_id = 458)
+           group by signature
+           having sum(activation) >= 50) t1
+    left join
+    ctop_kuaishou_video_get t2
+    on t1.signature = t2.signature and t1.account_id = t2.account_id
+    where t2.signature is not null
+    """
+    material_info_df = pd.read_sql(get_material_info_sql, online_engine)
+    material_info_df = material_info_df[(~material_info_df.signature.isnull()) & (~material_info_df.url.isnull())]
+
+    # 2 获取已经提交的任务
+    submit_task_sql = """select task_id, md5 signature from tb_asr_result """
+    submit_task_df = pd.read_sql(submit_task_sql, test_engine)
+
+    # 3 需要提交的任务
+    to_submit_task_df = material_info_df[~material_info_df.signature.isin(submit_task_df.signature.values)]
+
+    # 4 任务提交
+    for index, row in to_submit_task_df.iterrows():
+        material_md5 = row['signature']
+        material_url = row['url']
+        print(material_md5, material_url)
+        request_data = {"md5": material_md5, "url": material_url}
+        # 表单形式的入参,不是 json 块入参,请求方式不一样
+        request_full_path = task_submit_url + '?' + urlencode(request_data)
+        request = requests.post(request_full_path)
+        try:
+            result = json.loads(request.text)
+            print(result)
+        except:
+            print("error", request.text)
+
+    # 5、任务获取
+    get_result_sql = """select task_id from tb_asr_result where  word_text is null"""
+    get_result_df = pd.read_sql(get_result_sql, test_engine)
+    for index, row in get_result_df.iterrows():
+        task_id = row['task_id']
+        request_data = {'task_id': task_id}
+        request_full_path = task_result_url + '?' + urlencode(request_data)
+        request = requests.post(request_full_path)
+        try:
+            result = json.loads(request.text)
+            print(task_id, result['status'])
+        except:
+            print("error", task_id, request.text)

+ 23 - 17
main.py

@@ -8,8 +8,8 @@ import requests
 import uuid
 import os
 import uvicorn
-from asr_client import send_asr_request,send_task_request
-from database import insert,update,query,Task
+from asr_client import send_asr_request, send_task_request
+from database import insert, update, query, Task
 from datetime import datetime
 from fastapi.middleware.cors import CORSMiddleware
 import hashlib
@@ -39,37 +39,43 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
+
 class QueryItem():
-    md5:Optional[str]=None
-    url:Optional[str]=None
+    md5: Optional[str] = None
+    url: Optional[str] = None
+
 
 @app.get('/')
 def index():
     return {'message': '你已经正确创建 FastApi 服务!'}
 
+
 @app.post('/asr/task/submit')
-def task_submit(md5:str,url:str):
+def task_submit(md5: str, url: str):
     json = send_asr_request(url)
-    task = Task(md5=md5,task_id=json.Data.TaskId,task_result=json.to_json_string(),task_status=1)
+    task = Task(md5=md5, task_id=json.Data.TaskId, task_result=json.to_json_string(), task_status=1)
     insert(task)
-    return {'code':0,'taskId':json.Data.TaskId}
+    return {'code': 0, 'taskId': json.Data.TaskId}
+
 
 @app.post('/asr/task/result')
-def task_submit(task_id:int):
+def task_submit(task_id: int):
     json = send_task_request(task_id)
-    task = query(None,None,task_id)[0]
-    task.task_status=json.Data.Status
+    task = query(None, None, task_id)[0]
+    task.task_status = json.Data.Status
     task.task_result = json.to_json_string()
     if json.Data.Status == 2:
         task.word_text = json.Data.ResultDetail[0].FinalSentence
         task.word_split = json.Data.ResultDetail[0].SliceSentence
     update(task)
-    return {'code':0,'status':json.Data.StatusStr}
+    return {'code': 0, 'status': json.Data.StatusStr}
+
 
 @app.post('/asr/task/list')
-def task_submit(task_status:int):
-    task = query(None,task_status,None)
-    return {'code':0,'data':task}
+def task_submit(task_status: int):
+    task = query(None, task_status, None)
+    return {'code': 0, 'data': task}
+
 
 # @app.post('/asr/task/result')
 # def retry(item:Item):
@@ -97,7 +103,7 @@ def task_submit(task_status:int):
 
 #     return {'code':0,'data':{'taskId': task.id}}
 
-if __name__ == '__main__':
-    uvicorn.run(app='main:app', host="0.0.0.0", port=8000, reload=True, debug=True)
 
-#gunicorn main:app -w 4 -k uvicorn.workers.UvicornWorker #线上启动命令
+if __name__ == '__main__':
+    uvicorn.run(app='main:app', host="0.0.0.0", port=31012, reload=True, debug=True)
+# gunicorn main:app -w 4 -k uvicorn.workers.UvicornWorker #线上启动命令