12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879 |
- from concurrent.futures import ThreadPoolExecutor
- import requests
- import datetime
- import json
- import pandas as pd
- import uuid
- import traceback
- from sqlalchemy import create_engine
- from urllib import parse
- from urllib.parse import urlencode
- if __name__ == '__main__':
- online_db_con_str = "mysql+pymysql://%s:%s@%s:%d/%s" % ("readonly", parse.quote_plus("hcst@2021"), "139.186.27.96", 3390, "jeecg-boot")
- online_engine = create_engine(online_db_con_str, connect_args={'charset': 'utf8'})
- test_db_con_str = "mysql+pymysql://%s:%s@%s:%d/%s" % ("hcst", parse.quote_plus("hcst@2020"), "139.186.165.84", 3306, "db_ai_word")
- test_engine = create_engine(test_db_con_str, connect_args={'charset': 'utf8'})
- task_submit_url = 'http://139.186.165.84:31013/asr/task/submit'
- task_result_url = 'http://139.186.165.84:31013/asr/task/result'
- # 1、获取素材信息
- # get_material_info_sql = """
- # select t2.signature, t2.url
- # from
- # (select signature, account_id from ctop_kuaishou_report_daily_material
- # where account_id in (select account_id from ctop_user_allocation where project_id = 458)
- # group by signature
- # having sum(activation) <= 10 and sum(activation) >= 1
- # ) t1
- # left join
- # ctop_kuaishou_video_get t2
- # on t1.signature = t2.signature and t1.account_id = t2.account_id
- # where t2.signature is not null
- # """
- # material_info_df = pd.read_sql(get_material_info_sql, online_engine)
- # material_info_df = material_info_df[(~material_info_df.signature.isnull()) & (~material_info_df.url.isnull())]
- # N = 3000 if len(material_info_df) >= 3000 else len(material_info_df)
- # material_info_df = material_info_df.sample(n=N, random_state=2077)
- # material_info_df = pd.read_csv('merge_df_taote.csv')
- # material_info_df = pd.read_csv('merge_df_zhifubao.csv')
- # material_info_df = material_info_df[(~material_info_df.signature.isnull()) & (~material_info_df.url.isnull())]
- # 2 获取已经提交的任务
- # submit_task_sql = """select task_id, md5 signature from tb_asr_result """
- # submit_task_df = pd.read_sql(submit_task_sql, test_engine)
- # 3 需要提交的任务
- # to_submit_task_df = material_info_df[~material_info_df.signature.isin(submit_task_df.signature.values)]
- # 4 任务提交
- # for index, row in to_submit_task_df.iterrows():
- # material_md5 = row['signature']
- # material_url = row['url']
- # print(material_md5, material_url)
- # request_data = {"md5": material_md5, "url": material_url}
- # # 表单形式的入参,不是 json 块入参,请求方式不一样
- # request_full_path = task_submit_url + '?' + urlencode(request_data)
- # request = requests.post(request_full_path)
- # try:
- # result = json.loads(request.text)
- # print(result)
- # except:
- # print("error", request.text)
- # 5、任务获取
- get_result_sql = """select task_id from tb_asr_result where word_text is null"""
- get_result_df = pd.read_sql(get_result_sql, test_engine)
- for index, row in get_result_df.iterrows():
- task_id = row['task_id']
- request_data = {'task_id': task_id}
- request_full_path = task_result_url + '?' + urlencode(request_data)
- request = requests.post(request_full_path)
- try:
- result = json.loads(request.text)
- print(task_id, result['status'])
- except:
- print("error", task_id, request.text)
|