tmp_task.py 3.4 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283
  1. import datetime
  2. import os
  3. import sys
  4. import pandas as pd
  5. import yaml
  6. from common_func import mysql_replace_into
  7. curr_path = os.path.abspath(os.path.dirname(__file__))
  8. project_root_path = curr_path[:curr_path.find("video_to_word") + len("video_to_word")]
  9. sys.path.append(project_root_path)
  10. from common_func import get_db_engine, get_logger
  11. from get_material_and_script.get_script_from_tengxunyun import GetScriptFromTengXunYunServer
  12. if __name__ == '__main__':
  13. # 创建日志对象
  14. logger = get_logger(log_file_name="/data/pythonProject/video_to_word/logs/get_high_quality_material.log",
  15. log_name='get_high_quality_material_logger')
  16. logger.info("get_high_quality_material_logger started! id of logger is: %s" % id(logger))
  17. # 1 读取配置文件
  18. with open('/data/pythonProject/video_to_word/config/config.yaml', mode='r', encoding='utf-8') as f:
  19. config = yaml.load(f.read(), Loader=yaml.FullLoader)
  20. # 1-1 数据库连接引擎,依据开发环境/生产环境 进行切换
  21. # 读数据库引擎使用生产数据库,写数据库引擎依据系统环境进行切换(测试数据库/生产数据库)
  22. # 注意:该项目的读和写 都使用测试数据库
  23. # TODO 等数据库迁移,服务上线后需要依据环境进行切换
  24. ai_word_engine = get_db_engine(config['ai_word_dev_db'])
  25. jeecg_boot_engine = get_db_engine(config['jeecg_boot_product_db'])
  26. # 1-2 分批写入数据库的行数
  27. chunk_size = config['chunk_size']
  28. # 1-3 渠道编码&名称
  29. source_name = config['source_name_map']
  30. # 2 获取素材编码和素材url
  31. project_id = 1860892
  32. query_word = '爱奇艺极速版(新)'
  33. # `channel_type` int(2) DEFAULT '0' COMMENT '0:自产 1:素造',
  34. sql = f"select signature,url video_url, channel_type from ctop_kuaishou_video_get where account_id in " \
  35. f"(select account_id from ctop_user_allocation where project_id = {project_id}) " \
  36. f"and channel_type = 0 and stat_date>='2021-08-01' group by signature"
  37. df = pd.read_sql(sql, jeecg_boot_engine)
  38. df['query_word'] = query_word
  39. df['project_id'] = project_id
  40. df['stat_date'] = datetime.date.today().strftime("%Y-%m-%d")
  41. df.to_sql(name="ctop_ai_video_info_from_huichuang",
  42. con=ai_word_engine,
  43. if_exists='append',
  44. index=False,
  45. chunksize=1000,
  46. method=mysql_replace_into)
  47. # 0 记录任务执行情况的字段
  48. size = 0
  49. message = ""
  50. task_status = 0
  51. inst = None
  52. # 2 调用腾讯云的语音转脚本服务,获取脚本
  53. if not df.empty:
  54. get_script_ins = GetScriptFromTengXunYunServer(logger=logger,
  55. db_engine=ai_word_engine,
  56. task_df=df[['signature', 'video_url']],
  57. task_ids=None)
  58. get_script_ins.submit_task()
  59. if get_script_ins.task_ids:
  60. get_script_ins.get_result()
  61. size = len(get_script_ins.task_ids)
  62. else:
  63. size = 0
  64. # 3 任务执行情况写入数据库
  65. task_info = {'query_word': query_word,
  66. 'stat_date': datetime.date.today().strftime('%Y-%m-%d'),
  67. 'source_code': 1,
  68. 'size': size}
  69. logger.info(f"{task_info}")