tmp_task.py 3.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384
  1. import datetime
  2. import os
  3. import sys
  4. import pandas as pd
  5. import yaml
  6. from common_func import mysql_replace_into
  7. curr_path = os.path.abspath(os.path.dirname(__file__))
  8. project_root_path = curr_path[:curr_path.find("video_to_word") + len("video_to_word")]
  9. sys.path.append(project_root_path)
  10. from common_func import get_db_engine, get_logger
  11. from get_material_and_script.get_script_from_tengxunyun import GetScriptFromTengXunYunServer
  12. from get_material_and_script.get_material_from_huichuang import GetMaterialFromHuiChuang
  13. if __name__ == '__main__':
  14. # 创建日志对象
  15. logger = get_logger(log_file_name="/data/pythonProject/video_to_word/logs/get_high_quality_material.log",
  16. log_name='get_high_quality_material_logger')
  17. logger.info("get_high_quality_material_logger started! id of logger is: %s" % id(logger))
  18. # 1 读取配置文件
  19. with open('/data/pythonProject/video_to_word/config/config.yaml', mode='r', encoding='utf-8') as f:
  20. config = yaml.load(f.read(), Loader=yaml.FullLoader)
  21. # 1-1 数据库连接引擎,依据开发环境/生产环境 进行切换
  22. # 读数据库引擎使用生产数据库,写数据库引擎依据系统环境进行切换(测试数据库/生产数据库)
  23. # 注意:该项目的读和写 都使用测试数据库
  24. # TODO 等数据库迁移,服务上线后需要依据环境进行切换
  25. ai_word_engine = get_db_engine(config['ai_word_dev_db'])
  26. jeecg_boot_engine = get_db_engine(config['jeecg_boot_product_db'])
  27. # 1-2 分批写入数据库的行数
  28. chunk_size = config['chunk_size']
  29. # 1-3 渠道编码&名称
  30. source_name = config['source_name_map']
  31. # 2 获取素材编码和素材url
  32. project_id = 458
  33. query_word = '淘特'
  34. # `channel_type` int(2) DEFAULT '0' COMMENT '0:自产 1:素造',
  35. sql = f"select signature,url video_url, channel_type from ctop_kuaishou_video_get where account_id in " \
  36. f"(select account_id from ctop_user_allocation where project_id = {project_id}) " \
  37. f"and channel_type = 0 and stat_date>='2021-08-01' group by signature limit 100"
  38. df = pd.read_sql(sql, jeecg_boot_engine)
  39. df['query_word'] = query_word
  40. df['stat_date'] = datetime.date.today().strftime("%Y-%m-%d")
  41. df.to_sql(name="ctop_ai_video_info_from_huichuang",
  42. con=ai_word_engine,
  43. if_exists='append',
  44. index=False,
  45. chunksize=1000,
  46. method=mysql_replace_into)
  47. # 0 记录任务执行情况的字段
  48. size = 0
  49. message = ""
  50. task_status = 0
  51. inst = None
  52. # 2 调用腾讯云的语音转脚本服务,获取脚本
  53. if not df.emtpy:
  54. inst.video_df = inst.video_df[~inst.video_df.signature.isnull()]
  55. get_script_ins = GetScriptFromTengXunYunServer(logger=logger,
  56. db_engine=ai_word_engine,
  57. task_df=df[['signature', 'video_url']],
  58. task_ids=None)
  59. get_script_ins.submit_task()
  60. if get_script_ins.task_ids:
  61. get_script_ins.get_result()
  62. size = len(get_script_ins.task_ids)
  63. else:
  64. size = 0
  65. # 3 任务执行情况写入数据库
  66. task_info = {'query_word': query_word,
  67. 'stat_date': datetime.date.today().strftime('%Y-%m-%d'),
  68. 'source_code': 1,
  69. 'size': size}
  70. logger.info(f"{task_info}")