|
@@ -7,6 +7,18 @@ from functools import reduce
|
|
|
from itertools import product
|
|
|
from datetime import datetime
|
|
|
from statsmodels.stats.proportion import proportions_ztest
|
|
|
+import traceback
|
|
|
+
|
|
|
+import logging
|
|
|
+from concurrent_log import ConcurrentTimedRotatingFileHandler
|
|
|
+
|
|
|
+log_formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%Y/%m/%d %I:%M:%S %p')
|
|
|
+log_handler = ConcurrentTimedRotatingFileHandler("logs/BayesCombine.log", when="midnight", backupCount=100)
|
|
|
+log_handler.setFormatter(log_formatter)
|
|
|
+logger = logging.getLogger('bayes_combine_logger')
|
|
|
+logger.addHandler(log_handler)
|
|
|
+logger.setLevel(logging.DEBUG)
|
|
|
+print('id of bayes_combine_logger %s' % id(logger))
|
|
|
|
|
|
|
|
|
class BayesFeatures(object):
|
|
@@ -190,62 +202,66 @@ class BayesCombine(object):
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
- # 1、读取配置文件
|
|
|
- with open('config/config.yaml', mode='r', encoding='utf-8') as f:
|
|
|
- config = yaml.load(f.read(), Loader=yaml.FullLoader)
|
|
|
-
|
|
|
- # 1-1 数据库连接引擎,依据开发环境/生产环境 进行切换
|
|
|
- if os.getenv('LYY_DEV', 'unknown') == 'dev':
|
|
|
- engine = get_db_engine(config['devDB'])
|
|
|
- else:
|
|
|
- engine = get_db_engine(config['productDB'])
|
|
|
-
|
|
|
- # 2、 参与定向组合的维度
|
|
|
- target_dim = [key for key in config['bayesDim'].keys() if config['bayesDim'][key]['isOn']]
|
|
|
-
|
|
|
- # 4、计算贝叶斯组合入库
|
|
|
- for project_id in config['projectId']:
|
|
|
- # 4.1 获取指定项目下当前活跃的素材信息, 如近3天内累计激活个数达到50个(开发环境或者生产环境,这部分都读取生产数据库)
|
|
|
- product_engine = get_db_engine(config['productDB'])
|
|
|
- sql = '''
|
|
|
- select signature from ctop_kuaishou_report_daily_material
|
|
|
- where account_id in (select account_id from ctop_user_allocation where project_id = %s)
|
|
|
- and datediff(now(),stat_date) <= %s
|
|
|
- group by signature
|
|
|
- having sum(activation) >= %s
|
|
|
- ''' % (project_id,
|
|
|
- config['activeMaterialFilterRule']['days'],
|
|
|
- config['activeMaterialFilterRule']['activation'])
|
|
|
- df = pd.read_sql(sql, product_engine)
|
|
|
- active_signatures = df[~df.signature.isnull()].signature.values
|
|
|
-
|
|
|
- # 4.2 在素材人群报表筛选里面近一个月累计100个激活
|
|
|
- # TODO 等生产的素材人群表存在之后,修改表名,添加 days
|
|
|
- sql = """
|
|
|
- select signature from ctop_kuaishou_audience_daily_report_by_signature_age
|
|
|
- where signature in %s
|
|
|
- group by signature
|
|
|
- having sum(activation) >= %s
|
|
|
- """ % (tuple(active_signatures),
|
|
|
- config['getBaysCombineMaterialFilterRule']['activation'])
|
|
|
- df = pd.read_sql(sql, engine)
|
|
|
- signature_lst = df['signature'].values
|
|
|
-
|
|
|
- # TODO 这行代码用于测试
|
|
|
- signature_lst = ['0070efb7557b2a04cf3d4a6f243c3cd8', '03b93728f0d82ea7865c3c7cf632bc1b']
|
|
|
-
|
|
|
- # 4.2 计算指定素材的贝叶斯特征
|
|
|
- for sig in signature_lst:
|
|
|
- for t_type in config['targetType']:
|
|
|
- bayes_feature_lst = []
|
|
|
- for dimension in target_dim:
|
|
|
- cls = BayesFeatures(sig, t_type, dimension, config['bayesDim'][dimension])
|
|
|
- # 计算滑窗组合
|
|
|
- cls.get_window_combine()
|
|
|
- cls.get_bayes_feature()
|
|
|
- bayes_feature_lst.append(cls.bayes_feature)
|
|
|
-
|
|
|
- # 依据特征值,计算多维度的组合预估值
|
|
|
- cls = BayesCombine(sig, project_id, t_type, bayes_feature_lst)
|
|
|
- cls.get_bayes_estimate()
|
|
|
- cls.write_to_db()
|
|
|
+ try:
|
|
|
+ # 1、读取配置文件
|
|
|
+ with open('config/config.yaml', mode='r', encoding='utf-8') as f:
|
|
|
+ config = yaml.load(f.read(), Loader=yaml.FullLoader)
|
|
|
+
|
|
|
+ # 1-1 数据库连接引擎,依据开发环境/生产环境 进行切换
|
|
|
+ if os.getenv('LYY_DEV', 'unknown') == 'dev':
|
|
|
+ engine = get_db_engine(config['devDB'])
|
|
|
+ else:
|
|
|
+ engine = get_db_engine(config['productDB'])
|
|
|
+
|
|
|
+ # 2、 参与定向组合的维度
|
|
|
+ target_dim = [key for key in config['bayesDim'].keys() if config['bayesDim'][key]['isOn']]
|
|
|
+
|
|
|
+ # 4、计算贝叶斯组合入库
|
|
|
+ for project_id in config['projectId']:
|
|
|
+ # 4.1 获取指定项目下当前活跃的素材信息, 如近3天内累计激活个数达到50个(开发环境或者生产环境,这部分都读取生产数据库)
|
|
|
+ product_engine = get_db_engine(config['productDB'])
|
|
|
+ sql = '''
|
|
|
+ select signature from ctop_kuaishou_report_daily_material
|
|
|
+ where account_id in (select account_id from ctop_user_allocation where project_id = %s)
|
|
|
+ and datediff(now(),stat_date) <= %s
|
|
|
+ group by signature
|
|
|
+ having sum(activation) >= %s
|
|
|
+ ''' % (project_id,
|
|
|
+ config['activeMaterialFilterRule']['days'],
|
|
|
+ config['activeMaterialFilterRule']['activation'])
|
|
|
+ df = pd.read_sql(sql, product_engine)
|
|
|
+ active_signatures = df[~df.signature.isnull()].signature.values
|
|
|
+
|
|
|
+ # 4.2 在素材人群报表筛选里面近一个月累计100个激活
|
|
|
+ # TODO 等生产的素材人群表存在之后,修改表名,添加 days
|
|
|
+ sql = """
|
|
|
+ select signature from ctop_kuaishou_audience_daily_report_by_signature_age
|
|
|
+ where signature in %s
|
|
|
+ group by signature
|
|
|
+ having sum(activation) >= %s
|
|
|
+ """ % (tuple(active_signatures),
|
|
|
+ config['getBaysCombineMaterialFilterRule']['activation'])
|
|
|
+ df = pd.read_sql(sql, engine)
|
|
|
+ signature_lst = df['signature'].values
|
|
|
+
|
|
|
+ # TODO 这行代码用于测试
|
|
|
+ signature_lst = ['0070efb7557b2a04cf3d4a6f243c3cd8', '03b93728f0d82ea7865c3c7cf632bc1b']
|
|
|
+
|
|
|
+ # 4.2 计算指定素材的贝叶斯特征
|
|
|
+ for sig in signature_lst:
|
|
|
+ for t_type in config['targetType']:
|
|
|
+ bayes_feature_lst = []
|
|
|
+ for dimension in target_dim:
|
|
|
+ cls = BayesFeatures(sig, t_type, dimension, config['bayesDim'][dimension])
|
|
|
+ # 计算滑窗组合
|
|
|
+ cls.get_window_combine()
|
|
|
+ cls.get_bayes_feature()
|
|
|
+ bayes_feature_lst.append(cls.bayes_feature)
|
|
|
+
|
|
|
+ # 依据特征值,计算多维度的组合预估值
|
|
|
+ cls = BayesCombine(sig, project_id, t_type, bayes_feature_lst)
|
|
|
+ cls.get_bayes_estimate()
|
|
|
+ cls.write_to_db()
|
|
|
+ logger.info('project_id is %s 完成贝叶斯组合计算!' % project_id)
|
|
|
+ except:
|
|
|
+ logger.error("traceback is %s" % (traceback.format_exc()))
|