|
@@ -1,11 +1,12 @@
|
|
|
import yaml
|
|
|
+import os
|
|
|
import pandas as pd
|
|
|
-from sqlalchemy import create_engine
|
|
|
-from urllib import parse
|
|
|
-from commonFunc import update_dict
|
|
|
+import numpy as np
|
|
|
+from commonFunc import update_dict, get_db_engine
|
|
|
from functools import reduce
|
|
|
from itertools import product
|
|
|
from datetime import datetime
|
|
|
+from statsmodels.stats.proportion import proportions_ztest
|
|
|
|
|
|
|
|
|
class BayesFeatures(object):
|
|
@@ -52,7 +53,7 @@ class BayesFeatures(object):
|
|
|
-- and datediff(now(),stat_date)<=30
|
|
|
group by signature, %s
|
|
|
''' % (self.file_name, self.table, self.signature, self.file_name)
|
|
|
- df = pd.read_sql(sql, test_engine)
|
|
|
+ df = pd.read_sql(sql, engine)
|
|
|
|
|
|
if self.target_type == 'action_ratio':
|
|
|
# 计算行为率(bclick/aclick)的组合特征值
|
|
@@ -61,6 +62,7 @@ class BayesFeatures(object):
|
|
|
sig_neg_pct = 1 - sig_pos_pct # 素材曝光-未点击的概率
|
|
|
self.bayes_feature['sig_pos_pct'] = sig_pos_pct
|
|
|
self.bayes_feature['sig_neg_pct'] = sig_neg_pct
|
|
|
+ self.bayes_feature['sample_size'] = df['aclick'].sum() # 素材曝光量,后续计算显著差异用
|
|
|
for sub_combine in self.window_combine:
|
|
|
if sub_combine != ['不限']:
|
|
|
key = self.dim + '_' + '|'.join(sub_combine)
|
|
@@ -89,6 +91,7 @@ class BayesFeatures(object):
|
|
|
sig_neg_pct = 1 - sig_pos_pct # 素材点击-未转化的概率
|
|
|
self.bayes_feature['sig_pos_pct'] = sig_pos_pct
|
|
|
self.bayes_feature['sig_neg_pct'] = sig_neg_pct
|
|
|
+ self.bayes_feature['sample_size'] = df['bclick'].sum() # 素材行为量,后续计算显著差异用
|
|
|
for sub_combine in self.window_combine:
|
|
|
if sub_combine != '不限':
|
|
|
key = self.dim + '_' + '|'.join(sub_combine)
|
|
@@ -115,18 +118,24 @@ class BayesCombine(object):
|
|
|
"""
|
|
|
依据特征值,计算多维度的组合预估值
|
|
|
"""
|
|
|
+
|
|
|
def __init__(self, signature, project_id, target_type, dim_features):
|
|
|
self.signature = signature
|
|
|
self.project_id = project_id
|
|
|
self.target_type = target_type
|
|
|
self.dim_features = dim_features
|
|
|
self.bayes_combine_df = pd.DataFrame()
|
|
|
- self.actual_prob = self.dim_features[0]['sig_pos_pct']
|
|
|
+ self.actual_prob = None
|
|
|
+ self.sample_size = None
|
|
|
|
|
|
def get_bayes_estimate(self):
|
|
|
- dim_combine_lst = [[key for key in fea.keys() if key not in ['sig_pos_pct', 'sig_neg_pct']] for fea in self.dim_features]
|
|
|
+ dim_combine_lst = [[key for key in fea.keys() if key not in ['sig_pos_pct', 'sig_neg_pct', 'sample_size']]
|
|
|
+ for fea in self.dim_features]
|
|
|
self.dim_features = reduce(update_dict, self.dim_features)
|
|
|
-
|
|
|
+
|
|
|
+ self.actual_prob = self.dim_features['sig_pos_pct']
|
|
|
+ self.sample_size = self.dim_features['sample_size']
|
|
|
+
|
|
|
for ele in product(*dim_combine_lst):
|
|
|
pos_pct_lst = [self.dim_features[key]['pos_pct'] for key in ele]
|
|
|
prob_pos = reduce(lambda x, y: x * y, pos_pct_lst)
|
|
@@ -137,6 +146,14 @@ class BayesCombine(object):
|
|
|
prob = (prob_pos * self.dim_features['sig_pos_pct']) / (prob_neg * self.dim_features['sig_neg_pct'])
|
|
|
out_dict = dict(zip([e.split('_')[0] for e in ele], [e.split('_')[-1] for e in ele]))
|
|
|
|
|
|
+ # 计算该组合的概率值 与 实际投放的概率值 是否存在显著性差异:显著好、 显著差
|
|
|
+ count = np.array([self.sample_size * prob, self.sample_size * self.actual_prob])
|
|
|
+ nobs = np.array([self.sample_size, self.sample_size])
|
|
|
+ z_score, p_value = proportions_ztest(count=count, nobs=nobs, value=None, alternative='two-sided', prop_var=False)
|
|
|
+ out_dict['z_score'] = z_score
|
|
|
+ out_dict['p_value'] = p_value
|
|
|
+ out_dict['sample_size'] = self.sample_size
|
|
|
+
|
|
|
# TODO 调用人群预估覆盖接口,得到该组合的人群覆盖数
|
|
|
out_dict['crowd_coverage_cnt'] = 1000
|
|
|
|
|
@@ -152,7 +169,7 @@ class BayesCombine(object):
|
|
|
|
|
|
def write_to_db(self):
|
|
|
self.bayes_combine_df.to_sql(name="ctop_ai_kuaishou_signature_recommended_target_combine",
|
|
|
- con=test_engine,
|
|
|
+ con=engine,
|
|
|
if_exists='append',
|
|
|
index=False)
|
|
|
|
|
@@ -161,47 +178,52 @@ if __name__ == '__main__':
|
|
|
# 1、读取配置文件
|
|
|
with open('config/config.yaml', mode='r', encoding='utf-8') as f:
|
|
|
config = yaml.load(f.read(), Loader=yaml.FullLoader)
|
|
|
- project_ids = config['projectId']
|
|
|
- target_types = config['targetType']
|
|
|
- online_db = config['onlineDB']
|
|
|
- test_db = config['testDB']
|
|
|
- material_rule = config['materialRule']
|
|
|
- bayes_dim = config['bayesDim']
|
|
|
-
|
|
|
- # 2、数据库连接引擎
|
|
|
- db_con_str = 'mysql+pymysql://%s:%s@%s:%d/%s' % \
|
|
|
- (online_db['username'], parse.quote_plus(online_db['password']), online_db['host'], online_db['port'],
|
|
|
- online_db['database'])
|
|
|
- engine = create_engine(db_con_str, connect_args={'charset': 'utf8'})
|
|
|
-
|
|
|
- db_con_str = 'mysql+pymysql://%s:%s@%s:%d/%s' % \
|
|
|
- (test_db['username'], parse.quote_plus(test_db['password']), test_db['host'], test_db['port'],
|
|
|
- test_db['database'])
|
|
|
- test_engine = create_engine(db_con_str, connect_args={'charset': 'utf8'})
|
|
|
-
|
|
|
- # 3、 参与定向组合的维度
|
|
|
- target_dim = [key for key in bayes_dim.keys() if bayes_dim[key]['isOn']]
|
|
|
+
|
|
|
+ # 数据库连接引擎,依据开发环境/生产环境 进行切换
|
|
|
+ if os.getenv('environment', 'unknown') == 'dev':
|
|
|
+ engine = get_db_engine(config['devDB'])
|
|
|
+ else:
|
|
|
+ engine = get_db_engine(config['productDB'])
|
|
|
+
|
|
|
+ # 2、 参与定向组合的维度
|
|
|
+ target_dim = [key for key in config['bayesDim'].keys() if config['bayesDim'][key]['isOn']]
|
|
|
|
|
|
# 4、计算贝叶斯组合入库
|
|
|
- for project_id in project_ids:
|
|
|
- # 4.1 获取指定项目下当前活跃的素材信息, 如近3天内累计激活个数达到50个
|
|
|
+ for project_id in config['projectId']:
|
|
|
+ # 4.1 获取指定项目下当前活跃的素材信息, 如近3天内累计激活个数达到50个(开发环境或者生产环境,这部分都读取生产数据库)
|
|
|
+ product_engine = get_db_engine(config['productDB'])
|
|
|
sql = '''
|
|
|
select signature from ctop_kuaishou_report_daily_material
|
|
|
where account_id in (select account_id from ctop_user_allocation where project_id = %s)
|
|
|
and datediff(now(),stat_date) <= %s
|
|
|
group by signature
|
|
|
having sum(activation) >= %s
|
|
|
- ''' % (project_id, material_rule['days'], material_rule['activation'])
|
|
|
+ ''' % (project_id,
|
|
|
+ config['activeMaterialFilterRule']['days'],
|
|
|
+ config['activeMaterialFilterRule']['activation'])
|
|
|
+ df = pd.read_sql(sql, product_engine)
|
|
|
+ active_signatures = df[~df.signature.isnull()].signature.values
|
|
|
+
|
|
|
+ # 4.2 在素材人群报表筛选里面近一个月累计100个激活 TODO 等生产的素材人群表存在之后,修改表名,添加 days
|
|
|
+ sql = """
|
|
|
+ select signature from ctop_kuaishou_audience_daily_report_by_signature_age
|
|
|
+ where signature in %s
|
|
|
+ group by signature
|
|
|
+ having sum(activation) >= %s
|
|
|
+ """ % (tuple(active_signatures),
|
|
|
+ config['getBaysCombineMaterialFilterRule']['activation'])
|
|
|
df = pd.read_sql(sql, engine)
|
|
|
signature_lst = df['signature'].values
|
|
|
+
|
|
|
+ # TODO 这行代码用于测试
|
|
|
signature_lst = ['0070efb7557b2a04cf3d4a6f243c3cd8', '03b93728f0d82ea7865c3c7cf632bc1b']
|
|
|
|
|
|
# 4.2 计算指定素材的贝叶斯特征
|
|
|
for sig in signature_lst:
|
|
|
- for t_type in target_types:
|
|
|
+ for t_type in config['targetType']:
|
|
|
bayes_feature_lst = []
|
|
|
for dimension in target_dim:
|
|
|
- cls = BayesFeatures(sig, t_type, dimension, bayes_dim[dimension])
|
|
|
+ cls = BayesFeatures(sig, t_type, dimension, config['bayesDim'][dimension])
|
|
|
# 计算滑窗组合
|
|
|
cls.get_window_combine()
|
|
|
cls.get_bayes_feature()
|
|
@@ -211,35 +233,3 @@ if __name__ == '__main__':
|
|
|
cls = BayesCombine(sig, project_id, t_type, bayes_feature_lst)
|
|
|
cls.get_bayes_estimate()
|
|
|
cls.write_to_db()
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|