|
@@ -66,11 +66,10 @@ class BayesFeatures(object):
|
|
|
sum(activation) activation
|
|
|
from %s
|
|
|
where signature = '%s'
|
|
|
- and datediff(now(),stat_date)<=30
|
|
|
+ and datediff(now(),stat_date)<= %s
|
|
|
group by signature, %s
|
|
|
- ''' % (self.file_name, self.table, self.signature, self.file_name)
|
|
|
+ ''' % (self.file_name, self.table, self.signature, config['getBaysCombineMaterialFilterRule']['days'], self.file_name)
|
|
|
df = pd.read_sql(sql, product_engine)
|
|
|
-
|
|
|
if self.target_type == 'action_ratio':
|
|
|
# 计算行为率(bclick/aclick)的组合特征值
|
|
|
df['unbclick'] = df['aclick'] - df['bclick']
|
|
@@ -144,8 +143,8 @@ class BayesCombine(object):
|
|
|
get_sample_and_pct_sql = """
|
|
|
select sum(aclick) aclick, sum(bclick) bclick, sum(activation) activation from %s where
|
|
|
signature = '%s'
|
|
|
- and datediff(now(),stat_date)<=30
|
|
|
- """ % (config['bayesDim']['age']['table'], self.signature)
|
|
|
+ and datediff(now(),stat_date)<= %s
|
|
|
+ """ % (config['bayesDim']['age']['table'], self.signature, config['getBaysCombineMaterialFilterRule']['days'])
|
|
|
sample_pct_df = pd.read_sql(get_sample_and_pct_sql, product_engine)
|
|
|
if self.target_type == 'action_ratio':
|
|
|
self.sig_pos_pct = sample_pct_df['bclick'].sum() / sample_pct_df['aclick'].sum()
|
|
@@ -176,25 +175,29 @@ class BayesCombine(object):
|
|
|
out_dict['p_value'] = p_value
|
|
|
out_dict['sample_size'] = self.sample_size
|
|
|
|
|
|
- # TODO 调用人群预估覆盖接口,得到该组合的人群覆盖数 修改advertiser_id 的值 为 account_id
|
|
|
- # 获取该项目下在投的账号
|
|
|
- get_acc_sql = """select account_id from ctop_user_allocation where project_id = %s and account_status=0 limit 1""" % \
|
|
|
- self.project_id
|
|
|
- account_df = pd.read_sql(get_acc_sql, product_engine)
|
|
|
- account_id = account_df['account_id'].values[0]
|
|
|
- request_data = {'region': city_code_transform(out_dict.get('city')),
|
|
|
- 'ages_range': age_code_transform(out_dict.get('age')),
|
|
|
- 'gender': gender_code_transform(out_dict.get('gender')),
|
|
|
- 'advertiser_id': 9774238}
|
|
|
-
|
|
|
- request = requests.post(url=estimate_people_number_url,
|
|
|
- headers=headers,
|
|
|
- data=json.dumps(request_data, cls=NpEncoder))
|
|
|
- response_data = json.loads(request.text)
|
|
|
- if response_data['code'] == 0:
|
|
|
- out_dict['crowd_coverage_cnt'] = response_data['data'].get('audience_prediction_num')
|
|
|
- else:
|
|
|
- logger.error("人群预估覆盖接口调用报错,请求数据为%s,返回数据为%s" % (str(request_data), str(response_data)))
|
|
|
+ try:
|
|
|
+ # TODO 调用人群预估覆盖接口,得到该组合的人群覆盖数 修改advertiser_id 的值 为 account_id
|
|
|
+ # 获取该项目下在投的账号
|
|
|
+ get_acc_sql = """select account_id from ctop_user_allocation where project_id = %s and account_status=0 limit 1""" % \
|
|
|
+ self.project_id
|
|
|
+ account_df = pd.read_sql(get_acc_sql, product_engine)
|
|
|
+ account_id = account_df['account_id'].values[0]
|
|
|
+ request_data = {'region': city_code_transform(out_dict.get('city')),
|
|
|
+ 'ages_range': age_code_transform(out_dict.get('age')),
|
|
|
+ 'gender': gender_code_transform(out_dict.get('gender')),
|
|
|
+ 'advertiser_id': 9774238}
|
|
|
+
|
|
|
+ request = requests.post(url=estimate_people_number_url,
|
|
|
+ headers=headers,
|
|
|
+ data=json.dumps(request_data, cls=NpEncoder))
|
|
|
+ response_data = json.loads(request.text)
|
|
|
+ if response_data['code'] == 0:
|
|
|
+ out_dict['crowd_coverage_cnt'] = response_data['data'].get('audience_prediction_num')
|
|
|
+ else:
|
|
|
+ logger.error("人群预估覆盖接口调用报错,请求数据为%s,返回数据为%s" % (str(request_data), str(response_data)))
|
|
|
+ except:
|
|
|
+ logger.error("人群预估覆盖接口调用报错,请求数据为%s,请求地址为%s, 返回数据为%s, 异常信息为%s" %
|
|
|
+ (str(request_data), estimate_people_number_url, str(response_data), traceback.format_exc()))
|
|
|
|
|
|
out_dict['combine_estimate_prob'] = prob
|
|
|
out_dict['actual_prob'] = self.actual_prob
|
|
@@ -214,55 +217,55 @@ class BayesCombine(object):
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
- try:
|
|
|
- # 1、读取配置文件
|
|
|
- with open('config/config.yaml', mode='r', encoding='utf-8') as f:
|
|
|
- config = yaml.load(f.read(), Loader=yaml.FullLoader)
|
|
|
-
|
|
|
- # 1-1 数据库连接引擎,依据开发环境/生产环境 进行切换
|
|
|
- if os.getenv('LYY_DEV', 'unknown') == 'dev':
|
|
|
- engine = get_db_engine(config['devDB'])
|
|
|
- else:
|
|
|
- engine = get_db_engine(config['productDB'])
|
|
|
-
|
|
|
- product_engine = get_db_engine(config['productDB'])
|
|
|
-
|
|
|
- # 2、 参与定向组合的维度
|
|
|
- target_dim = [key for key in config['bayesDim'].keys() if config['bayesDim'][key]['isOn']]
|
|
|
-
|
|
|
- # 4、计算贝叶斯组合入库
|
|
|
- for project_id in config['projectId']:
|
|
|
- # 4.1 获取指定项目下当前活跃的素材信息, 如近3天内累计激活个数达到50个(开发环境或者生产环境,这部分都读取生产数据库)
|
|
|
- sql = '''
|
|
|
- select signature from ctop_kuaishou_report_daily_material
|
|
|
- where account_id in (select account_id from ctop_user_allocation where project_id = %s)
|
|
|
- and datediff(now(),stat_date) <= %s
|
|
|
- group by signature
|
|
|
- having sum(activation) >= %s
|
|
|
- ''' % (project_id,
|
|
|
- config['activeMaterialFilterRule']['days'],
|
|
|
- config['activeMaterialFilterRule']['activation'])
|
|
|
- df = pd.read_sql(sql, product_engine)
|
|
|
- active_signatures = df[~df.signature.isnull()].signature.values
|
|
|
-
|
|
|
- # 4.2 在素材人群报表筛选里面近一个月累计100个激活
|
|
|
- sql = """
|
|
|
- select signature from ctop_kuaishou_audience_report_daily_age_material
|
|
|
- where signature in %s
|
|
|
- and datediff(now(),stat_date) <= %s
|
|
|
- group by signature
|
|
|
- having sum(activation) >= %s
|
|
|
- """ % (tuple(active_signatures),
|
|
|
- config['getBaysCombineMaterialFilterRule']['days'],
|
|
|
- config['getBaysCombineMaterialFilterRule']['activation'])
|
|
|
- df = pd.read_sql(sql, product_engine)
|
|
|
- signature_lst = df['signature'].values
|
|
|
-
|
|
|
- # 这行代码用于测试
|
|
|
- # signature_lst = ['0070efb7557b2a04cf3d4a6f243c3cd8', '03b93728f0d82ea7865c3c7cf632bc1b']
|
|
|
-
|
|
|
- # 4.2 计算指定素材的贝叶斯特征
|
|
|
- for sig in signature_lst:
|
|
|
+ # 1、读取配置文件
|
|
|
+ with open('config/config.yaml', mode='r', encoding='utf-8') as f:
|
|
|
+ config = yaml.load(f.read(), Loader=yaml.FullLoader)
|
|
|
+
|
|
|
+ # 1-1 数据库连接引擎,依据开发环境/生产环境 进行切换
|
|
|
+ if os.getenv('LYY_DEV', 'unknown') == 'dev':
|
|
|
+ engine = get_db_engine(config['devDB'])
|
|
|
+ else:
|
|
|
+ engine = get_db_engine(config['productDB'])
|
|
|
+
|
|
|
+ product_engine = get_db_engine(config['productDB'])
|
|
|
+
|
|
|
+ # 2、 参与定向组合的维度
|
|
|
+ target_dim = [key for key in config['bayesDim'].keys() if config['bayesDim'][key]['isOn']]
|
|
|
+
|
|
|
+ # 4、计算贝叶斯组合入库
|
|
|
+ for project_id in config['projectId']:
|
|
|
+ # 4.1 获取指定项目下当前活跃的素材信息, 如近3天内累计激活个数达到50个(开发环境或者生产环境,这部分都读取生产数据库)
|
|
|
+ sql = '''
|
|
|
+ select signature from ctop_kuaishou_report_daily_material
|
|
|
+ where account_id in (select account_id from ctop_user_allocation where project_id = %s)
|
|
|
+ and datediff(now(),stat_date) <= %s
|
|
|
+ group by signature
|
|
|
+ having sum(activation) >= %s
|
|
|
+ ''' % (project_id,
|
|
|
+ config['activeMaterialFilterRule']['days'],
|
|
|
+ config['activeMaterialFilterRule']['activation'])
|
|
|
+ df = pd.read_sql(sql, product_engine)
|
|
|
+ active_signatures = df[~df.signature.isnull()].signature.values
|
|
|
+
|
|
|
+ # 4.2 在素材人群报表筛选里面近一个月累计100个激活
|
|
|
+ sql = """
|
|
|
+ select signature from ctop_kuaishou_audience_report_daily_age_material
|
|
|
+ where signature in %s
|
|
|
+ and datediff(now(),stat_date) <= %s
|
|
|
+ group by signature
|
|
|
+ having sum(activation) >= %s
|
|
|
+ """ % (tuple(active_signatures),
|
|
|
+ config['getBaysCombineMaterialFilterRule']['days'],
|
|
|
+ config['getBaysCombineMaterialFilterRule']['activation'])
|
|
|
+ df = pd.read_sql(sql, product_engine)
|
|
|
+ signature_lst = df['signature'].values
|
|
|
+
|
|
|
+ # 这行代码用于测试
|
|
|
+ # signature_lst = ['4a56bc00ce51420f3a460ab51c6f5110']
|
|
|
+
|
|
|
+ # 4.2 计算指定素材的贝叶斯特征
|
|
|
+ for sig in signature_lst:
|
|
|
+ try:
|
|
|
for t_type in config['targetType']:
|
|
|
bayes_feature_lst = []
|
|
|
for dimension in target_dim:
|
|
@@ -276,6 +279,8 @@ if __name__ == '__main__':
|
|
|
cls = BayesCombine(sig, project_id, t_type, bayes_feature_lst)
|
|
|
cls.get_bayes_estimate()
|
|
|
cls.write_to_db()
|
|
|
- logger.info('project_id is %s 完成贝叶斯组合计算!' % project_id)
|
|
|
- except:
|
|
|
- logger.error("traceback is %s" % (traceback.format_exc()))
|
|
|
+ logger.info('project_id=%s, signature= %s 完成贝叶斯组合计算!' % (project_id, sig))
|
|
|
+ except:
|
|
|
+ logger.error('project_id=%s, signature=%s 贝叶斯组合计算出错,异常信息为%s!' % (project_id, sig, traceback.format_exc()))
|
|
|
+
|
|
|
+ logger.info('project_id is %s 完成贝叶斯组合计算!' % project_id)
|