|
@@ -4,6 +4,7 @@ import tornado
|
|
|
import json
|
|
|
import traceback
|
|
|
import pandas as pd
|
|
|
+import numpy as np
|
|
|
import pymysql
|
|
|
import os
|
|
|
import yaml
|
|
@@ -23,7 +24,6 @@ logger.addHandler(log_handler)
|
|
|
logger.setLevel(logging.DEBUG)
|
|
|
print('id of ai_target_combine_logger %s' % id(logger))
|
|
|
|
|
|
-
|
|
|
with open('config/config.yaml', mode='r', encoding='utf-8') as f:
|
|
|
config = yaml.load(f.read(), Loader=yaml.FullLoader)
|
|
|
|
|
@@ -90,18 +90,20 @@ class GetTargetAndAssemblyParameters(object):
|
|
|
self.engine = None
|
|
|
self.product_engine = None
|
|
|
self.db_config = None
|
|
|
- self.signature_target_combine = None
|
|
|
- self.final_target_combine = []
|
|
|
self.advertiser_strategy_id = None
|
|
|
self.advertiser_strategy = {}
|
|
|
self.ai_strategy_uuid = str(uuid.uuid4())
|
|
|
self.request_data = {}
|
|
|
self.campaign_id = None
|
|
|
self.operation_type = 1 # 新增广告组
|
|
|
+ self.target_combine_from_table = []
|
|
|
+ self.target_combine_filter_by_subset = []
|
|
|
+ self.target_combine_to_create = []
|
|
|
self.get_database_engine() # 获取数据库信息
|
|
|
self.get_advertiser_strategy_info() # 获取账户配置信息
|
|
|
- self.get_signature_and_target() # 获取定向组合
|
|
|
- self.target_combine_is_subset() # 过滤出为账户配置子集的定向组合
|
|
|
+ self.get_signature_and_target() # 从素材定向组合表中获取最新的定向组合
|
|
|
+ self.filter_target_combine_by_whether_is_subset() # 筛选出为账户配置子集的定向组合
|
|
|
+ self.filter_target_combine_by_creative_cnt() # 依据素材在指定账户下关联的创意个数,进行筛选创建个数,防止超限导致的创建失败
|
|
|
|
|
|
def get_database_engine(self):
|
|
|
# 数据库连接引擎,依据开发环境/生产环境 进行切换
|
|
@@ -197,29 +199,33 @@ class GetTargetAndAssemblyParameters(object):
|
|
|
"""
|
|
|
从 ctop_ai_kuaishou_signature_recommended_target_combine 表中读取素材和对应的定向
|
|
|
读取配置文件的条件,筛选出符合条件的定向组合
|
|
|
+ TODO 放开sql语句中注释的代码
|
|
|
"""
|
|
|
sql = """
|
|
|
select * from ctop_ai_kuaishou_signature_recommended_target_combine where project_id = %s
|
|
|
- -- and stat_date = curdate()
|
|
|
+ -- and stat_date = (select max(stat_date) from ctop_ai_kuaishou_signature_recommended_target_combine)
|
|
|
""" % self.project_id
|
|
|
df = pd.read_sql(sql, self.engine)
|
|
|
- # 计算组合的概率,相对于实际投放概率高出了百分之多少
|
|
|
+
|
|
|
+ # 计算组合的概率,相对于实际投放概率高出了百分之多少,样本量是否达标(读取配置文件),且p值小于等于0.05
|
|
|
df['improve_ratio'] = (df['combine_estimate_prob'] - df['actual_prob']) / df['actual_prob']
|
|
|
|
|
|
# 两种类型的过滤标准不一样,分开进行判断,然后对结果进行合并
|
|
|
df_1 = df[(df['target_type'] == 'action_ratio') &
|
|
|
(df['improve_ratio'] >= config['filterTargetCombine']['actionRatio']['improveRatio']) &
|
|
|
- (df['sample_size'] >= config['filterTargetCombine']['actionRatio']['sampleSize'])]
|
|
|
+ (df['sample_size'] >= config['filterTargetCombine']['actionRatio']['sampleSize']) &
|
|
|
+ df['p_value'] <= 0.05]
|
|
|
|
|
|
df_2 = df[(df['target_type'] == 'convertRatio') &
|
|
|
(df['improve_ratio'] >= config['filterTargetCombine']['convertRatio']['improveRatio']) &
|
|
|
- (df['sample_size'] >= config['filterTargetCombine']['convertRatio']['sampleSize'])]
|
|
|
+ (df['sample_size'] >= config['filterTargetCombine']['convertRatio']['sampleSize']) &
|
|
|
+ df['p_value'] <= 0.05]
|
|
|
|
|
|
merge_df = pd.concat([df_1, df_2], axis=0)
|
|
|
|
|
|
# 'age', 'gender', 'city', 'business', 'province', 'client'
|
|
|
features = [key for key, value in config['bayesDim'].items() if value['isOn']]
|
|
|
- self.signature_target_combine = merge_df[['id', 'signature'] + features].to_dict(orient='records')
|
|
|
+ self.target_combine_from_table = merge_df[['id', 'signature'] + features].to_dict(orient='records')
|
|
|
|
|
|
def write_intelligence_strategy_table(self):
|
|
|
"""
|
|
@@ -240,7 +246,7 @@ class GetTargetAndAssemblyParameters(object):
|
|
|
if_exists='append',
|
|
|
index=False)
|
|
|
|
|
|
- def target_combine_is_subset(self):
|
|
|
+ def filter_target_combine_by_whether_is_subset(self):
|
|
|
"""
|
|
|
ctop_ai_kuaishou_signature_recommended_target_combine 表中的 city 和 province 都对应 self.advertiser_strategy 的 region [] varchar
|
|
|
age 对应 ages_range [] varchar
|
|
@@ -252,7 +258,7 @@ class GetTargetAndAssemblyParameters(object):
|
|
|
age_bool = True
|
|
|
region_bool = True
|
|
|
|
|
|
- for item in self.signature_target_combine:
|
|
|
+ for item in self.target_combine_from_table:
|
|
|
if 'gender' in item.keys():
|
|
|
gender_bool, gender_dict = is_contains_gender(item['gender'], self.advertiser_strategy['gender'])
|
|
|
if 'age' in item.keys():
|
|
@@ -268,7 +274,7 @@ class GetTargetAndAssemblyParameters(object):
|
|
|
target_combine.update(gender_dict) if 'gender' in item.keys() else None
|
|
|
target_combine.update(age_dict) if 'age' in item.keys() else None
|
|
|
target_combine.update(region_dict) if 'city' in item.keys() else None
|
|
|
- self.final_target_combine.append(target_combine)
|
|
|
+ self.target_combine_filter_by_subset.append(target_combine)
|
|
|
else:
|
|
|
logger.info("推荐定向:%s 与 账户配置信息里的定向(gender:%s, age_min: %s,age_max: %s, ages_ranges:%s,region:%s )存在冲突" %
|
|
|
(item,
|
|
@@ -278,10 +284,42 @@ class GetTargetAndAssemblyParameters(object):
|
|
|
self.advertiser_strategy['ages_range'],
|
|
|
self.advertiser_strategy['region']))
|
|
|
|
|
|
+ def filter_target_combine_by_creative_cnt(self):
|
|
|
+ """
|
|
|
+ 依据素材已经关联的创意个数,来定素材可以创建的定向组合个数, 并从中随机随着N个
|
|
|
+ """
|
|
|
+ # 1-1 获取素材在该账户下关联的创意个数,过滤掉素材关联创意个数超过200的情况
|
|
|
+ df = pd.DataFrame(self.target_combine_filter_by_subset)
|
|
|
+
|
|
|
+ sql = """
|
|
|
+ select t1.signature, t2.creative_count
|
|
|
+ from
|
|
|
+ (select account_id, signature, photo_id from ctop_kuaishou_video_get
|
|
|
+ where account_id = %s and signature in %s
|
|
|
+ group by account_id, signature) t1
|
|
|
+ left join
|
|
|
+ ctop_kuaishou_video_relate_creatives t2
|
|
|
+ on t1.account_id = t2.account_id and t1.photo_id = t2.photo_id
|
|
|
+ """ % (self.account_id, tuple(df.signature.unique()))
|
|
|
+ creative_cnt_df = pd.read_sql(sql, self.engine)
|
|
|
+ creative_cnt_df = creative_cnt_df[creative_cnt_df.creative_count < 200]
|
|
|
+
|
|
|
+ # 1-2 计算每个素材还能创建的广告组(定向)个数: (200 - 已关联创意个数) / 15
|
|
|
+ creative_cnt_df['target_combine_cnt'] = np.floor((200 - creative_cnt_df['creative_count']) / 15)
|
|
|
+
|
|
|
+ final_target_combine = pd.DataFrame([])
|
|
|
+ for sig in creative_cnt_df.signature.unique():
|
|
|
+ n = creative_cnt_df[creative_cnt_df.signature == sig].target_combine_cnt.values[0]
|
|
|
+ now = datetime.datetime.now()
|
|
|
+ sig_target_df = df[df.signature == sig].sample(n, axis=0, random_state=(now.year + now.month + now.day))
|
|
|
+ final_target_combine = final_target_combine.append(sig_target_df)
|
|
|
+
|
|
|
+ self.target_combine_to_create = final_target_combine.to_dict(orient='records')
|
|
|
+
|
|
|
# 从集合中随机选取N个组合
|
|
|
combine_cnt = config['filterTargetCombine']['combineCnt']
|
|
|
- random.shuffle(self.final_target_combine)
|
|
|
- self.final_target_combine = self.final_target_combine[:combine_cnt]
|
|
|
+ random.shuffle(self.target_combine_to_create)
|
|
|
+ self.target_combine_to_create = self.target_combine_to_create[:combine_cnt]
|
|
|
|
|
|
def assembly_group_and_creative_params(self):
|
|
|
"""
|
|
@@ -295,7 +333,7 @@ class GetTargetAndAssemblyParameters(object):
|
|
|
batch_group_params_to_db = []
|
|
|
batch_creative_params_to_db = []
|
|
|
|
|
|
- for item in self.final_target_combine:
|
|
|
+ for item in self.target_combine_to_create:
|
|
|
# 使用优质定向更新组层级的参数
|
|
|
target_combine = item.copy()
|
|
|
del target_combine['target_combine_id']
|
|
@@ -309,8 +347,6 @@ class GetTargetAndAssemblyParameters(object):
|
|
|
if col in target_combine.keys():
|
|
|
group_params_to_db[col] = str(group_params_to_db[col])
|
|
|
|
|
|
-
|
|
|
-
|
|
|
group_uuid = str(uuid.uuid4())
|
|
|
group_name = group_params['unit_name']
|
|
|
|
|
@@ -531,7 +567,7 @@ class GetTargetAndAssemblyParameters(object):
|
|
|
|
|
|
# 6、用于组装发送请求的部分字段,需要转化为list类型
|
|
|
cols_to_list = ['app_store', 'scene_id', 'region', 'day_budget_schedule', 'ages_range', 'device_brand', 'business_interest',
|
|
|
- 'fans_star', 'interest_video', 'app_interest', 'app_interest_ids', 'app_ids', 'population','district_ids',
|
|
|
+ 'fans_star', 'interest_video', 'app_interest', 'app_interest_ids', 'app_ids', 'population', 'district_ids',
|
|
|
'exclude_population', 'paid_audience']
|
|
|
for col in cols_to_list:
|
|
|
if col in group_params.keys():
|
|
@@ -549,7 +585,7 @@ class GetTargetAndAssemblyParameters(object):
|
|
|
'sticker_title': self.advertiser_strategy.get('sticker_title'),
|
|
|
'overlay_type': self.advertiser_strategy.get('overlay_type'),
|
|
|
'expose_tag': self.advertiser_strategy.get('expose_tag'),
|
|
|
- 'new_expose_tag':self.advertiser_strategy.get('new_expose_tag'),
|
|
|
+ 'new_expose_tag': self.advertiser_strategy.get('new_expose_tag'),
|
|
|
'site_id': self.advertiser_strategy.get('site_id'),
|
|
|
'click_track_url': self.advertiser_strategy.get('click_track_url'),
|
|
|
'impression_url': self.advertiser_strategy.get('impression_url'),
|
|
@@ -566,10 +602,9 @@ class GetTargetAndAssemblyParameters(object):
|
|
|
random.shuffle(description_lst)
|
|
|
creative_params['description'] = description_lst[0]
|
|
|
|
|
|
-
|
|
|
# 2 写入数据库的创意参数
|
|
|
creative_params_to_db = creative_params.copy()
|
|
|
- drop_cols = ['put_status','live_creative_type']
|
|
|
+ drop_cols = ['put_status', 'live_creative_type']
|
|
|
for col in drop_cols:
|
|
|
if col in creative_params_to_db:
|
|
|
del creative_params_to_db[col]
|
|
@@ -606,4 +641,4 @@ class GetTargetAndAssemblyParameters(object):
|
|
|
SET message = %s
|
|
|
WHERE ai_strategy_uuid = %s""", (str(message), self.ai_strategy_uuid))
|
|
|
db_con.commit()
|
|
|
- db_con.close()
|
|
|
+ db_con.close()
|