|  | @@ -4,6 +4,7 @@ import tornado
 | 
	
		
			
				|  |  |  import json
 | 
	
		
			
				|  |  |  import traceback
 | 
	
		
			
				|  |  |  import pandas as pd
 | 
	
		
			
				|  |  | +import numpy as np
 | 
	
		
			
				|  |  |  import pymysql
 | 
	
		
			
				|  |  |  import os
 | 
	
		
			
				|  |  |  import yaml
 | 
	
	
		
			
				|  | @@ -23,7 +24,6 @@ logger.addHandler(log_handler)
 | 
	
		
			
				|  |  |  logger.setLevel(logging.DEBUG)
 | 
	
		
			
				|  |  |  print('id of ai_target_combine_logger %s' % id(logger))
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -
 | 
	
		
			
				|  |  |  with open('config/config.yaml', mode='r', encoding='utf-8') as f:
 | 
	
		
			
				|  |  |      config = yaml.load(f.read(), Loader=yaml.FullLoader)
 | 
	
		
			
				|  |  |  
 | 
	
	
		
			
				|  | @@ -90,18 +90,20 @@ class GetTargetAndAssemblyParameters(object):
 | 
	
		
			
				|  |  |          self.engine = None
 | 
	
		
			
				|  |  |          self.product_engine = None
 | 
	
		
			
				|  |  |          self.db_config = None
 | 
	
		
			
				|  |  | -        self.signature_target_combine = None
 | 
	
		
			
				|  |  | -        self.final_target_combine = []
 | 
	
		
			
				|  |  |          self.advertiser_strategy_id = None
 | 
	
		
			
				|  |  |          self.advertiser_strategy = {}
 | 
	
		
			
				|  |  |          self.ai_strategy_uuid = str(uuid.uuid4())
 | 
	
		
			
				|  |  |          self.request_data = {}
 | 
	
		
			
				|  |  |          self.campaign_id = None
 | 
	
		
			
				|  |  |          self.operation_type = 1  # 新增广告组
 | 
	
		
			
				|  |  | +        self.target_combine_from_table = []
 | 
	
		
			
				|  |  | +        self.target_combine_filter_by_subset = []
 | 
	
		
			
				|  |  | +        self.target_combine_to_create = []
 | 
	
		
			
				|  |  |          self.get_database_engine()  # 获取数据库信息
 | 
	
		
			
				|  |  |          self.get_advertiser_strategy_info()  # 获取账户配置信息
 | 
	
		
			
				|  |  | -        self.get_signature_and_target()  # 获取定向组合
 | 
	
		
			
				|  |  | -        self.target_combine_is_subset()  # 过滤出为账户配置子集的定向组合
 | 
	
		
			
				|  |  | +        self.get_signature_and_target()  # 从素材定向组合表中获取最新的定向组合
 | 
	
		
			
				|  |  | +        self.filter_target_combine_by_whether_is_subset()  # 筛选出为账户配置子集的定向组合
 | 
	
		
			
				|  |  | +        self.filter_target_combine_by_creative_cnt()  # 依据素材在指定账户下关联的创意个数,进行筛选创建个数,防止超限导致的创建失败
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  |      def get_database_engine(self):
 | 
	
		
			
				|  |  |          # 数据库连接引擎,依据开发环境/生产环境 进行切换
 | 
	
	
		
			
				|  | @@ -197,29 +199,33 @@ class GetTargetAndAssemblyParameters(object):
 | 
	
		
			
				|  |  |          """
 | 
	
		
			
				|  |  |          从 ctop_ai_kuaishou_signature_recommended_target_combine 表中读取素材和对应的定向
 | 
	
		
			
				|  |  |          读取配置文件的条件,筛选出符合条件的定向组合
 | 
	
		
			
				|  |  | +        TODO 放开sql语句中注释的代码
 | 
	
		
			
				|  |  |          """
 | 
	
		
			
				|  |  |          sql = """
 | 
	
		
			
				|  |  |          select * from ctop_ai_kuaishou_signature_recommended_target_combine where project_id = %s 
 | 
	
		
			
				|  |  | -        -- and stat_date = curdate()
 | 
	
		
			
				|  |  | +        -- and stat_date = (select max(stat_date) from ctop_ai_kuaishou_signature_recommended_target_combine)
 | 
	
		
			
				|  |  |          """ % self.project_id
 | 
	
		
			
				|  |  |          df = pd.read_sql(sql, self.engine)
 | 
	
		
			
				|  |  | -        # 计算组合的概率,相对于实际投放概率高出了百分之多少
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +        # 计算组合的概率,相对于实际投放概率高出了百分之多少,样本量是否达标(读取配置文件),且p值小于等于0.05
 | 
	
		
			
				|  |  |          df['improve_ratio'] = (df['combine_estimate_prob'] - df['actual_prob']) / df['actual_prob']
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  |          # 两种类型的过滤标准不一样,分开进行判断,然后对结果进行合并
 | 
	
		
			
				|  |  |          df_1 = df[(df['target_type'] == 'action_ratio') &
 | 
	
		
			
				|  |  |                    (df['improve_ratio'] >= config['filterTargetCombine']['actionRatio']['improveRatio']) &
 | 
	
		
			
				|  |  | -                  (df['sample_size'] >= config['filterTargetCombine']['actionRatio']['sampleSize'])]
 | 
	
		
			
				|  |  | +                  (df['sample_size'] >= config['filterTargetCombine']['actionRatio']['sampleSize']) &
 | 
	
		
			
				|  |  | +                  df['p_value'] <= 0.05]
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  |          df_2 = df[(df['target_type'] == 'convertRatio') &
 | 
	
		
			
				|  |  |                    (df['improve_ratio'] >= config['filterTargetCombine']['convertRatio']['improveRatio']) &
 | 
	
		
			
				|  |  | -                  (df['sample_size'] >= config['filterTargetCombine']['convertRatio']['sampleSize'])]
 | 
	
		
			
				|  |  | +                  (df['sample_size'] >= config['filterTargetCombine']['convertRatio']['sampleSize']) &
 | 
	
		
			
				|  |  | +                  df['p_value'] <= 0.05]
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  |          merge_df = pd.concat([df_1, df_2], axis=0)
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  |          # 'age', 'gender', 'city', 'business', 'province', 'client'
 | 
	
		
			
				|  |  |          features = [key for key, value in config['bayesDim'].items() if value['isOn']]
 | 
	
		
			
				|  |  | -        self.signature_target_combine = merge_df[['id', 'signature'] + features].to_dict(orient='records')
 | 
	
		
			
				|  |  | +        self.target_combine_from_table = merge_df[['id', 'signature'] + features].to_dict(orient='records')
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  |      def write_intelligence_strategy_table(self):
 | 
	
		
			
				|  |  |          """
 | 
	
	
		
			
				|  | @@ -240,7 +246,7 @@ class GetTargetAndAssemblyParameters(object):
 | 
	
		
			
				|  |  |                    if_exists='append',
 | 
	
		
			
				|  |  |                    index=False)
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -    def target_combine_is_subset(self):
 | 
	
		
			
				|  |  | +    def filter_target_combine_by_whether_is_subset(self):
 | 
	
		
			
				|  |  |          """
 | 
	
		
			
				|  |  |          ctop_ai_kuaishou_signature_recommended_target_combine 表中的 city 和 province 都对应 self.advertiser_strategy 的 region []  varchar
 | 
	
		
			
				|  |  |          age 对应 ages_range []  varchar
 | 
	
	
		
			
				|  | @@ -252,7 +258,7 @@ class GetTargetAndAssemblyParameters(object):
 | 
	
		
			
				|  |  |          age_bool = True
 | 
	
		
			
				|  |  |          region_bool = True
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -        for item in self.signature_target_combine:
 | 
	
		
			
				|  |  | +        for item in self.target_combine_from_table:
 | 
	
		
			
				|  |  |              if 'gender' in item.keys():
 | 
	
		
			
				|  |  |                  gender_bool, gender_dict = is_contains_gender(item['gender'], self.advertiser_strategy['gender'])
 | 
	
		
			
				|  |  |              if 'age' in item.keys():
 | 
	
	
		
			
				|  | @@ -268,7 +274,7 @@ class GetTargetAndAssemblyParameters(object):
 | 
	
		
			
				|  |  |                  target_combine.update(gender_dict) if 'gender' in item.keys() else None
 | 
	
		
			
				|  |  |                  target_combine.update(age_dict) if 'age' in item.keys() else None
 | 
	
		
			
				|  |  |                  target_combine.update(region_dict) if 'city' in item.keys() else None
 | 
	
		
			
				|  |  | -                self.final_target_combine.append(target_combine)
 | 
	
		
			
				|  |  | +                self.target_combine_filter_by_subset.append(target_combine)
 | 
	
		
			
				|  |  |              else:
 | 
	
		
			
				|  |  |                  logger.info("推荐定向:%s 与 账户配置信息里的定向(gender:%s, age_min: %s,age_max: %s, ages_ranges:%s,region:%s )存在冲突" %
 | 
	
		
			
				|  |  |                              (item,
 | 
	
	
		
			
				|  | @@ -278,10 +284,42 @@ class GetTargetAndAssemblyParameters(object):
 | 
	
		
			
				|  |  |                               self.advertiser_strategy['ages_range'],
 | 
	
		
			
				|  |  |                               self.advertiser_strategy['region']))
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | +    def filter_target_combine_by_creative_cnt(self):
 | 
	
		
			
				|  |  | +        """
 | 
	
		
			
				|  |  | +        依据素材已经关联的创意个数,来定素材可以创建的定向组合个数, 并从中随机随着N个
 | 
	
		
			
				|  |  | +        """
 | 
	
		
			
				|  |  | +        # 1-1 获取素材在该账户下关联的创意个数,过滤掉素材关联创意个数超过200的情况
 | 
	
		
			
				|  |  | +        df = pd.DataFrame(self.target_combine_filter_by_subset)
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +        sql = """
 | 
	
		
			
				|  |  | +                select t1.signature, t2.creative_count
 | 
	
		
			
				|  |  | +                from
 | 
	
		
			
				|  |  | +                (select account_id, signature, photo_id from ctop_kuaishou_video_get 
 | 
	
		
			
				|  |  | +                 where account_id = %s and signature in %s 
 | 
	
		
			
				|  |  | +                 group by account_id, signature) t1
 | 
	
		
			
				|  |  | +                left join
 | 
	
		
			
				|  |  | +                ctop_kuaishou_video_relate_creatives t2
 | 
	
		
			
				|  |  | +                on t1.account_id = t2.account_id and t1.photo_id = t2.photo_id 
 | 
	
		
			
				|  |  | +                """ % (self.account_id, tuple(df.signature.unique()))
 | 
	
		
			
				|  |  | +        creative_cnt_df = pd.read_sql(sql, self.engine)
 | 
	
		
			
				|  |  | +        creative_cnt_df = creative_cnt_df[creative_cnt_df.creative_count < 200]
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +        # 1-2 计算每个素材还能创建的广告组(定向)个数: (200 - 已关联创意个数) / 15
 | 
	
		
			
				|  |  | +        creative_cnt_df['target_combine_cnt'] = np.floor((200 - creative_cnt_df['creative_count']) / 15)
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +        final_target_combine = pd.DataFrame([])
 | 
	
		
			
				|  |  | +        for sig in creative_cnt_df.signature.unique():
 | 
	
		
			
				|  |  | +            n = creative_cnt_df[creative_cnt_df.signature == sig].target_combine_cnt.values[0]
 | 
	
		
			
				|  |  | +            now = datetime.datetime.now()
 | 
	
		
			
				|  |  | +            sig_target_df = df[df.signature == sig].sample(n, axis=0, random_state=(now.year + now.month + now.day))
 | 
	
		
			
				|  |  | +            final_target_combine = final_target_combine.append(sig_target_df)
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +        self.target_combine_to_create = final_target_combine.to_dict(orient='records')
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  |          # 从集合中随机选取N个组合
 | 
	
		
			
				|  |  |          combine_cnt = config['filterTargetCombine']['combineCnt']
 | 
	
		
			
				|  |  | -        random.shuffle(self.final_target_combine)
 | 
	
		
			
				|  |  | -        self.final_target_combine = self.final_target_combine[:combine_cnt]
 | 
	
		
			
				|  |  | +        random.shuffle(self.target_combine_to_create)
 | 
	
		
			
				|  |  | +        self.target_combine_to_create = self.target_combine_to_create[:combine_cnt]
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  |      def assembly_group_and_creative_params(self):
 | 
	
		
			
				|  |  |          """
 | 
	
	
		
			
				|  | @@ -295,7 +333,7 @@ class GetTargetAndAssemblyParameters(object):
 | 
	
		
			
				|  |  |          batch_group_params_to_db = []
 | 
	
		
			
				|  |  |          batch_creative_params_to_db = []
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -        for item in self.final_target_combine:
 | 
	
		
			
				|  |  | +        for item in self.target_combine_to_create:
 | 
	
		
			
				|  |  |              # 使用优质定向更新组层级的参数
 | 
	
		
			
				|  |  |              target_combine = item.copy()
 | 
	
		
			
				|  |  |              del target_combine['target_combine_id']
 | 
	
	
		
			
				|  | @@ -309,8 +347,6 @@ class GetTargetAndAssemblyParameters(object):
 | 
	
		
			
				|  |  |                  if col in target_combine.keys():
 | 
	
		
			
				|  |  |                      group_params_to_db[col] = str(group_params_to_db[col])
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -
 | 
	
		
			
				|  |  | -
 | 
	
		
			
				|  |  |              group_uuid = str(uuid.uuid4())
 | 
	
		
			
				|  |  |              group_name = group_params['unit_name']
 | 
	
		
			
				|  |  |  
 | 
	
	
		
			
				|  | @@ -531,7 +567,7 @@ class GetTargetAndAssemblyParameters(object):
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  |          # 6、用于组装发送请求的部分字段,需要转化为list类型
 | 
	
		
			
				|  |  |          cols_to_list = ['app_store', 'scene_id', 'region', 'day_budget_schedule', 'ages_range', 'device_brand', 'business_interest',
 | 
	
		
			
				|  |  | -                        'fans_star', 'interest_video', 'app_interest', 'app_interest_ids', 'app_ids', 'population','district_ids',
 | 
	
		
			
				|  |  | +                        'fans_star', 'interest_video', 'app_interest', 'app_interest_ids', 'app_ids', 'population', 'district_ids',
 | 
	
		
			
				|  |  |                          'exclude_population', 'paid_audience']
 | 
	
		
			
				|  |  |          for col in cols_to_list:
 | 
	
		
			
				|  |  |              if col in group_params.keys():
 | 
	
	
		
			
				|  | @@ -549,7 +585,7 @@ class GetTargetAndAssemblyParameters(object):
 | 
	
		
			
				|  |  |                             'sticker_title': self.advertiser_strategy.get('sticker_title'),
 | 
	
		
			
				|  |  |                             'overlay_type': self.advertiser_strategy.get('overlay_type'),
 | 
	
		
			
				|  |  |                             'expose_tag': self.advertiser_strategy.get('expose_tag'),
 | 
	
		
			
				|  |  | -                           'new_expose_tag':self.advertiser_strategy.get('new_expose_tag'),
 | 
	
		
			
				|  |  | +                           'new_expose_tag': self.advertiser_strategy.get('new_expose_tag'),
 | 
	
		
			
				|  |  |                             'site_id': self.advertiser_strategy.get('site_id'),
 | 
	
		
			
				|  |  |                             'click_track_url': self.advertiser_strategy.get('click_track_url'),
 | 
	
		
			
				|  |  |                             'impression_url': self.advertiser_strategy.get('impression_url'),
 | 
	
	
		
			
				|  | @@ -566,10 +602,9 @@ class GetTargetAndAssemblyParameters(object):
 | 
	
		
			
				|  |  |              random.shuffle(description_lst)
 | 
	
		
			
				|  |  |              creative_params['description'] = description_lst[0]
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -
 | 
	
		
			
				|  |  |          # 2 写入数据库的创意参数
 | 
	
		
			
				|  |  |          creative_params_to_db = creative_params.copy()
 | 
	
		
			
				|  |  | -        drop_cols = ['put_status','live_creative_type']
 | 
	
		
			
				|  |  | +        drop_cols = ['put_status', 'live_creative_type']
 | 
	
		
			
				|  |  |          for col in drop_cols:
 | 
	
		
			
				|  |  |              if col in creative_params_to_db:
 | 
	
		
			
				|  |  |                  del creative_params_to_db[col]
 | 
	
	
		
			
				|  | @@ -606,4 +641,4 @@ class GetTargetAndAssemblyParameters(object):
 | 
	
		
			
				|  |  |                                  SET message = %s
 | 
	
		
			
				|  |  |                                  WHERE ai_strategy_uuid = %s""", (str(message), self.ai_strategy_uuid))
 | 
	
		
			
				|  |  |          db_con.commit()
 | 
	
		
			
				|  |  | -        db_con.close()
 | 
	
		
			
				|  |  | +        db_con.close()
 |