liyuyi@c-top.com.cn před 4 roky
rodič
revize
8afb99871b
3 změnil soubory, kde provedl 131 přidání a 68 odebrání
  1. 10 1
      BayesCombine.py
  2. 58 23
      ai_target_combine_handler.py
  3. 63 44
      utils/commonFunc.py

+ 10 - 1
BayesCombine.py

@@ -2,7 +2,7 @@ import yaml
 import os
 import pandas as pd
 import numpy as np
-from utils.commonFunc import update_dict, get_db_engine
+from utils.commonFunc import update_dict, get_db_engine, city_code_transform,age_code_transform, gender_code_transform
 from functools import reduce
 from itertools import product
 from datetime import datetime
@@ -155,6 +155,15 @@ class BayesCombine(object):
             out_dict['sample_size'] = self.sample_size
 
             # TODO 调用人群预估覆盖接口,得到该组合的人群覆盖数
+            # 获取该项目下在投的账号
+            get_acc_sql = """select account_id from ctop_user_allocation where project_id = %s and account_status=0 limit 1""" % \
+                          self.project_id
+            account_df = pd.read_sql(get_acc_sql, engine)
+            account_id = account_df['account_id'].values[0]
+            request_data = {'region': city_code_transform(out_dict.get('city')),
+                            'ages_range': age_code_transform(out_dict.get('age')),
+                            'gender': gender_code_transform(out_dict.get('gender')),
+                            'advertiser_id': account_id}
             out_dict['crowd_coverage_cnt'] = 1000
 
             out_dict['combine_estimate_prob'] = prob

+ 58 - 23
ai_target_combine_handler.py

@@ -4,6 +4,7 @@ import tornado
 import json
 import traceback
 import pandas as pd
+import numpy as np
 import pymysql
 import os
 import yaml
@@ -23,7 +24,6 @@ logger.addHandler(log_handler)
 logger.setLevel(logging.DEBUG)
 print('id of ai_target_combine_logger %s' % id(logger))
 
-
 with open('config/config.yaml', mode='r', encoding='utf-8') as f:
     config = yaml.load(f.read(), Loader=yaml.FullLoader)
 
@@ -90,18 +90,20 @@ class GetTargetAndAssemblyParameters(object):
         self.engine = None
         self.product_engine = None
         self.db_config = None
-        self.signature_target_combine = None
-        self.final_target_combine = []
         self.advertiser_strategy_id = None
         self.advertiser_strategy = {}
         self.ai_strategy_uuid = str(uuid.uuid4())
         self.request_data = {}
         self.campaign_id = None
         self.operation_type = 1  # 新增广告组
+        self.target_combine_from_table = []
+        self.target_combine_filter_by_subset = []
+        self.target_combine_to_create = []
         self.get_database_engine()  # 获取数据库信息
         self.get_advertiser_strategy_info()  # 获取账户配置信息
-        self.get_signature_and_target()  # 获取定向组合
-        self.target_combine_is_subset()  # 过滤出为账户配置子集的定向组合
+        self.get_signature_and_target()  # 从素材定向组合表中获取最新的定向组合
+        self.filter_target_combine_by_whether_is_subset()  # 筛选出为账户配置子集的定向组合
+        self.filter_target_combine_by_creative_cnt()  # 依据素材在指定账户下关联的创意个数,进行筛选创建个数,防止超限导致的创建失败
 
     def get_database_engine(self):
         # 数据库连接引擎,依据开发环境/生产环境 进行切换
@@ -197,29 +199,33 @@ class GetTargetAndAssemblyParameters(object):
         """
         从 ctop_ai_kuaishou_signature_recommended_target_combine 表中读取素材和对应的定向
         读取配置文件的条件,筛选出符合条件的定向组合
+        TODO 放开sql语句中注释的代码
         """
         sql = """
         select * from ctop_ai_kuaishou_signature_recommended_target_combine where project_id = %s 
-        -- and stat_date = curdate()
+        -- and stat_date = (select max(stat_date) from ctop_ai_kuaishou_signature_recommended_target_combine)
         """ % self.project_id
         df = pd.read_sql(sql, self.engine)
-        # 计算组合的概率,相对于实际投放概率高出了百分之多少
+
+        # 计算组合的概率,相对于实际投放概率高出了百分之多少,样本量是否达标(读取配置文件),且p值小于等于0.05
         df['improve_ratio'] = (df['combine_estimate_prob'] - df['actual_prob']) / df['actual_prob']
 
         # 两种类型的过滤标准不一样,分开进行判断,然后对结果进行合并
         df_1 = df[(df['target_type'] == 'action_ratio') &
                   (df['improve_ratio'] >= config['filterTargetCombine']['actionRatio']['improveRatio']) &
-                  (df['sample_size'] >= config['filterTargetCombine']['actionRatio']['sampleSize'])]
+                  (df['sample_size'] >= config['filterTargetCombine']['actionRatio']['sampleSize']) &
+                  df['p_value'] <= 0.05]
 
         df_2 = df[(df['target_type'] == 'convertRatio') &
                   (df['improve_ratio'] >= config['filterTargetCombine']['convertRatio']['improveRatio']) &
-                  (df['sample_size'] >= config['filterTargetCombine']['convertRatio']['sampleSize'])]
+                  (df['sample_size'] >= config['filterTargetCombine']['convertRatio']['sampleSize']) &
+                  df['p_value'] <= 0.05]
 
         merge_df = pd.concat([df_1, df_2], axis=0)
 
         # 'age', 'gender', 'city', 'business', 'province', 'client'
         features = [key for key, value in config['bayesDim'].items() if value['isOn']]
-        self.signature_target_combine = merge_df[['id', 'signature'] + features].to_dict(orient='records')
+        self.target_combine_from_table = merge_df[['id', 'signature'] + features].to_dict(orient='records')
 
     def write_intelligence_strategy_table(self):
         """
@@ -240,7 +246,7 @@ class GetTargetAndAssemblyParameters(object):
                   if_exists='append',
                   index=False)
 
-    def target_combine_is_subset(self):
+    def filter_target_combine_by_whether_is_subset(self):
         """
         ctop_ai_kuaishou_signature_recommended_target_combine 表中的 city 和 province 都对应 self.advertiser_strategy 的 region []  varchar
         age 对应 ages_range []  varchar
@@ -252,7 +258,7 @@ class GetTargetAndAssemblyParameters(object):
         age_bool = True
         region_bool = True
 
-        for item in self.signature_target_combine:
+        for item in self.target_combine_from_table:
             if 'gender' in item.keys():
                 gender_bool, gender_dict = is_contains_gender(item['gender'], self.advertiser_strategy['gender'])
             if 'age' in item.keys():
@@ -268,7 +274,7 @@ class GetTargetAndAssemblyParameters(object):
                 target_combine.update(gender_dict) if 'gender' in item.keys() else None
                 target_combine.update(age_dict) if 'age' in item.keys() else None
                 target_combine.update(region_dict) if 'city' in item.keys() else None
-                self.final_target_combine.append(target_combine)
+                self.target_combine_filter_by_subset.append(target_combine)
             else:
                 logger.info("推荐定向:%s 与 账户配置信息里的定向(gender:%s, age_min: %s,age_max: %s, ages_ranges:%s,region:%s )存在冲突" %
                             (item,
@@ -278,10 +284,42 @@ class GetTargetAndAssemblyParameters(object):
                              self.advertiser_strategy['ages_range'],
                              self.advertiser_strategy['region']))
 
+    def filter_target_combine_by_creative_cnt(self):
+        """
+        依据素材已经关联的创意个数,来定素材可以创建的定向组合个数, 并从中随机随着N个
+        """
+        # 1-1 获取素材在该账户下关联的创意个数,过滤掉素材关联创意个数超过200的情况
+        df = pd.DataFrame(self.target_combine_filter_by_subset)
+
+        sql = """
+                select t1.signature, t2.creative_count
+                from
+                (select account_id, signature, photo_id from ctop_kuaishou_video_get 
+                 where account_id = %s and signature in %s 
+                 group by account_id, signature) t1
+                left join
+                ctop_kuaishou_video_relate_creatives t2
+                on t1.account_id = t2.account_id and t1.photo_id = t2.photo_id 
+                """ % (self.account_id, tuple(df.signature.unique()))
+        creative_cnt_df = pd.read_sql(sql, self.engine)
+        creative_cnt_df = creative_cnt_df[creative_cnt_df.creative_count < 200]
+
+        # 1-2 计算每个素材还能创建的广告组(定向)个数: (200 - 已关联创意个数) / 15
+        creative_cnt_df['target_combine_cnt'] = np.floor((200 - creative_cnt_df['creative_count']) / 15)
+
+        final_target_combine = pd.DataFrame([])
+        for sig in creative_cnt_df.signature.unique():
+            n = creative_cnt_df[creative_cnt_df.signature == sig].target_combine_cnt.values[0]
+            now = datetime.datetime.now()
+            sig_target_df = df[df.signature == sig].sample(n, axis=0, random_state=(now.year + now.month + now.day))
+            final_target_combine = final_target_combine.append(sig_target_df)
+
+        self.target_combine_to_create = final_target_combine.to_dict(orient='records')
+
         # 从集合中随机选取N个组合
         combine_cnt = config['filterTargetCombine']['combineCnt']
-        random.shuffle(self.final_target_combine)
-        self.final_target_combine = self.final_target_combine[:combine_cnt]
+        random.shuffle(self.target_combine_to_create)
+        self.target_combine_to_create = self.target_combine_to_create[:combine_cnt]
 
     def assembly_group_and_creative_params(self):
         """
@@ -295,7 +333,7 @@ class GetTargetAndAssemblyParameters(object):
         batch_group_params_to_db = []
         batch_creative_params_to_db = []
 
-        for item in self.final_target_combine:
+        for item in self.target_combine_to_create:
             # 使用优质定向更新组层级的参数
             target_combine = item.copy()
             del target_combine['target_combine_id']
@@ -309,8 +347,6 @@ class GetTargetAndAssemblyParameters(object):
                 if col in target_combine.keys():
                     group_params_to_db[col] = str(group_params_to_db[col])
 
-
-
             group_uuid = str(uuid.uuid4())
             group_name = group_params['unit_name']
 
@@ -531,7 +567,7 @@ class GetTargetAndAssemblyParameters(object):
 
         # 6、用于组装发送请求的部分字段,需要转化为list类型
         cols_to_list = ['app_store', 'scene_id', 'region', 'day_budget_schedule', 'ages_range', 'device_brand', 'business_interest',
-                        'fans_star', 'interest_video', 'app_interest', 'app_interest_ids', 'app_ids', 'population','district_ids',
+                        'fans_star', 'interest_video', 'app_interest', 'app_interest_ids', 'app_ids', 'population', 'district_ids',
                         'exclude_population', 'paid_audience']
         for col in cols_to_list:
             if col in group_params.keys():
@@ -549,7 +585,7 @@ class GetTargetAndAssemblyParameters(object):
                            'sticker_title': self.advertiser_strategy.get('sticker_title'),
                            'overlay_type': self.advertiser_strategy.get('overlay_type'),
                            'expose_tag': self.advertiser_strategy.get('expose_tag'),
-                           'new_expose_tag':self.advertiser_strategy.get('new_expose_tag'),
+                           'new_expose_tag': self.advertiser_strategy.get('new_expose_tag'),
                            'site_id': self.advertiser_strategy.get('site_id'),
                            'click_track_url': self.advertiser_strategy.get('click_track_url'),
                            'impression_url': self.advertiser_strategy.get('impression_url'),
@@ -566,10 +602,9 @@ class GetTargetAndAssemblyParameters(object):
             random.shuffle(description_lst)
             creative_params['description'] = description_lst[0]
 
-
         # 2 写入数据库的创意参数
         creative_params_to_db = creative_params.copy()
-        drop_cols = ['put_status','live_creative_type']
+        drop_cols = ['put_status', 'live_creative_type']
         for col in drop_cols:
             if col in creative_params_to_db:
                 del creative_params_to_db[col]
@@ -606,4 +641,4 @@ class GetTargetAndAssemblyParameters(object):
                                 SET message = %s
                                 WHERE ai_strategy_uuid = %s""", (str(message), self.ai_strategy_uuid))
         db_con.commit()
-        db_con.close()
+        db_con.close()

+ 63 - 44
utils/commonFunc.py

@@ -1,6 +1,7 @@
 from sqlalchemy import create_engine
 from urllib import parse
 import pandas as pd
+from utils.code_dict import age_dict
 import yaml
 
 
@@ -44,7 +45,6 @@ def is_contains_region(val1, val2):
                 on t1.city_name = t2.name
                 where t1.city_level in %s
                 """ % (val_tuple,)
-    print(sql)
     with open('config/config.yaml', mode='r', encoding='utf-8') as f:
         config = yaml.load(f.read(), Loader=yaml.FullLoader)
 
@@ -100,33 +100,16 @@ def is_contains_gender(val1, val2):
     :param val2: ctop_ai_kuaishou_advertiser_strategy 中的 gender 字段: 1:女性, 2:男性,0表示不限
     :return: val1 是否为 val2的子集,以及传递给快手后台的值
     """
-    is_contains = False
-    val = None
+    if val1 is None:
+        # 表示该字段没有参与贝叶斯的计算, 直接返回客户投放策略里的值
+        return True, {'gender': val2}
+
     if val1 == '男' and val2 in (2, 0):
-        is_contains = True
-        val = 2
+        return True, {'gender': 2}
     if val1 == '女' and val2 in (1, 0):
-        is_contains = True
-        val = 1
+        return True, {'gender': 1}
     if val1 == '不限' and val2 == 0:
-        is_contains = True
-        val = val2
-
-    # 如果val1为空,表示该字段没有参与定向组合运算,就直接取客户投放策略里面的值
-    if val1 is None:
-        is_contains = True
-        val = val2
-
-    return is_contains, {'gender': val}
-
-
-def is_contains_platform_os(val1, val2):
-    """
-    :param val1: ctop_ai_kuaishou_signature_recommended_target_combine 中的 client 字段: '男'、'女'、'不限'、None
-    :param val2:
-    :return:
-    """
-    pass
+        return True, {'gender': 0}
 
 
 def is_contains_age(val1, val2_min, val2_max, val2_range):
@@ -146,30 +129,66 @@ def is_contains_age(val1, val2_min, val2_max, val2_range):
     if val1 is None:
         return True, {'age_min': val2_min, 'age_max': val2_max, 'ages_range': val2_range}
 
-    age_dict = {'18-23岁': 18, '24-30岁': 24, '31-40岁': 31, '41-49岁': 41, '50+岁': 50, '50-100岁': 50}
+    if val1 == '不限':
+        if (not val2_min) and (not val2_max) and (not val2_range or val2_range == '不限'):
+            return True, {'age_min': val2_min, 'age_max': val2_max, 'ages_range': val2_range}
+        else:
+            return False, -1
+    else:
+        val1 = [age_dict[ele] for ele in val1.split('|')]
+        if (not val2_min) and (not val2_max) and (not val2_range or val2_range == '不限'):
+            return True, {'ages_range': val1}
+        if (val2_min is None and val2_max is None) and val2_range:
+            if set(val1).issubset(set(eval(val2_range))):
+                return True, {'ages_range': val1}
+            else:
+                return False, -1
+        if (val2_min and val2_max) and (not val2_range):
+            if val1[0] >= val2_min and val1[-1] <= val2_max:
+                return True, {'age_min': val1[0], 'age_max': val1[-1]}
+            else:
+                return False, -1
 
-    if (not val2_min) and (not val2_max) and (not val2_range or val2_range == '不限'):
-        return True, {'ages_range': val1}
+
+def age_code_transform(val):
+    if (val is None) or (val == '不限'):
+        ages_range = None
     else:
-        if val1 == '不限':
-            return False, -1
-        else:
-            age1 = [age_dict[item] for item in val1.split('|')]
-            if (val2_min is None and val2_max is None) and val2_range:
-                if set(age1).issubset(set(eval(val2_range))):
-                    return True, {'ages_range': age1}
-                else:
-                    return False, -1
-            if (val2_min and val2_max) and (not val2_range):
-                if age1[0] >= val2_min and age1[-1] <= val2_max:
-                    return True, {'age_min': age1[0], 'age_max': age1[-1]}
-                else:
-                    return False, -1
+        ages_range = [age_dict[item] for item in val.split('|')]
+    return ages_range
+
+
+def gender_code_transform(val):
+    if (val is None) or (val == '不限'):
+        gender = 0
+    elif val == '女':
+        gender = 1
+    else:
+        gender = 2
+    return gender
+
+
+def city_code_transform(val):
+    if (val is None) or (val == '不限'):
+        region = None
+    else:
+        sql = """select t1.city_name, t2.level, t2.region_id, t2.parent from ctop_kuaishou_city_level t1
+                left join ctop_kuaishou_region_list_parent t2
+                on t1.city_name = t2.name
+                where t1.city_level in %s
+                 """ % (tuple(val.split('|')),)
 
+        with open('config/config.yaml', mode='r', encoding='utf-8') as f:
+            config = yaml.load(f.read(), Loader=yaml.FullLoader)
+        city_df = pd.read_sql(sql, get_db_engine(config['productDB']))
 
-def is_contains_business_interest(val1, val2):
-    pass
+        # 如果【吉林】在city_level里面的话,在ctop_kuaishou_region_list_parent会同时把 吉林省和吉林市查询出来
+        # 按level进行降序排列后,按city_name进行去重,保留高等级的值
+        city_df = city_df.sort_values(by='level', ascending=False)
+        city_df.drop_duplicates(subset=['city_name'], keep='first', inplace=True)
 
+        region = city_df.region_id.values()
+    return region