liyuyi@c-top.com.cn 4 년 전
부모
커밋
9c17acaab9
3개의 변경된 파일155개의 추가작업 그리고 117개의 파일을 삭제
  1. 105 72
      ai_target_combine_handler.py
  2. 1 2
      time_task/time_task_create_ad_by_target.py
  3. 49 43
      utils/commonFunc.py

+ 105 - 72
ai_target_combine_handler.py

@@ -30,21 +30,33 @@ class AiTargetCombine(tornado.web.RequestHandler):
             ins = GetTargetAndAssemblyParameters(account_id, project_id)
             # 2、判断当天是否已经存在包含【优质定向】的广告计划,如果有则获取该计划id,否则需要新建计划
             sql = """select campaign_id, campaign_name from ctop_kuaishou_campaign 
-                    where account_id = %s 
+                    where account_id = %s
                     and put_create_time >= curdate()
-                    and campaign_name like %%优质定向%% """ % ins.account_id
+                    and campaign_name like '%s'""" % (ins.account_id, '%%优质定向%%')
             df = pd.read_sql(sql, ins.engine)
             if df.empty:
-                ins.add_campaign()
+                # ins.add_campaign()
+                ins.campaign_id = 123123
             else:
                 ins.campaign_id = df['campaign_id'].values[0]
-            # 3、发送创建组和创意的请求
-            request = requests.post(create_group_and_creative_url,
-                                    headers=headers,
-                                    data=json.dumps(ins.request_data, cls=NpEncoder))
-            response_data = json.loads(request.text)
-            # 4 接口的返回信息更新到数据库表
-            ins.update_intelligence_strategy_table(message=response_data)
+
+            # 3、拼装广告组和创意的参数
+            ins.assembly_group_and_creative_params()
+
+            # 4、信息写如到智能策略表中
+            ins.write_intelligence_strategy_table()
+
+            # 5、发送创建组和创意的请求
+            # request = requests.post(create_group_and_creative_url,
+            #                         headers=headers,
+            #                         data=json.dumps(ins.request_data, cls=NpEncoder))
+            # response_data = json.loads(request.text)
+
+            # 6、接口的返回信息更新到数据库表
+            # ins.update_intelligence_strategy_table(message=response_data)
+            ins.update_intelligence_strategy_table(message="异步执行")
+            self.write(json.dumps({"account_id": account_id, "message": "异步执行", "code": 1}))
+            self.flush()
         except:
             # 这个是打印到控制台吗?
             self.write(json.dumps({"account_id": account_id, "message": traceback.format_exc(), "code": -1}))
@@ -56,21 +68,20 @@ class GetTargetAndAssemblyParameters(object):
         self.account_id = account_id
         self.project_id = project_id
         self.engine = None
+        self.product_engine = None
         self.db_config = None
         self.signature_target_combine = None
         self.final_target_combine = []
         self.advertiser_strategy_id = None
         self.advertiser_strategy = {}
         self.ai_strategy_uuid = str(uuid.uuid4())
-        self.request_data = None
+        self.request_data = {}
         self.campaign_id = None
         self.operation_type = 1  # 新增广告组
         self.get_database_engine()  # 获取数据库信息
         self.get_advertiser_strategy_info()  # 获取账户配置信息
         self.get_signature_and_target()  # 获取定向组合
         self.target_combine_is_subset()  # 过滤出为账户相信子集的定向组合
-        self.assembly_group_and_creative_params()   # 拼装广告组和创意的参数
-        self.write_intelligence_strategy_table()  # 以上信息写如到智能策略表中
 
     def get_database_engine(self):
         # 数据库连接引擎,依据开发环境/生产环境 进行切换
@@ -81,6 +92,9 @@ class GetTargetAndAssemblyParameters(object):
             self.engine = get_db_engine(config['productDB'])
             self.db_config = config['productDB']
 
+        # 有些表需要连接生产数据库,不管当前是开发环境还是测试环境
+        self.product_engine = get_db_engine(config['productDB'])
+
     def get_advertiser_strategy_info(self):
         """
         获取指定账户下客户的策略信息
@@ -155,7 +169,8 @@ class GetTargetAndAssemblyParameters(object):
         读取配置文件的条件,筛选出符合条件的定向组合
         """
         sql = """
-        select * from ctop_ai_kuaishou_signature_recommended_target_combine where project_id = %s and stat_date = curdate()
+        select * from ctop_ai_kuaishou_signature_recommended_target_combine where project_id = %s 
+        -- and stat_date = curdate()
         """ % self.project_id
         df = pd.read_sql(sql, self.engine)
         # 计算组合的概率,相对于实际投放概率高出了百分之多少
@@ -171,9 +186,10 @@ class GetTargetAndAssemblyParameters(object):
                   (df['sample_size'] >= config['filterTargetCombine']['convertRatio']['sampleSize'])]
 
         merge_df = pd.concat([df_1, df_2], axis=0)
-        self.signature_target_combine = merge_df[['signature', 'age', 'gender', 'city', 'business', 'province', 'client']]. \
-            to_dict(orient='records')
 
+        # 'age', 'gender', 'city', 'business', 'province', 'client'
+        features = [key for key, value in config['bayesDim'].items() if value['isOn']]
+        self.signature_target_combine = merge_df[['id', 'signature'] + features].to_dict(orient='records')
 
     def write_intelligence_strategy_table(self):
         """
@@ -202,19 +218,26 @@ class GetTargetAndAssemblyParameters(object):
         business_interest 对应 business_interest varchar
         gender 对应 gender int
         """
+        gender_bool = True
+        age_bool = True
+        region_bool = True
+
         for item in self.signature_target_combine:
-            gender_bool, gender_dict = is_contains_gender(item['gender'], self.advertiser_strategy['gender'])
-            age_bool, age_dict = is_contains_age(item['age'],
-                                                 self.advertiser_strategy['age_min'],
-                                                 self.advertiser_strategy['age_max'],
-                                                 self.advertiser_strategy['ages_range'])
-            region_bool, region_dict = is_contains_region(item['city'], self.advertiser_strategy['region'])
+            if 'gender' in item.keys():
+                gender_bool, gender_dict = is_contains_gender(item['gender'], self.advertiser_strategy['gender'])
+            if 'age' in item.keys():
+                age_bool, age_dict = is_contains_age(item['age'],
+                                                     self.advertiser_strategy['age_min'],
+                                                     self.advertiser_strategy['age_max'],
+                                                     self.advertiser_strategy['ages_range'])
+            if 'city' in item.keys():
+                region_bool, region_dict = is_contains_region(item['city'], self.advertiser_strategy['region'])
 
             if gender_bool & age_bool & region_bool:
-                target_combine = {'signature': item['signature']}
-                target_combine.update(gender_dict)
-                target_combine.update(age_dict)
-                target_combine.update(region_dict)
+                target_combine = {'signature': item['signature'], 'target_combine_id': item['id']}
+                target_combine.update(gender_dict) if 'gender' in item.keys() else None
+                target_combine.update(age_dict) if 'age' in item.keys() else None
+                target_combine.update(region_dict) if 'city' in item.keys() else None
                 self.final_target_combine.append(target_combine)
             else:
                 # TODO logger
@@ -222,7 +245,8 @@ class GetTargetAndAssemblyParameters(object):
 
         # 从集合中随机选取N个组合
         combine_cnt = config['filterTargetCombine']['combineCnt']
-        self.final_target_combine = random.shuffle(self.final_target_combine)[:combine_cnt]
+        random.shuffle(self.final_target_combine)
+        self.final_target_combine = self.final_target_combine[:combine_cnt]
 
     def assembly_group_and_creative_params(self):
         """
@@ -237,20 +261,28 @@ class GetTargetAndAssemblyParameters(object):
         batch_creative_params_to_db = []
 
         for item in self.final_target_combine:
+            # 使用优质定向更新组层级的参数
+            target_combine = item.copy()
+            del target_combine['target_combine_id']
+            del target_combine['signature']
+            group_params.update(target_combine)
+            group_params_to_db.update(target_combine)
+
             group_uuid = str(uuid.uuid4())
             group_name = group_params['unit_name']
 
             # 获取素材名称, 在广告组命名中进行替换 (同时获取photo_id, 用于拼装创意层级的参数)
             sql = """select photo_id, photo_name from ctop_kuaishou_video_get where account_id = %s and signature = '%s' limit 1""" % \
                   (self.account_id, item['signature'])
-            signature_df = pd.read(sql, self.engine)
-            photo_name = signature_df['photo_name'].values[0]
-            photo_id = signature_df['photo_id'].values[0]
+            signature_df = pd.read_sql(sql, self.product_engine)
+            photo_name = signature_df['photo_name'].values[0] if not signature_df.empty else 'unknown'
+            photo_id = signature_df['photo_id'].values[0] if not signature_df.empty else None
             group_name.replace('{{素材名称}}', photo_name)
             group_name = group_name + '_优质定向' + '_' + str(group_cnt)
 
             # 写入数据表中组信息
             single_group_params_to_db = group_params_to_db.copy()
+            single_group_params_to_db['target_combine_id'] = item['target_combine_id']
             single_group_params_to_db['group_uuid'] = group_uuid
             single_group_params_to_db['group_name'] = group_name
             single_group_params_to_db['create_time'] = datetime.datetime.now()
@@ -265,50 +297,51 @@ class GetTargetAndAssemblyParameters(object):
 
             # 拼装广告组下面的创意信息
             # 一个素材组合为15个一样的创意,因信息流已经下架,不需要提供封面信息
-            creative_cnt = 1
-            for cnt in range(15):
+            for creative_cnt in range(15):
                 creative_uuid = str(uuid.uuid4())
-                creative_name = creative_params_to_db['creative_name'] + '_' + str(creative_cnt)
+                creative_name = creative_params_to_db['creative_name'] + '_' + str(creative_cnt + 1)
 
                 single_creative_params_to_db = creative_params_to_db.copy()
                 single_creative_params_to_db['creative_uuid'] = creative_uuid
                 single_creative_params_to_db['creative_name'] = creative_name
-                single_creative_params_to_db['photo_id'] = int(photo_id)
+                single_creative_params_to_db['photo_id'] = int(photo_id) if photo_id else None
                 single_creative_params_to_db['create_time'] = datetime.datetime.now()
                 batch_creative_params_to_db.append(single_creative_params_to_db)
 
                 single_creative_params_to_request = creative_params.copy()
                 single_creative_params_to_request['creative_uuid'] = creative_uuid
                 single_creative_params_to_request['creative_name'] = creative_name
-                single_creative_params_to_request['photo_id'] = int(photo_id)
+                single_creative_params_to_request['photo_id'] = int(photo_id) if photo_id else None
 
                 # 单个创意信息加入到组里面的 creative_list 中
                 single_group_params_to_request['creative_list'].append(single_creative_params_to_request)
-                creative_cnt += 1
 
             # 组和创意信息添加到最终返回数据 group_list 中
             self.request_data['group_list'].append(single_group_params_to_request)
 
-            # 创意层级信息批量写入数据库
-            creative_df = pd.DataFrame(batch_creative_params_to_db)
-            creative_df.to_sql(name="ctop_ai_kuaishou_creative_level_operation_record",
-                               con=self.engine,
-                               if_exists='append',
-                               index=False)
-            # 组层级信息批量写入数据库
-            group_df = pd.DataFrame(batch_group_params_to_db)
-            group_df.to_sql(name="ctop_ai_kuaishou_unit_level_operation_record",
-                            con=self.engine,
-                            if_exists='append',
-                            index=False)
-
-            self.request_data['account_id'] = self.account_id
-            self.request_data['campaign_id'] = self.campaign_id
+        # 创意层级信息批量写入数据库
+        creative_df = pd.DataFrame(batch_creative_params_to_db)
+        creative_df.to_csv('creative_df.csv',index=False)
+        creative_df.to_sql(name="ctop_ai_kuaishou_creative_level_operation_record",
+                           con=self.engine,
+                           if_exists='append',
+                           index=False)
+
+        # 组层级信息批量写入数据库
+        group_df = pd.DataFrame(batch_group_params_to_db)
+        group_df.to_csv('group_df.csv', index=False)
+        # group_df.to_sql(name="ctop_ai_kuaishou_unit_level_operation_record",
+        #                 con=self.engine,
+        #                 if_exists='append',
+        #                 index=False)
+
+        self.request_data['account_id'] = self.account_id
+        self.request_data['campaign_id'] = self.campaign_id
 
     def get_group_params(self):
         # 1、从账户信息表读取基本的配置信息
         group_params = {'unit_name': self.advertiser_strategy.get('group_name'),
-                        'put_status': self.advertiser_strategy.get('group_status'),
+                        # 'put_status': self.advertiser_strategy.get('group_status'),
                         'template_id': self.advertiser_strategy.get('template_id'),
                         'bid_type': self.advertiser_strategy.get('bid_type'),
                         'use_app_market': self.advertiser_strategy.get('use_app_market'),
@@ -339,19 +372,19 @@ class GetTargetAndAssemblyParameters(object):
                         'video_landing_page': self.advertiser_strategy.get('video_landing_page'),
                         'auto_target': self.advertiser_strategy.get('auto_target'),
                         'auto_create_photo': self.advertiser_strategy.get('auto_create_photo'),
-                        'item_id': self.advertiser_strategy.get('item_id'),
-                        'merchant_item_put_type	': self.advertiser_strategy.get('merchant_item_put_type	'),
-                        'fiction_id': self.advertiser_strategy.get('fiction_id'),
+                        # 'item_id': self.advertiser_strategy.get('item_id'),
+                        # 'merchant_item_put_type': self.advertiser_strategy.get('merchant_item_put_type'),
+                        # 'fiction_id': self.advertiser_strategy.get('fiction_id'),
                         'smart_cover': self.advertiser_strategy.get('smart_cover'),
                         'asset_mining': self.advertiser_strategy.get('asset_mining'),
-                        'consult_id': self.advertiser_strategy.get('consult_id'),
-                        'adv_card_option': self.advertiser_strategy.get('adv_card_option'),
-                        'adv_card_list': self.advertiser_strategy.get('adv_card_list'),
-                        'card_type': self.advertiser_strategy.get('card_type'),
-                        'merchandise_id': self.advertiser_strategy.get('merchandise_id'),
-                        'merchandise_type': self.advertiser_strategy.get('merchandise_type'),
-                        'intention_target': self.advertiser_strategy.get('intention_target'),
-                        'dpa_unit_param': self.advertiser_strategy.get('dpa_unit_param'),
+                        # 'consult_id': self.advertiser_strategy.get('consult_id'),
+                        # 'adv_card_option': self.advertiser_strategy.get('adv_card_option'),
+                        # 'adv_card_list': self.advertiser_strategy.get('adv_card_list'),
+                        # 'card_type': self.advertiser_strategy.get('card_type'),
+                        # 'merchandise_id': self.advertiser_strategy.get('merchandise_id'),
+                        # 'merchandise_type': self.advertiser_strategy.get('merchandise_type'),
+                        # 'intention_target': self.advertiser_strategy.get('intention_target'),
+                        # 'dpa_unit_param': self.advertiser_strategy.get('dpa_unit_param'),
                         'region': self.advertiser_strategy.get('region'),
                         'district_ids': self.advertiser_strategy.get('district_ids'),
                         'user_type': self.advertiser_strategy.get('user_type'),
@@ -376,11 +409,11 @@ class GetTargetAndAssemblyParameters(object):
                         if self.advertiser_strategy.get('interest_video') else None,
                         'app_interest': eval(self.advertiser_strategy.get('app_interest'))
                         if self.advertiser_strategy.get('app_interest') else None,
-                        'app_interest_ids': eval(self.advertiser_strategy.get('app_interest_ids'))
-                        if self.advertiser_strategy.get('app_interest_ids') else None,
+                        # 'app_interest_ids': eval(self.advertiser_strategy.get('app_interest_ids'))
+                        # if self.advertiser_strategy.get('app_interest_ids') else None,
                         'app_ids': eval(self.advertiser_strategy.get('app_ids'))
                         if self.advertiser_strategy.get('app_ids') else None,
-                        'filter_converted_level	': self.advertiser_strategy.get('filter_converted_level	'),
+                        'filter_converted_level': self.advertiser_strategy.get('filter_converted_level'),
                         'population': eval(self.advertiser_strategy.get('population'))
                         if self.advertiser_strategy.get('population') else None,
                         'exclude_population': eval(self.advertiser_strategy.get('exclude_population'))
@@ -401,11 +434,11 @@ class GetTargetAndAssemblyParameters(object):
         # 3、广告组名称,通配符的替换
         # 900299 -{{广告位置}}--{{创意制作方式}}--{{日期}}-  (日期、素材名称、应用名称、应用标记、创意制作方式、广告位置)
         # 3-1 获取应用标记和应用名称
-        sql = """select app_version, app_name from ctop_kuaishou_app_list where app_id = % limit 1""" % \
-              group_params.get('app_id')
+        sql = """select app_version, app_name from ctop_kuaishou_app_list where app_id = %s limit 1""" % \
+              group_params.get('app_id', -1)
         app_info_df = pd.read_sql(sql, self.engine)
-        app_version = app_info_df['app_version'].values[0]
-        app_name = app_info_df['app_name'].values[0]
+        app_version = app_info_df['app_version'].values[0] if not app_info_df.empty else 'unknown'
+        app_name = app_info_df['app_name'].values[0] if not app_info_df.empty else 'unknown'
         # 3-2 素材名称, 在拼装广告组和创意层级函数中获取
         # 3-3 获取创意制作方式
         unit_type = '自定义' if group_params.get('unit_type') == 4 \
@@ -453,7 +486,7 @@ class GetTargetAndAssemblyParameters(object):
     def get_creative_params(self):
         # 1、从账户信息表读取基本的配置信息
         creative_params = {'creative_name': self.advertiser_strategy.get('creative_name'),
-                           'put_status': self.advertiser_strategy.get('put_status'),
+                           # 'put_status': self.advertiser_strategy.get('put_status'),
                            'action_bar_text': self.advertiser_strategy.get('action_bar_text'),
                            'description': self.advertiser_strategy.get('description'),
                            'short_slogan': self.advertiser_strategy.get('short_slogan'),
@@ -470,7 +503,7 @@ class GetTargetAndAssemblyParameters(object):
                            'creative_category': self.advertiser_strategy.get('creative_category'),
                            'creative_tag': eval(self.advertiser_strategy.get('creative_tag'))
                            if self.advertiser_strategy.get('creative_tag') else None,
-                           'live_creative_type': self.advertiser_strategy.get('live_creative_type')
+                           # 'live_creative_type': self.advertiser_strategy.get('live_creative_type')
                            }
 
         # 2 写入数据库的创意参数

+ 1 - 2
time_task/time_task_create_ad_by_target.py

@@ -21,8 +21,7 @@ def send_request(args):
         request_data = json.dumps({"account_id": args[0], "project_id": args[1]})
         print('target_combine_url', target_combine_url)
         request = requests.post(target_combine_url, request_data)
-
-        return json.loads(request.text)
+        return request.text
     except Exception:
         return "account_id = %s, request = %s, traceback = %s " % (args[0], request, traceback.format_exc())
 

+ 49 - 43
utils/commonFunc.py

@@ -28,53 +28,57 @@ def is_contains_region(val1, val2):
                                 仅计划的campaign_type为5时,支持设置三级地域(例:山西-大同-左云,左云是三级地域)
 
     :param val1: 推荐的城市等级组合,如 "一线城市|新一线城市|二线城市|三线城市"  , "不限"
-    :param val2: 从账户配置信息表里读取的地区信息
+    :param val2: 从账户配置信息表里读取的地区信息, 如 '[]', None, '[14,15,21,22,23,31,32]'
     :return:
     """
     if val1 == '不限' or val1 is None:
-        if not val2:
+        if val2 is None or val2 == '[]':
             return True, {'region': None}
         else:
             return False, -1
     else:
-        val = []
-        sql = """
-        select t1.city_name, t2.level, t2.region_id, t2.parent from ctop_kuaishou_city_level t1
-        left join ctop_kuaishou_region_list_parent t2
-        on t1.city_name = t2.name
-        where t1.city_level in %s
-        """ % (tuple(val1.split('|')),)
-        city_df = pd.read_sql(sql, None)
-
-        # level1
-        if city_df[city_df.level == 1].region_id.values in eval(val2):
-            val.extend(city_df[city_df.level == 1].region_id.values)
-        else:
+        if val2 is None or val2 == '[]':
             return False, -1
-
-        # level2
-        # 元素的父级是否存在于 val2,如果存在则为包含于的关系
-        # 元素本身是否存在于 val2 ,如果存在则为包含于的关系
-        # 以上都不满足,val1 不包含于 val2
-        for item in city_df[city_df.level == 2].region_id.values:
-            parent = city_df[city_df.region_id == item].parent.values[0]
-            if parent in eval(val2):
-                val.append(item)
-            elif item in eval(val2):
-                val.append(item)
+        else:
+            val = []
+            sql = """
+            select t1.city_name, t2.level, t2.region_id, t2.parent from ctop_kuaishou_city_level t1
+            left join ctop_kuaishou_region_list_parent t2
+            on t1.city_name = t2.name
+            where t1.city_level in %s
+            """ % (tuple(val1.split('|')),)
+            city_df = pd.read_sql(sql, None)
+
+            # level1
+            if set(city_df[city_df.level == 1].region_id.values).issubset(set(eval(val2))):
+                val.extend(city_df[city_df.level == 1].region_id.values)
             else:
                 return False, -1
 
-        # level3
-        for item in city_df[city_df.level == 3].region_id.values:
-            parent = city_df[city_df.region_id == item].parent.values[0]
-            if parent in eval(val2):
-                val.append(item)
-            elif item in eval(val2):
-                val.append(item)
-            else:
-                return False, -1
-    return True, {'region': val}
+            # level2
+            # 元素的父级是否存在于 val2,如果存在则为包含于的关系
+            # 元素本身是否存在于 val2 ,如果存在则为包含于的关系
+            # 以上都不满足,val1 不包含于 val2
+            for item in city_df[city_df.level == 2].region_id.values:
+                parent = city_df[city_df.region_id == item].parent.values[0]
+                if parent in eval(val2):
+                    val.append(item)
+                elif item in eval(val2):
+                    val.append(item)
+                else:
+                    return False, -1
+
+            # level3
+            for item in city_df[city_df.level == 3].region_id.values:
+                parent = city_df[city_df.region_id == item].parent.values[0]
+                if parent in eval(val2):
+                    val.append(item)
+                elif item in eval(val2):
+                    val.append(item)
+                else:
+                    return False, -1
+
+            return True, {'region': val}
 
 
 def is_contains_gender(val1, val2):
@@ -87,10 +91,10 @@ def is_contains_gender(val1, val2):
     val = None
     if val1 == '男' and val2 in (2, 0):
         is_contains = True
-        val = val1
+        val = 2
     if val1 == '女' and val2 in (1, 0):
         is_contains = True
-        val = val1
+        val = 1
     if val1 == '不限' and val2 == 0:
         is_contains = True
         val = val2
@@ -125,14 +129,16 @@ def is_contains_age(val1, val2_min, val2_max, val2_range):
     与age不能同时传;【18:表示18-23岁】;【24:表示24-30岁】;【31:表示31-40岁】;【41:表示41-49岁】;【50:表示50-100岁】
     :return:
     """
-    age_dict = {'18-23岁': 18, '24-30岁': 24, '31-40岁': 31, '41-49岁': 41, '50-100岁': 50}
-    if val1 == '不限' or ((not val2_min) and (not val2_max) and (not val2_range)):
-        return True, {'ages_range': None}
-
+    age_dict = {'18-23岁': 18, '24-30岁': 24, '31-40岁': 31, '41-49岁': 41, '50+岁': 50, '50-100岁': 50}
+    if val1 == '不限' or not val1:
+        if (not val2_min) and (not val2_max) and (not val2_range or val2_range == '不限'):
+            return True, {'ages_range': None}
+        else:
+            return False, -1
     else:
         age1 = [age_dict[item] for item in val1.split('|')]
         if (val2_min is None and val2_max is None) and val2_range:
-            if age1 in eval(val2_range):
+            if set(age1).issubset(set(eval(val2_range))):
                 return True, {'ages_range': age1}
             else:
                 return False, -1