liyuyi@c-top.com.cn 4 anos atrás
pai
commit
309fa45fad

+ 1 - 1
BayesCombine.py

@@ -2,7 +2,7 @@ import yaml
 import os
 import pandas as pd
 import numpy as np
-from commonFunc import update_dict, get_db_engine
+from utils.commonFunc import update_dict, get_db_engine
 from functools import reduce
 from itertools import product
 from datetime import datetime

+ 162 - 34
ai_target_combine_handler.py

@@ -2,11 +2,15 @@ import uuid
 import datetime
 import tornado
 import json
+import traceback
 import pandas as pd
+import pymysql
 import os
 import yaml
+import random
 import requests
-from commonFunc import get_db_engine, is_contains_gender, is_contains_age, is_contains_region
+from utils.commonFunc import get_db_engine, is_contains_gender, is_contains_age, is_contains_region
+from utils.BaseClass import NpEncoder
 from config.url import create_campaign_url, create_group_and_creative_url, headers
 
 with open('config/config.yaml', mode='r', encoding='utf-8') as f:
@@ -15,21 +19,36 @@ with open('config/config.yaml', mode='r', encoding='utf-8') as f:
 
 class AiTargetCombine(tornado.web.RequestHandler):
     def post(self):
-        res = {'code': 0,
-               'message': "SUCCESS"}
-
+        # 解析接收到的参数
         data = self.request.body
         data = str(data, 'utf8')
         data = json.loads(data, encoding='utf8')
-
+        account_id = data['account_id']
+        project_id = data['project_id']
         try:
-            pass
-        except Exception:
-            pass
-
-        # 返回接口结果
-        self.write(json.dumps(res))
-        self.flush()
+            # 1、创建类实例
+            ins = GetTargetAndAssemblyParameters(account_id, project_id)
+            # 2、判断当天是否已经存在包含【优质定向】的广告计划,如果有则获取该计划id,否则需要新建计划
+            sql = """select campaign_id, campaign_name from ctop_kuaishou_campaign 
+                    where account_id = %s 
+                    and put_create_time >= curdate()
+                    and campaign_name like %%优质定向%% """ % ins.account_id
+            df = pd.read_sql(sql, ins.engine)
+            if df.empty:
+                ins.add_campaign()
+            else:
+                ins.campaign_id = df['campaign_id'].values[0]
+            # 3、发送创建组和创意的请求
+            request = requests.post(create_group_and_creative_url,
+                                    headers=headers,
+                                    data=json.dumps(ins.request_data, cls=NpEncoder))
+            response_data = json.loads(request.text)
+            # 4 接口的返回信息更新到数据库表
+            ins.update_intelligence_strategy_table(message=response_data)
+        except:
+            # 这个是打印到控制台吗?
+            self.write(json.dumps({"account_id": account_id, "message": traceback.format_exc(), "code": -1}))
+            self.flush()
 
 
 class GetTargetAndAssemblyParameters(object):
@@ -37,7 +56,7 @@ class GetTargetAndAssemblyParameters(object):
         self.account_id = account_id
         self.project_id = project_id
         self.engine = None
-        self.get_database_engine()
+        self.db_config = None
         self.signature_target_combine = None
         self.final_target_combine = []
         self.advertiser_strategy_id = None
@@ -46,13 +65,21 @@ class GetTargetAndAssemblyParameters(object):
         self.request_data = None
         self.campaign_id = None
         self.operation_type = 1  # 新增广告组
+        self.get_database_engine()  # 获取数据库信息
+        self.get_advertiser_strategy_info()  # 获取账户配置信息
+        self.get_signature_and_target()  # 获取定向组合
+        self.target_combine_is_subset()  # 过滤出为账户相信子集的定向组合
+        self.assembly_group_and_creative_params()   # 拼装广告组和创意的参数
+        self.write_intelligence_strategy_table()  # 以上信息写如到智能策略表中
 
     def get_database_engine(self):
         # 数据库连接引擎,依据开发环境/生产环境 进行切换
         if os.getenv('environment', 'unknown') == 'dev':
             self.engine = get_db_engine(config['devDB'])
+            self.db_config = config['devDB']
         else:
             self.engine = get_db_engine(config['productDB'])
+            self.db_config = config['productDB']
 
     def get_advertiser_strategy_info(self):
         """
@@ -147,6 +174,7 @@ class GetTargetAndAssemblyParameters(object):
         self.signature_target_combine = merge_df[['signature', 'age', 'gender', 'city', 'business', 'province', 'client']]. \
             to_dict(orient='records')
 
+
     def write_intelligence_strategy_table(self):
         """
         拼接好的组创意的参数写入到 ctop_ai_kuaishou_intelligence_strategy 中的 ai_strategy_request_content
@@ -192,30 +220,93 @@ class GetTargetAndAssemblyParameters(object):
                 # TODO logger
                 pass
 
+        # 从集合中随机选取N个组合
+        combine_cnt = config['filterTargetCombine']['combineCnt']
+        self.final_target_combine = random.shuffle(self.final_target_combine)[:combine_cnt]
+
     def assembly_group_and_creative_params(self):
         """
         拼接参数,请求创建接口
         """
         group_params, group_params_to_db = self.get_group_params()
-        creative_params = self.get_creative_params()
+        creative_params, creative_params_to_db = self.get_creative_params()
 
         self.request_data['group_list'] = []
         group_cnt = 1
+        batch_group_params_to_db = []
+        batch_creative_params_to_db = []
+
         for item in self.final_target_combine:
             group_uuid = str(uuid.uuid4())
             group_name = group_params['unit_name']
 
-            # 1-2 获取素材名称
-            sql = """select * from ctop_kuaishou_video_get where account_id = %s and signature = '%s' limit 1""" % \
+            # 获取素材名称, 在广告组命名中进行替换 (同时获取photo_id, 用于拼装创意层级的参数)
+            sql = """select photo_id, photo_name from ctop_kuaishou_video_get where account_id = %s and signature = '%s' limit 1""" % \
                   (self.account_id, item['signature'])
-            signature_name_df = pd.read(sql, self.engine)
-            signature_name = signature_name_df['photo_name'].values[0]
-            group_name.replace('{{素材名称}}', signature_name)
-
+            signature_df = pd.read(sql, self.engine)
+            photo_name = signature_df['photo_name'].values[0]
+            photo_id = signature_df['photo_id'].values[0]
+            group_name.replace('{{素材名称}}', photo_name)
             group_name = group_name + '_优质定向' + '_' + str(group_cnt)
+
+            # 写入数据表中组信息
+            single_group_params_to_db = group_params_to_db.copy()
+            single_group_params_to_db['group_uuid'] = group_uuid
+            single_group_params_to_db['group_name'] = group_name
+            single_group_params_to_db['create_time'] = datetime.datetime.now()
+            batch_group_params_to_db.append(single_group_params_to_db)
+
+            # 拼装返回json的组信息
+            single_group_params_to_request = group_params.copy()
+            single_group_params_to_request['group_uuid'] = group_uuid
+            single_group_params_to_request['unit_name'] = group_name
+            single_group_params_to_request['creative_list'] = []
             group_cnt += 1
 
+            # 拼装广告组下面的创意信息
+            # 一个素材组合为15个一样的创意,因信息流已经下架,不需要提供封面信息
+            creative_cnt = 1
+            for cnt in range(15):
+                creative_uuid = str(uuid.uuid4())
+                creative_name = creative_params_to_db['creative_name'] + '_' + str(creative_cnt)
+
+                single_creative_params_to_db = creative_params_to_db.copy()
+                single_creative_params_to_db['creative_uuid'] = creative_uuid
+                single_creative_params_to_db['creative_name'] = creative_name
+                single_creative_params_to_db['photo_id'] = int(photo_id)
+                single_creative_params_to_db['create_time'] = datetime.datetime.now()
+                batch_creative_params_to_db.append(single_creative_params_to_db)
+
+                single_creative_params_to_request = creative_params.copy()
+                single_creative_params_to_request['creative_uuid'] = creative_uuid
+                single_creative_params_to_request['creative_name'] = creative_name
+                single_creative_params_to_request['photo_id'] = int(photo_id)
+
+                # 单个创意信息加入到组里面的 creative_list 中
+                single_group_params_to_request['creative_list'].append(single_creative_params_to_request)
+                creative_cnt += 1
+
+            # 组和创意信息添加到最终返回数据 group_list 中
+            self.request_data['group_list'].append(single_group_params_to_request)
+
+            # 创意层级信息批量写入数据库
+            creative_df = pd.DataFrame(batch_creative_params_to_db)
+            creative_df.to_sql(name="ctop_ai_kuaishou_creative_level_operation_record",
+                               con=self.engine,
+                               if_exists='append',
+                               index=False)
+            # 组层级信息批量写入数据库
+            group_df = pd.DataFrame(batch_group_params_to_db)
+            group_df.to_sql(name="ctop_ai_kuaishou_unit_level_operation_record",
+                            con=self.engine,
+                            if_exists='append',
+                            index=False)
+
+            self.request_data['account_id'] = self.account_id
+            self.request_data['campaign_id'] = self.campaign_id
+
     def get_group_params(self):
+        # 1、从账户信息表读取基本的配置信息
         group_params = {'unit_name': self.advertiser_strategy.get('group_name'),
                         'put_status': self.advertiser_strategy.get('group_status'),
                         'template_id': self.advertiser_strategy.get('template_id'),
@@ -303,23 +394,23 @@ class GetTargetAndAssemblyParameters(object):
                         'behavior_interest': self.advertiser_strategy.get('behavior_interest')
                         }
 
-        # smart_cover 和 asset_mining 需要将0/1 转化为 False/True
+        # 2、smart_cover 和 asset_mining 需要将0/1 转化为 False/True
         group_params['smart_cover'] = True if group_params.get('smart_cover', 0) == 1 else False
         group_params['asset_mining'] = True if group_params.get('asset_mining', 0) == 1 else False
 
-        # 广告组名称,通配符的替换
+        # 3、广告组名称,通配符的替换
         # 900299 -{{广告位置}}--{{创意制作方式}}--{{日期}}-  (日期、素材名称、应用名称、应用标记、创意制作方式、广告位置)
-        # 1-1 获取应用标记和应用名称
+        # 3-1 获取应用标记和应用名称
         sql = """select app_version, app_name from ctop_kuaishou_app_list where app_id = % limit 1""" % \
               group_params.get('app_id')
         app_info_df = pd.read_sql(sql, self.engine)
         app_version = app_info_df['app_version'].values[0]
         app_name = app_info_df['app_name'].values[0]
-        # 1-2 素材名称, 在拼装广告组和创意层级函数中获取
-        # 1-3 获取创意制作方式
+        # 3-2 素材名称, 在拼装广告组和创意层级函数中获取
+        # 3-3 获取创意制作方式
         unit_type = '自定义' if group_params.get('unit_type') == 4 \
             else ('程序化创意' if group_params.get('unit_type') == 5 else '程序化创意2.0')
-        # 1-4 获取广告位置
+        # 3-4 获取广告位置
         scene_id_dict = {1: '优选广告位',
                          3: '视频播放页广告-便利贴广告',
                          5: '联盟广告',
@@ -328,19 +419,26 @@ class GetTargetAndAssemblyParameters(object):
                          10: '联盟场景'}
         scene_id = '+'.join([scene_id_dict[item] for item in group_params.get('scene_id')])
 
-        group_name = group_params['group_name'].replace('{{应用标记}}', app_version). \
+        unit_name = group_params['unit_name'].replace('{{应用标记}}', app_version). \
             replace('{{应用名称}}', app_name). \
             replace('{{创意制作方式}}', unit_type). \
             replace('{{广告位置}}', scene_id)
-        # 1-2 日期, 在拼装广告组和创意层级函数中获取
+
+        # 3-5 日期, 在拼装广告组和创意层级函数中获取
         now_date = str(datetime.datetime.now().date())
-        if '日期' in group_params['group_name']:
-            group_name = group_name.replace('{{日期}}', '_' + now_date)
+        if '日期' in group_params['unit_name']:
+            unit_name = unit_name.replace('{{日期}}', now_date)
         else:
-            group_name = group_name + '_' + now_date
-        group_params['unit_name'] = group_name
+            unit_name = unit_name + '_' + now_date
+        group_params['unit_name'] = unit_name
 
-        # 2 写入数据库的组参数
+        # 4、处理 cpa_bid
+        # 因为存在 【单出价请输入单数字,阶梯出价请用/隔开】 如:1900/2000/2100 cpa_bid varchar(255) DEFAULT NULL COMMENT '出价'
+        # 处理方式按'/'进行分割,取最后一个值(也就是最大值),并转化为int型
+        group_params['cpa_bid'] = int(group_params['cpa_bid'].split('/')[-1]) if '/' in group_params['cpa_bid'] \
+            else (int(group_params['cpa_bid'].split('-')[-1]) if '-' in group_params['cpa_bid'] else int(group_params['cpa_bid']))
+
+        # 5、 写入数据库的组参数
         group_params_to_db = group_params.copy()
         group_params_to_db['group_name'] = group_params_to_db.pop('unit_name')
         group_params_to_db['ai_strategy_uuid'] = self.ai_strategy_uuid
@@ -353,6 +451,7 @@ class GetTargetAndAssemblyParameters(object):
         return group_params, group_params_to_db
 
     def get_creative_params(self):
+        # 1、从账户信息表读取基本的配置信息
         creative_params = {'creative_name': self.advertiser_strategy.get('creative_name'),
                            'put_status': self.advertiser_strategy.get('put_status'),
                            'action_bar_text': self.advertiser_strategy.get('action_bar_text'),
@@ -373,4 +472,33 @@ class GetTargetAndAssemblyParameters(object):
                            if self.advertiser_strategy.get('creative_tag') else None,
                            'live_creative_type': self.advertiser_strategy.get('live_creative_type')
                            }
-        return creative_params
+
+        # 2 写入数据库的创意参数
+        creative_params_to_db = creative_params.copy()
+        creative_params_to_db['ai_strategy_uuid'] = self.ai_strategy_uuid
+        creative_params_to_db['account_id'] = self.account_id
+        creative_params_to_db['campaign_id'] = self.campaign_id
+        creative_params_to_db['operation_type'] = self.operation_type
+        creative_params_to_db['status'] = 1
+        creative_params_to_db['message'] = None
+
+        return creative_params, creative_params_to_db
+
+    def update_intelligence_strategy_table(self, message=None):
+        """
+        更新 intelligence_strategy_table
+        :return:
+        """
+        db_con = pymysql.connect(host=self.db_config['host'],
+                                 port=self.db_config['port'],
+                                 user=self.db_config['username'],
+                                 password=self.db_config['password'],
+                                 database=self.db_config['database'],
+                                 charset='utf8')
+        cursor = db_con.cursor()
+        if message:
+            cursor.execute("""UPDATE ctop_ai_kuaishou_intelligence_strategy
+                                SET message = %s
+                                WHERE ai_strategy_uuid = %s""", (str(message), self.ai_strategy_uuid))
+        db_con.commit()
+        db_con.close()

+ 1 - 3
ai_target_server.py

@@ -8,8 +8,6 @@ from ai_target_combine_handler import AiTargetCombine
 
 def create_application():
     web_application = tornado.web.Application([
-        # (r"/ai_callback_add_group", AiCallBackAddGroup),
-        # (r"/ai_callback_add_creative", AiCallBackAddCreative),
         (r"/ai_target_combine", AiTargetCombine),
     ])
     return web_application
@@ -22,6 +20,6 @@ if __name__ == "__main__":
     if platform.system() == 'Windows':
         http_server.start()
     else:
-        http_server.start(num_processes=5)
+        http_server.start(num_processes=1)
     tornado.ioloop.IOLoop.instance().start()
 

+ 3 - 0
config/config.yaml

@@ -1,5 +1,7 @@
+# 目前支持优质定向功能的项目列表
 projectId:
   - 458
+  - 458
 
 targetType:
   - 'action_ratio'
@@ -81,6 +83,7 @@ filterTargetCombine:
     convertRatio:
         sampleSize: 10000
         improveRatio: 0.2
+    combineCnt: 20
 
 
 

+ 11 - 2
config/url.py

@@ -6,8 +6,17 @@ os_env = os.getenv('environment', 'unknown')
 product_url = 'http://api.tjyourong.com.cn/jeecg-boot/'
 dev_url = 'http://192.168.1.8:8080/jeecg-boot/'
 
+dev_server_port = 'http://139.186.165.84:31012/'
+product_server_port = 'http://139.186.27.96:31012/'
+
 # 创建计划url
-create_campaign_url = dev_url if os_env == 'dev' else product_url + 'kuaishou/create/campaignCreate'
+create_campaign_url = dev_url if os_env == 'dev' else product_url
+create_campaign_url += 'kuaishou/create/campaignCreate'
 
 # 创建组和创意url
-create_group_and_creative_url = dev_url if os_env == 'dev' else product_url + 'kuaishou/create/createUnitAndCreative'
+create_group_and_creative_url = dev_url if os_env == 'dev' else product_url
+create_group_and_creative_url += 'kuaishou/create/createUnitAndCreative'
+
+# ai_target_combine 服务的接口
+target_combine_url = dev_server_port  if os_env == 'dev' else product_server_port
+target_combine_url += 'ai_target_combine'

+ 81 - 0
time_task/time_task_create_ad_by_target.py

@@ -0,0 +1,81 @@
+from concurrent.futures import ThreadPoolExecutor
+import requests
+import datetime
+import json
+import pandas as pd
+import uuid
+import traceback
+import yaml
+from utils.commonFunc import get_db_engine
+from config.url import target_combine_url
+
+with open('../config/config.yaml', mode='r', encoding='utf-8') as f:
+    config = yaml.load(f.read(), Loader=yaml.FullLoader)
+product_db_engine = get_db_engine(config['productDB'])
+project_ids = config['projectId']  # 目前支持优质定向功能的项目列表
+
+
+def send_request(args):
+    request = None
+    try:
+        request_data = json.dumps({"account_id": args[0], "project_id": args[1]})
+        print('target_combine_url', target_combine_url)
+        request = requests.post(target_combine_url, request_data)
+
+        return json.loads(request.text)
+    except Exception:
+        return "account_id = %s, request = %s, traceback = %s " % (args[0], request, traceback.format_exc())
+
+
+# 0、 打印该定时任务被调用的时间
+unique_uuid = str(uuid.uuid4())
+print("***********", "优质定向创建-Start, unique_uuid = ", unique_uuid, datetime.datetime.now(), "**************")
+
+# 1、获取配置了【优质定向】的 account_id 及其 project_id
+# TODO 读取是否配置优质定向的字段,进行判断
+sql = """
+select t1.account_id, t2.project_id
+from
+(select account_id from ctop_ai_kuaishou_advertiser_strategy where status = 1
+ -- and open_program_create in (0,2)
+ ) t1
+ left join 
+ (select project_id,account_id from ctop_user_allocation where project_id in %s) t2
+ on t1.account_id = t2.account_id
+""" % (tuple(project_ids),)
+
+account_df = pd.read_sql(sql, product_db_engine)
+account_df.drop_duplicates('account_id', keep='first', inplace=True)
+acc_list = [(item['account_id'], item['project_id']) for index, item in account_df.iterrows()]
+
+acc_list = [(9774238, 458)]
+
+# 2、 5个账户一组进行发送请求
+batch_num = 5
+for i in range(0, len(acc_list), batch_num):
+    if i + batch_num < len(acc_list):
+        print(datetime.datetime.now(), tuple(acc_list[i:i+batch_num]))
+        with ThreadPoolExecutor(max_workers=batch_num) as pool:
+            results = pool.map(send_request, tuple(acc_list[i: i+batch_num]))
+            for r in results:
+                print('res = %s' % r)
+    else:
+        with ThreadPoolExecutor(max_workers=batch_num) as pool:
+            print(datetime.datetime.now(), acc_list[i: len(acc_list)])
+            results = pool.map(send_request, tuple(acc_list[i: len(acc_list)]))
+            for r in results:
+                print('res = %s' % r)
+
+
+# 3、 打印该定时任务结束的时间
+print("***********", "优质定向创建-End, unique_uuid = ", unique_uuid, datetime.datetime.now(), "**************")
+
+
+
+
+
+
+
+
+
+

+ 30 - 0
utils/BaseClass.py

@@ -0,0 +1,30 @@
+import tornado.web
+import json
+import numpy as np
+
+
+class BaseHandler(tornado.web.RequestHandler):
+    def set_default_headers(self):
+        print("setting headers!!!")
+        self.set_header("access-control-allow-origin", "*")
+        self.set_header("Access-Control-Allow-Headers", "x-requested-with")
+        self.set_header('Access-Control-Allow-Methods', 'GET, PUT, DELETE, OPTIONS')
+        # HEADERS!
+        self.set_header("Access-Control-Allow-Headers", "access-control-allow-origin,authorization,content-type")
+
+    def options(self):
+        # no body
+        self.set_status(204)
+        self.finish()
+
+
+class NpEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, np.integer):
+            return int(obj)
+        elif isinstance(obj, np.floating):
+            return float(obj)
+        elif isinstance(obj, np.ndarray):
+            return obj.tolist()
+        else:
+            return super(NpEncoder, self).default(obj)

+ 0 - 4
commonFunc.py

@@ -1,10 +1,6 @@
 from sqlalchemy import create_engine
 from urllib import parse
 import pandas as pd
-import yaml
-
-with open('config/config.yaml', mode='r', encoding='utf-8') as f:
-    config = yaml.load(f.read(), Loader=yaml.FullLoader)
 
 
 def update_dict(x, y):