Selaa lähdekoodia

1、程序化2.0在组层级添加了两个字段2、解决多进程日志丢失使用concurrent_log模块代替之前的TimedRotatingFileHandler

liyuyi@c-top.com.cn 4 vuotta sitten
vanhempi
commit
ab62ac7edb

+ 5 - 1
advertiser_strategy_operate.py

@@ -2,6 +2,10 @@ import datetime
 import json
 import traceback
 from logging.handlers import TimedRotatingFileHandler
+from concurrent_log import ConcurrentTimedRotatingFileHandler
+import pymysql
+from utils.CommonFunction import get_lower_case_name
+
 import pandas as pd
 from utils.BaseClass import *
 from utils.DataBaseConfig import *
@@ -9,7 +13,7 @@ import logging
 
 log_formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%m/%d/%Y %I:%M:%S %p')
 
-log_handler = TimedRotatingFileHandler("logs/advertiser_strategy_operate.log",
+log_handler = ConcurrentTimedRotatingFileHandler("logs/advertiser_strategy_operate.log",
                                        when="midnight", backupCount=100, encoding='utf-8')
 log_handler.setFormatter(log_formatter)
 logger = logging.getLogger('advertiser_strategy_operate_logger')

+ 2 - 1
ai_callback_handler.py

@@ -5,9 +5,10 @@ import tornado.web
 import pymysql
 from utils.DataBaseConfig import *
 import logging
+from concurrent_log import ConcurrentTimedRotatingFileHandler
 
 log_formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%m/%d/%Y %I:%M:%S %p')
-log_handler = TimedRotatingFileHandler("logs/ai_callback.log",
+log_handler = ConcurrentTimedRotatingFileHandler("logs/ai_callback.log",
                                        when="midnight", backupCount=100)
 log_handler.setFormatter(log_formatter)
 logger = logging.getLogger('ai_callback_logger')

+ 24 - 12
ai_strategy_request_func.py

@@ -2,6 +2,7 @@ import datetime
 import json
 import uuid
 from logging.handlers import TimedRotatingFileHandler
+from concurrent_log import ConcurrentTimedRotatingFileHandler
 import pandas as pd
 import pymysql
 import requests
@@ -10,8 +11,9 @@ from utils.UrlConfig import *
 import logging
 
 log_formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%m/%d/%Y %I:%M:%S %p')
-log_handler = TimedRotatingFileHandler("logs/ai_strategy_request.log",
-                                       when="midnight", backupCount=100)
+# log_handler = TimedRotatingFileHandler("logs/ai_strategy_request.log",
+#                                        when="midnight", backupCount=100)
+log_handler = ConcurrentTimedRotatingFileHandler("logs/ai_strategy_request.log", when="midnight", backupCount=100)
 log_handler.setFormatter(log_formatter)
 logger = logging.getLogger('ai_strategy_request_logger')
 logger.addHandler(log_handler)
@@ -206,29 +208,34 @@ class ParseAddCampaignOrAddGroupRequest(object):
                      'sticker_title', 'overlay_type', 'expose_tag', 'new_expose_tag', 'site_id',
                      'click_track_url', 'impression_url', 'ad_photo_played_t3s_url', 'actionbar_click_url',
                      'creative_category', 'creative_tag', 'image_cnt',
-                     'create_time', 'effective_time', 'expiry_time', 'timer_task_status', 'smart_cover', 'asset_mining']
+                     'create_time', 'effective_time', 'expiry_time', 'timer_task_status']
         for col in drop_cols:
             if col in group_params.keys():
                 del group_params[col]
 
+
         # 2 从请求的信息中更新组层级信息, 如组的名称、测试定向、测试出价等, update客户策略信息
         if (self.group_info is not None) and (len(self.group_info) > 0):
             for key, value in self.group_info.items():
                 group_params.update({key: value})
 
-        # 3 拼装json的组参数
+        # 3 smart_cover 和 asset_mining 需要将0/1 转化为 False/True
+        group_params['smart_cover'] = True if group_params.get('smart_cover', 0) == 1 else False
+        group_params['asset_mining'] = True if group_params.get('asset_mining', 0) == 1 else False
+
+        # 4 拼装json的组参数
         group_params_to_request = group_params.copy()
-        # 3-1 拼接发送请求json时, 以下字段需要转为 list 类型
+        # 4-1 拼接发送请求json时, 以下字段需要转为 list 类型
         cols_to_list = ['app_store', 'scene_id', 'region', 'district_ids', 'ages_range', 'device_brand',
                         'device_price', 'business_interest', 'fans_star', 'interest_video', 'app_interest',
                         'app_ids', 'population', 'exclude_population', 'paid_audience', 'behavior_interest']
         for col in cols_to_list:
             group_params_to_request[col] = eval(group_params_to_request[col]) \
                 if ((group_params_to_request[col] is not None) and (group_params_to_request[col] != '')) else None
-        # 3-2 发送请求,需要将 group_name 改为 unit_name
+        # 4-2 发送请求,需要将 group_name 改为 unit_name
         group_params_to_request['unit_name'] = group_params_to_request.pop('group_name')
 
-        # 4 写入数据库的组参数
+        # 5 写入数据库的组参数
         group_params_to_db = group_params.copy()
         group_params_to_db['ai_strategy_uuid'] = self.ai_strategy_uuid
         group_params_to_db['account_id'] = self.account_id
@@ -277,7 +284,7 @@ class ParseAddCampaignOrAddGroupRequest(object):
         # 1 从客户策略表中获取 创意层级基本信息
         creative_cols = ['creative_name', 'action_bar_text', 'description', 'site_id',
                          'click_track_url', 'actionbar_click_url',
-                         'creative_category', 'creative_tag', 'smart_cover', 'asset_mining']
+                         'creative_category', 'creative_tag']
         creative_params = {}
         for col in creative_cols:
             creative_params[col] = self.advertiser_strategy.get(col)
@@ -397,6 +404,7 @@ class ParseAddCampaignOrAddGroupRequest(object):
         # 每5个视频一组,来拼接组层级 和 程序化创意 层级的参数
         self.res_data['group_list'] = []
         cnt = 1
+        is_smart_cover = group_params_to_request.get('smart_cover', 0)
         for i in range(0, len(self.video), 5):
             group_uuid = str(uuid.uuid4())
             group_name = group_params_to_db['group_name'] + '-' + str(cnt)
@@ -431,6 +439,7 @@ class ParseAddCampaignOrAddGroupRequest(object):
                     if self.video[j]['material_type'] == 1:
                         vertical_photo_ids.append(self.video[j]['photo_id'])
                         cover_image_tokens.append(self.video[j]['imageList'][0])
+
                 if len(cover_image_tokens) >= 5:
                     cover_image_tokens = cover_image_tokens[:4]
 
@@ -441,7 +450,7 @@ class ParseAddCampaignOrAddGroupRequest(object):
                 single_creative_params_to_db['package_name'] = package_name
                 single_creative_params_to_db['horizontal_photo_ids'] = str(horizontal_photo_ids)
                 single_creative_params_to_db['vertical_photo_ids'] = str(vertical_photo_ids)
-                single_creative_params_to_db['cover_image_tokens'] = str(cover_image_tokens)
+                single_creative_params_to_db['cover_image_tokens'] = str(cover_image_tokens) if not is_smart_cover else None
                 single_creative_params_to_db['create_time'] = datetime.datetime.now()
                 df = pd.DataFrame.from_dict(single_creative_params_to_db, orient='index').T
                 df.to_sql(name="ctop_ai_kuaishou_program_creative_level_operation_record",
@@ -454,7 +463,9 @@ class ParseAddCampaignOrAddGroupRequest(object):
                 single_creative_params_to_request['package_name'] = package_name
                 single_creative_params_to_request['horizontal_photo_ids'] = horizontal_photo_ids
                 single_creative_params_to_request['vertical_photo_ids'] = vertical_photo_ids
-                single_creative_params_to_request['image_md5s'] = cover_image_tokens
+                if not is_smart_cover:
+                    single_creative_params_to_request['image_md5s'] = cover_image_tokens
+
             else:
                 # material_type=1 竖版
                 # material_type=2 横版
@@ -475,7 +486,7 @@ class ParseAddCampaignOrAddGroupRequest(object):
                 single_creative_params_to_db['package_name'] = package_name
                 single_creative_params_to_db['horizontal_photo_ids'] = str(horizontal_photo_ids)
                 single_creative_params_to_db['vertical_photo_ids'] = str(vertical_photo_ids)
-                single_creative_params_to_db['cover_image_tokens'] = str(cover_image_tokens)
+                single_creative_params_to_db['cover_image_tokens'] = str(cover_image_tokens) if not is_smart_cover else None
                 single_creative_params_to_db['create_time'] = datetime.datetime.now()
                 df = pd.DataFrame.from_dict(single_creative_params_to_db, orient='index').T
                 df.to_sql(name="ctop_ai_kuaishou_program_creative_level_operation_record",
@@ -488,7 +499,8 @@ class ParseAddCampaignOrAddGroupRequest(object):
                 single_creative_params_to_request['package_name'] = package_name
                 single_creative_params_to_request['horizontal_photo_ids'] = horizontal_photo_ids
                 single_creative_params_to_request['vertical_photo_ids'] = vertical_photo_ids
-                single_creative_params_to_request['image_md5s'] = cover_image_tokens
+                if not is_smart_cover:
+                    single_creative_params_to_request['image_md5s'] = cover_image_tokens
 
             single_group_params_to_request['programCreative'] = single_creative_params_to_request
             cnt += 1

+ 3 - 3
ai_time_task_creative_handler.py

@@ -4,7 +4,8 @@ import json
 import logging
 import traceback
 import tornado.web
-from logging.handlers import TimedRotatingFileHandler
+from concurrent_log import ConcurrentTimedRotatingFileHandler
+# from logging.handlers import TimedRotatingFileHandler
 from utils.CommonFunction import get_history_video_info, get_new_video_info, get_missing_video_info, \
                                  get_top_video_info, get_campaign_and_group_name_rule
 from utils.ConstantConfig import *
@@ -12,8 +13,7 @@ from ai_strategy_request_func import ai_strategy_request_parse
 from utils.DataBaseConfig import *
 
 log_formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%m/%d/%Y %I:%M:%S %p')
-log_handler = TimedRotatingFileHandler("logs/ai_auto_create.log",
-                                       when="midnight", backupCount=100)
+log_handler = ConcurrentTimedRotatingFileHandler("logs/ai_auto_create.log", when="midnight", backupCount=100)
 log_handler.setFormatter(log_formatter)
 logger = logging.getLogger('ai_time_task_creative_logger')
 logger.addHandler(log_handler)

+ 0 - 0
logs/.__ai_auto_create.lock


+ 0 - 0
logs/.__ai_callback.lock


+ 0 - 0
logs/.__ai_strategy_request.lock


+ 0 - 0
logs/advertiser_strategy_operate.log


Tiedoston diff-näkymää rajattu, sillä se on liian suuri
+ 184 - 153
logs/ai_auto_create.log


Tiedoston diff-näkymää rajattu, sillä se on liian suuri
+ 140 - 0
logs/ai_callback.log


+ 42 - 0
logs/ai_strategy_request.log

@@ -0,0 +1,42 @@
+02/02/2021 11:59:42 AM INFO 账户信息=9845260, 当前进程=8360, 当前进程的主进程=9536
+02/02/2021 11:59:43 AM INFO the res of create campaign is {'code': 0, 'data': {'campaign_create_time': '2021-02-02 11:59:47', 'account_id': 9845260, 'campaign_id': 33299135}, 'message': 'SUCCESS'}
+02/02/2021 11:59:43 AM INFO 广告计划创建成功,campaign_id = 33299135,  ai_strategy_uuid = 424ad6b4-f67f-4a39-8ad2-e75ed4e4145a
+02/02/2021 12:00:14 PM INFO 策略uuid = 424ad6b4-f67f-4a39-8ad2-e75ed4e4145a 的请求返回信息:{'code': 0, 'message': '异步执行中,等待回调...'}
+02/02/2021 01:55:39 PM INFO 账户信息=9845260, 当前进程=7068, 当前进程的主进程=9536
+02/02/2021 01:55:40 PM INFO the res of create campaign is {'code': 0, 'data': {'campaign_create_time': '2021-02-02 13:55:40', 'account_id': 9845260, 'campaign_id': 33317660}, 'message': 'SUCCESS'}
+02/02/2021 01:55:40 PM INFO 广告计划创建成功,campaign_id = 33317660,  ai_strategy_uuid = 6ab9b3ac-f005-42f4-af2d-3f780a3478f5
+02/02/2021 01:56:44 PM INFO 账户信息=9845260, 当前进程=18716, 当前进程的主进程=9536
+02/02/2021 01:56:45 PM INFO the res of create campaign is {'code': 0, 'data': {'campaign_create_time': '2021-02-02 13:56:44', 'account_id': 9845260, 'campaign_id': 33318186}, 'message': 'SUCCESS'}
+02/02/2021 01:56:45 PM INFO 广告计划创建成功,campaign_id = 33318186,  ai_strategy_uuid = 1fd6f34c-7e73-4424-b3f2-28d7b83a3fca
+02/02/2021 01:56:47 PM INFO 策略uuid = 1fd6f34c-7e73-4424-b3f2-28d7b83a3fca 的请求返回信息:{'code': 0, 'message': '异步执行中,等待回调...'}
+02/02/2021 02:07:44 PM INFO 账户信息=9845260, 当前进程=18716, 当前进程的主进程=9536
+02/02/2021 02:07:46 PM INFO the res of create campaign is {'code': 0, 'data': {'campaign_create_time': '2021-02-02 14:07:46', 'account_id': 9845260, 'campaign_id': 33320595}, 'message': 'SUCCESS'}
+02/02/2021 02:07:46 PM INFO 广告计划创建成功,campaign_id = 33320595,  ai_strategy_uuid = 6ed85b86-e805-4cea-8942-aa7fbc29ebcb
+02/02/2021 02:07:48 PM INFO 策略uuid = 6ed85b86-e805-4cea-8942-aa7fbc29ebcb 的请求返回信息:{'code': 0, 'message': '异步执行中,等待回调...'}
+02/02/2021 02:23:36 PM INFO 账户信息=9845260, 当前进程=8116, 当前进程的主进程=9536
+02/02/2021 02:23:38 PM INFO the res of create campaign is {'code': 0, 'data': {'campaign_create_time': '2021-02-02 14:23:37', 'account_id': 9845260, 'campaign_id': 33327870}, 'message': 'SUCCESS'}
+02/02/2021 02:23:38 PM INFO 广告计划创建成功,campaign_id = 33327870,  ai_strategy_uuid = cdb34066-f8ab-4f37-94b8-24dd342184fd
+02/02/2021 02:25:08 PM INFO 账户信息=9845260, 当前进程=17940, 当前进程的主进程=9536
+02/02/2021 02:25:09 PM INFO the res of create campaign is {'code': 0, 'data': {'campaign_create_time': '2021-02-02 14:25:08', 'account_id': 9845260, 'campaign_id': 33328565}, 'message': 'SUCCESS'}
+02/02/2021 02:25:09 PM INFO 广告计划创建成功,campaign_id = 33328565,  ai_strategy_uuid = e4184104-92c7-4d1b-9912-93d41121852c
+02/02/2021 02:25:11 PM INFO 策略uuid = e4184104-92c7-4d1b-9912-93d41121852c 的请求返回信息:{'code': 0, 'message': '异步执行中,等待回调...'}
+02/02/2021 02:28:00 PM INFO 账户信息=9845260, 当前进程=17940, 当前进程的主进程=9536
+02/02/2021 02:28:01 PM INFO the res of create campaign is {'code': 0, 'data': {'campaign_create_time': '2021-02-02 14:28:01', 'account_id': 9845260, 'campaign_id': 33329272}, 'message': 'SUCCESS'}
+02/02/2021 02:28:01 PM INFO 广告计划创建成功,campaign_id = 33329272,  ai_strategy_uuid = 7ba1863d-7fa1-4393-996c-4bd0d2f4e58d
+02/02/2021 02:28:05 PM INFO 策略uuid = 7ba1863d-7fa1-4393-996c-4bd0d2f4e58d 的请求返回信息:{'code': 0, 'message': '异步执行中,等待回调...'}
+02/02/2021 02:43:00 PM INFO 账户信息=9845260, 当前进程=13992, 当前进程的主进程=9536
+02/02/2021 02:43:00 PM INFO the res of create campaign is {'code': 0, 'data': {'campaign_create_time': '2021-02-02 14:43:00', 'account_id': 9845260, 'campaign_id': 33335107}, 'message': 'SUCCESS'}
+02/02/2021 02:43:00 PM INFO 广告计划创建成功,campaign_id = 33335107,  ai_strategy_uuid = 54e40072-15a0-4bf4-ace1-db01dcb0b6fe
+02/02/2021 02:43:04 PM INFO 策略uuid = 54e40072-15a0-4bf4-ace1-db01dcb0b6fe 的请求返回信息:{'code': 0, 'message': '异步执行中,等待回调...'}
+02/02/2021 02:46:38 PM INFO 账户信息=9845260, 当前进程=13992, 当前进程的主进程=9536
+02/02/2021 02:46:39 PM INFO the res of create campaign is {'code': 0, 'data': {'campaign_create_time': '2021-02-02 14:46:39', 'account_id': 9845260, 'campaign_id': 33338276}, 'message': 'SUCCESS'}
+02/02/2021 02:46:39 PM INFO 广告计划创建成功,campaign_id = 33338276,  ai_strategy_uuid = 2fbb09eb-9fc1-4960-b370-b2e64a6a7b61
+02/02/2021 02:46:41 PM INFO 策略uuid = 2fbb09eb-9fc1-4960-b370-b2e64a6a7b61 的请求返回信息:{'code': 0, 'message': '异步执行中,等待回调...'}
+02/02/2021 02:50:13 PM INFO 账户信息=9845260, 当前进程=13992, 当前进程的主进程=9536
+02/02/2021 02:50:14 PM INFO the res of create campaign is {'code': 0, 'data': {'campaign_create_time': '2021-02-02 14:50:14', 'account_id': 9845260, 'campaign_id': 33340373}, 'message': 'SUCCESS'}
+02/02/2021 02:50:14 PM INFO 广告计划创建成功,campaign_id = 33340373,  ai_strategy_uuid = f9f3e2a1-3829-40bd-9103-c73fb7ad8919
+02/02/2021 02:50:16 PM INFO 策略uuid = f9f3e2a1-3829-40bd-9103-c73fb7ad8919 的请求返回信息:{'code': 0, 'message': '异步执行中,等待回调...'}
+02/02/2021 02:51:05 PM INFO 账户信息=9845260, 当前进程=13992, 当前进程的主进程=9536
+02/02/2021 02:51:06 PM INFO the res of create campaign is {'code': 0, 'data': {'campaign_create_time': '2021-02-02 14:51:06', 'account_id': 9845260, 'campaign_id': 33340487}, 'message': 'SUCCESS'}
+02/02/2021 02:51:06 PM INFO 广告计划创建成功,campaign_id = 33340487,  ai_strategy_uuid = 4d95ce63-96c7-4745-b3cb-361abf6d468b
+02/02/2021 02:51:08 PM INFO 策略uuid = 4d95ce63-96c7-4745-b3cb-361abf6d468b 的请求返回信息:{'code': 0, 'message': '异步执行中,等待回调...'}