liyuyi@c-top.com.cn 4 år sedan
förälder
incheckning
911aee1049
6 ändrade filer med 317 tillägg och 80 borttagningar
  1. 3 0
      .idea/ai_target.iml
  2. 55 65
      BayesCombine.py
  3. 130 12
      ai_target_combine_handler.py
  4. 100 0
      commonFunc.py
  5. 16 3
      config/config.yaml
  6. 13 0
      config/url.py

+ 3 - 0
.idea/ai_target.iml

@@ -6,4 +6,7 @@
     <orderEntry type="sourceFolder" forTests="false" />
     <orderEntry type="module" module-name="ai_ads" />
   </component>
+  <component name="PyDocumentationSettings">
+    <option name="renderExternalDocumentation" value="true" />
+  </component>
 </module>

+ 55 - 65
BayesCombine.py

@@ -1,11 +1,12 @@
 import yaml
+import os
 import pandas as pd
-from sqlalchemy import create_engine
-from urllib import parse
-from commonFunc import update_dict
+import numpy as np
+from commonFunc import update_dict, get_db_engine
 from functools import reduce
 from itertools import product
 from datetime import datetime
+from statsmodels.stats.proportion import proportions_ztest
 
 
 class BayesFeatures(object):
@@ -52,7 +53,7 @@ class BayesFeatures(object):
            -- and datediff(now(),stat_date)<=30
            group by signature,  %s
            ''' % (self.file_name, self.table, self.signature, self.file_name)
-        df = pd.read_sql(sql, test_engine)
+        df = pd.read_sql(sql, engine)
 
         if self.target_type == 'action_ratio':
             # 计算行为率(bclick/aclick)的组合特征值
@@ -61,6 +62,7 @@ class BayesFeatures(object):
             sig_neg_pct = 1 - sig_pos_pct  # 素材曝光-未点击的概率
             self.bayes_feature['sig_pos_pct'] = sig_pos_pct
             self.bayes_feature['sig_neg_pct'] = sig_neg_pct
+            self.bayes_feature['sample_size'] = df['aclick'].sum()  # 素材曝光量,后续计算显著差异用
             for sub_combine in self.window_combine:
                 if sub_combine != ['不限']:
                     key = self.dim + '_' + '|'.join(sub_combine)
@@ -89,6 +91,7 @@ class BayesFeatures(object):
             sig_neg_pct = 1 - sig_pos_pct  # 素材点击-未转化的概率
             self.bayes_feature['sig_pos_pct'] = sig_pos_pct
             self.bayes_feature['sig_neg_pct'] = sig_neg_pct
+            self.bayes_feature['sample_size'] = df['bclick'].sum()  # 素材行为量,后续计算显著差异用
             for sub_combine in self.window_combine:
                 if sub_combine != '不限':
                     key = self.dim + '_' + '|'.join(sub_combine)
@@ -115,18 +118,24 @@ class BayesCombine(object):
     """
     依据特征值,计算多维度的组合预估值
     """
+
     def __init__(self, signature, project_id, target_type, dim_features):
         self.signature = signature
         self.project_id = project_id
         self.target_type = target_type
         self.dim_features = dim_features
         self.bayes_combine_df = pd.DataFrame()
-        self.actual_prob = self.dim_features[0]['sig_pos_pct']
+        self.actual_prob = None
+        self.sample_size = None
 
     def get_bayes_estimate(self):
-        dim_combine_lst = [[key for key in fea.keys() if key not in ['sig_pos_pct', 'sig_neg_pct']] for fea in self.dim_features]
+        dim_combine_lst = [[key for key in fea.keys() if key not in ['sig_pos_pct', 'sig_neg_pct', 'sample_size']]
+                           for fea in self.dim_features]
         self.dim_features = reduce(update_dict, self.dim_features)
-        
+
+        self.actual_prob = self.dim_features['sig_pos_pct']
+        self.sample_size = self.dim_features['sample_size']
+
         for ele in product(*dim_combine_lst):
             pos_pct_lst = [self.dim_features[key]['pos_pct'] for key in ele]
             prob_pos = reduce(lambda x, y: x * y, pos_pct_lst)
@@ -137,6 +146,14 @@ class BayesCombine(object):
             prob = (prob_pos * self.dim_features['sig_pos_pct']) / (prob_neg * self.dim_features['sig_neg_pct'])
             out_dict = dict(zip([e.split('_')[0] for e in ele], [e.split('_')[-1] for e in ele]))
 
+            # 计算该组合的概率值 与 实际投放的概率值 是否存在显著性差异:显著好、 显著差
+            count = np.array([self.sample_size * prob, self.sample_size * self.actual_prob])
+            nobs = np.array([self.sample_size, self.sample_size])
+            z_score, p_value = proportions_ztest(count=count, nobs=nobs, value=None, alternative='two-sided', prop_var=False)
+            out_dict['z_score'] = z_score
+            out_dict['p_value'] = p_value
+            out_dict['sample_size'] = self.sample_size
+
             # TODO 调用人群预估覆盖接口,得到该组合的人群覆盖数
             out_dict['crowd_coverage_cnt'] = 1000
 
@@ -152,7 +169,7 @@ class BayesCombine(object):
 
     def write_to_db(self):
         self.bayes_combine_df.to_sql(name="ctop_ai_kuaishou_signature_recommended_target_combine",
-                                     con=test_engine,
+                                     con=engine,
                                      if_exists='append',
                                      index=False)
 
@@ -161,47 +178,52 @@ if __name__ == '__main__':
     # 1、读取配置文件
     with open('config/config.yaml', mode='r', encoding='utf-8') as f:
         config = yaml.load(f.read(), Loader=yaml.FullLoader)
-        project_ids = config['projectId']
-        target_types = config['targetType']
-        online_db = config['onlineDB']
-        test_db = config['testDB']
-        material_rule = config['materialRule']
-        bayes_dim = config['bayesDim']
-
-    # 2、数据库连接引擎
-    db_con_str = 'mysql+pymysql://%s:%s@%s:%d/%s' % \
-                 (online_db['username'], parse.quote_plus(online_db['password']), online_db['host'], online_db['port'],
-                  online_db['database'])
-    engine = create_engine(db_con_str, connect_args={'charset': 'utf8'})
-
-    db_con_str = 'mysql+pymysql://%s:%s@%s:%d/%s' % \
-                 (test_db['username'], parse.quote_plus(test_db['password']), test_db['host'], test_db['port'],
-                  test_db['database'])
-    test_engine = create_engine(db_con_str, connect_args={'charset': 'utf8'})
-
-    # 3、 参与定向组合的维度
-    target_dim = [key for key in bayes_dim.keys() if bayes_dim[key]['isOn']]
+
+    # 数据库连接引擎,依据开发环境/生产环境 进行切换
+    if os.getenv('environment', 'unknown') == 'dev':
+        engine = get_db_engine(config['devDB'])
+    else:
+        engine = get_db_engine(config['productDB'])
+
+    # 2、 参与定向组合的维度
+    target_dim = [key for key in config['bayesDim'].keys() if config['bayesDim'][key]['isOn']]
 
     # 4、计算贝叶斯组合入库
-    for project_id in project_ids:
-        # 4.1 获取指定项目下当前活跃的素材信息, 如近3天内累计激活个数达到50个
+    for project_id in config['projectId']:
+        # 4.1 获取指定项目下当前活跃的素材信息, 如近3天内累计激活个数达到50个(开发环境或者生产环境,这部分都读取生产数据库)
+        product_engine = get_db_engine(config['productDB'])
         sql = '''
            select signature from ctop_kuaishou_report_daily_material 
            where account_id in (select account_id from ctop_user_allocation where project_id = %s)
            and datediff(now(),stat_date) <= %s
            group by signature
            having sum(activation) >= %s
-           ''' % (project_id, material_rule['days'], material_rule['activation'])
+           ''' % (project_id,
+                  config['activeMaterialFilterRule']['days'],
+                  config['activeMaterialFilterRule']['activation'])
+        df = pd.read_sql(sql, product_engine)
+        active_signatures = df[~df.signature.isnull()].signature.values
+
+        # 4.2 在素材人群报表筛选里面近一个月累计100个激活 TODO 等生产的素材人群表存在之后,修改表名,添加 days
+        sql = """
+        select signature from ctop_kuaishou_audience_daily_report_by_signature_age  
+        where signature in %s
+           group by signature
+           having sum(activation) >= %s
+        """ % (tuple(active_signatures),
+               config['getBaysCombineMaterialFilterRule']['activation'])
         df = pd.read_sql(sql, engine)
         signature_lst = df['signature'].values
+
+        # TODO 这行代码用于测试
         signature_lst = ['0070efb7557b2a04cf3d4a6f243c3cd8', '03b93728f0d82ea7865c3c7cf632bc1b']
 
         # 4.2 计算指定素材的贝叶斯特征
         for sig in signature_lst:
-            for t_type in target_types:
+            for t_type in config['targetType']:
                 bayes_feature_lst = []
                 for dimension in target_dim:
-                    cls = BayesFeatures(sig, t_type, dimension, bayes_dim[dimension])
+                    cls = BayesFeatures(sig, t_type, dimension, config['bayesDim'][dimension])
                     # 计算滑窗组合
                     cls.get_window_combine()
                     cls.get_bayes_feature()
@@ -211,35 +233,3 @@ if __name__ == '__main__':
                 cls = BayesCombine(sig, project_id, t_type, bayes_feature_lst)
                 cls.get_bayes_estimate()
                 cls.write_to_db()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-

+ 130 - 12
ai_target_combine_handler.py

@@ -1,5 +1,16 @@
+import uuid
+import datetime
 import tornado
 import json
+import pandas as pd
+import os
+import yaml
+import requests
+from commonFunc import get_db_engine, is_contains_gender, is_contains_age
+from config.url import create_campaign_url, create_group_and_creative_url, headers
+
+with open('config/config.yaml', mode='r', encoding='utf-8') as f:
+    config = yaml.load(f.read(), Loader=yaml.FullLoader)
 
 
 class AiTargetCombine(tornado.web.RequestHandler):
@@ -10,7 +21,6 @@ class AiTargetCombine(tornado.web.RequestHandler):
         data = self.request.body
         data = str(data, 'utf8')
         data = json.loads(data, encoding='utf8')
-        # logger.info("call back of add group, the raw data from request is %s" % data)
 
         try:
             pass
@@ -23,26 +33,112 @@ class AiTargetCombine(tornado.web.RequestHandler):
 
 
 class GetTargetAndAssemblyParameters(object):
-    def __init__(self):
-        pass
+    def __init__(self, account_id, project_id):
+        self.account_id = account_id
+        self.project_id = project_id
+        self.engine = None
+        self.get_database_engine()
+        self.signature_target_combine = None
+        self.advertiser_strategy_id = None
+        self.advertiser_strategy = {}
+        self.ai_strategy_uuid = None
+
+    def get_database_engine(self):
+        # 数据库连接引擎,依据开发环境/生产环境 进行切换
+        if os.getenv('environment', 'unknown') == 'dev':
+            self.engine = get_db_engine(config['devDB'])
+        else:
+            self.engine = get_db_engine(config['productDB'])
+
+    def get_advertiser_strategy_info(self):
+        """
+        获取指定账户下客户的策略信息
+        为后续的写入库表,校验一致性做准备
+        """
+        sql = """
+                select * 
+                from ctop_ai_kuaishou_advertiser_strategy 
+                where account_id = %d
+                limit 1
+                """ % self.account_id
+        advertiser_strategy_df = pd.read_sql(sql, self.engine)
+        self.advertiser_strategy_id = int(advertiser_strategy_df['id'].values[0])
+        self.advertiser_strategy = advertiser_strategy_df.T.to_dict()[0]
 
     def add_campaign(self):
         """
-        新增广告计划,如果存在则跳过
+        新增广告计划,如果存在则跳过。
+        response_data = {
+            "code": 0,
+            "data": {
+                "account_id": 23212,
+                "campaign_id": 30956133,
+                "campaign_create_time": "2021-01-13 13:50:16"
+            },
+            "message": "SUCCESS"}
         """
-        pass
+        # 1、请求创建广告计划的接口
+        create_campaign_request_data = {'account_id': self.account_id,
+                                        'campaign_name': self.advertiser_strategy['campaign_name'] + '_优质定向',
+                                        'type': self.advertiser_strategy['campaign_type']}
+
+        request = requests.post(url=create_campaign_url,
+                                headers=headers,
+                                data=json.JSONEncoder().encode(create_campaign_request_data))
+        response_data = json.loads(request.text)
+
+        # 2、对接口返回的结果处理
+        campaign_info_to_db = {
+            'campaign_uuid': str(uuid.uuid4()),
+            'account_id': self.account_id,
+            'ai_strategy_uuid': self.ai_strategy_uuid,
+            'campaign_name': self.advertiser_strategy['campaign_name'],
+            'campaign_type': self.advertiser_strategy['campaign_type'],
+            'create_time': datetime.datetime.now()
+        }
+        res = 0
+        if response_data['code'] == 0:
+            campaign_info_to_db['campaign_id'] = response_data['data'].get('campaign_id', None)
+            campaign_info_to_db['campaign_create_time'] = response_data['data'].get('campaign_create_time', None)
+            campaign_info_to_db['status'] = response_data.get('code', None)
+            campaign_info_to_db['message'] = response_data.get('message', None)
+
+        else:
+            campaign_info_to_db['message'] = response_data['message']
+            campaign_info_to_db['status'] = response_data['code']
+            res = -1
+
+        # 3、写入计划层级的操作表
+        df = pd.DataFrame.from_dict(campaign_info_to_db, orient='index').T
+        df.to_sql(name="ctop_ai_kuaishou_campaign_level_operation_record",
+                  con=self.engine,
+                  if_exists='append',
+                  index=False)
+        return res
 
     def get_signature_and_target(self):
         """
         从 ctop_ai_kuaishou_signature_recommended_target_combine 表中读取素材和对应的定向
         """
-        pass
+        sql = """
+        select * from ctop_ai_kuaishou_signature_recommended_target_combine where project_id = %s and stat_date = curdate()
+        """ % self.project_id
+        df = pd.read_sql(sql, self.engine)
+        # 计算组合的概率,相对于实际投放概率高出了百分之多少
+        df['improve_ratio'] = (df['combine_estimate_prob'] - df['actual_prob']) / df['actual_prob']
 
-    def get_advertiser_strategy_info(self):
-        """
-        从 ctop_ai_kuaishou_advertiser_strategy 读取账户的基本配置信息
-        """
-        pass
+        # 两种类型的过滤标准不一样,分开进行判断,然后对结果进行合并
+        df_1 = df[(df['target_type'] == 'action_ratio') &
+                  (df['improve_ratio'] >= config['filterTargetCombine']['actionRatio']['improveRatio']) &
+                  (df['sample_size'] >= config['filterTargetCombine']['actionRatio']['sampleSize'])]
+
+        df_2 = df[(df['target_type'] == 'convertRatio') &
+                  (df['improve_ratio'] >= config['filterTargetCombine']['convertRatio']['improveRatio']) &
+                  (df['sample_size'] >= config['filterTargetCombine']['convertRatio']['sampleSize'])]
+
+        merge_df = pd.concat([df_1, df_2], axis=0)
+        self.signature_target_combine = merge_df[['signature', 'age', 'gender', 'city', 'business', 'province', 'client']]. \
+            to_dict(orient='records')
 
     def write_intelligence_strategy_table(self):
         """
@@ -53,7 +149,30 @@ class GetTargetAndAssemblyParameters(object):
     def get_target_intersection(self):
         """
         两个表中读取的定向取交集,还是子集?
+        age	struct	可选	自定义年龄段	不传值表示不限,传值具体见下方表格;与ages_range不能同时传
+        age	字段	类型	是否必填	说明	         备注
+            min	int	必填	    年龄最小限制	年龄区间最小为18岁
+            max	int	必填	    年龄最大限制	年龄区间最大为55岁,且年龄最大限制须大于等于年龄最小限制
+
+        ages_range	int[]	可选	固定年龄段	与age不能同时传;【18:表示18-23岁】;【24:表示24-30岁】;【31:表示31-40岁】;【41:表示41-49岁】;【50:表示50-100岁】
+        ctop_ai_kuaishou_signature_recommended_target_combine 表中的 city 和 province 都对应 self.advertiser_strategy 的 region []  varchar
+        age 对应 ages_range []  varchar
+        client 对应 platform_os int
+        business_interest 对应 business_interest varchar
+        gender 对应 gender int
+
         """
+        for item in self.signature_target_combine:
+            gender_bool, gender_dict = is_contains_age(item['gender'], self.advertiser_strategy['gender'])
+            age_bool, age_dict = is_contains_age(item['age'], self.advertiser_strategy['age'])
+
+            if gender_bool & age_bool:
+                target_combine = {'signature': item['signature']}
+                target_combine.update(gender_dict)
+                target_combine.update(age_dict)
+            else:
+                pass
+
         pass
 
     def assembly_group_and_creative_params(self):
@@ -61,4 +180,3 @@ class GetTargetAndAssemblyParameters(object):
         拼接参数,请求创建接口
         """
         pass
-

+ 100 - 0
commonFunc.py

@@ -1,5 +1,105 @@
+from sqlalchemy import create_engine
+from urllib import parse
+import pandas as pd
+import yaml
+
+with open('config/config.yaml', mode='r', encoding='utf-8') as f:
+    config = yaml.load(f.read(), Loader=yaml.FullLoader)
+
+
 def update_dict(x, y):
     # 用字典y,来更新x
     # key 存在则更新,不存在则新增
     x.update(y)
     return x
+
+
+def get_db_engine(db_info):
+    db_con_str = 'mysql+pymysql://%s:%s@%s:%d/%s' % \
+                 (db_info['username'],
+                  parse.quote_plus(db_info['password']),
+                  db_info['host'],
+                  db_info['port'],
+                  db_info['database'])
+    engine = create_engine(db_con_str, connect_args={'charset': 'utf8'})
+    return engine
+
+
+def is_contains_region(val1, val2):
+    """
+    level1
+    region	long[]	可选	地域  传值为[]表示不限;传递上一级ID时,childrenID可以不传;不允许同时传parentID和childrenID;
+                                仅计划的campaign_type为5时,支持设置三级地域(例:山西-大同-左云,左云是三级地域)
+
+    :param val1:
+    :param val2:
+    :return:
+    """
+    is_contains = False
+    val = None
+    city_lst = []
+    sql = """
+    select t1.city_name, t2.level, t2.region_id from ctop_kuaishou_city_level t1
+    left join ctop_kuaishou_region_list_parent t2
+    on t1.city_name = t2.name
+    where t1.city_level in %s
+    """ % (tuple(val1.split('|')),)
+    city_df = pd.read_sql(sql, None)
+
+    # level1
+    if city_df[city_df.level == 1].region_id.values in eval(val2):
+        val.extend(city_df[city_df.level == 1].region_id.values)
+    else:
+        return False, -1
+
+    # level2
+    while not is_contains:
+        pass
+
+
+    # level3
+    pass
+
+
+
+def is_contains_gender(val1, val2):
+    """
+    :param val1: ctop_ai_kuaishou_signature_recommended_target_combine 中的 gender 字段: '男'、'女'、'不限'、None
+    :param val2: ctop_ai_kuaishou_advertiser_strategy 中的 gender 字段: 1:女性, 2:男性,0表示不限
+    :return: val1 是否为 val2的子集,以及传递给快手后台的值
+    """
+    is_contains = False
+    val = None
+    if val1 == '男' and val2 in (2, 0):
+        is_contains = True
+        val = val1
+    if val1 == '女' and val2 in (1, 0):
+        is_contains = True
+        val = val1
+    if val1 == '不限' and val2 == 0:
+        is_contains = True
+        val = val2
+
+    # 如果val1为空,表示该字段没有参与定向组合运算,就直接取客户投放策略里面的值
+    if val1 is None:
+        is_contains = True
+        val = val2
+
+    return is_contains, {'gender': val}
+
+
+def is_contains_platform_os(val1, val2):
+    """
+    :param val1: ctop_ai_kuaishou_signature_recommended_target_combine 中的 client 字段: '男'、'女'、'不限'、None
+    :param val2:
+    :return:
+    """
+    pass
+
+
+def is_contains_age(val1, val2):
+    pass
+
+
+def is_contains_business_interest(val1, val2):
+    pass

+ 16 - 3
config/config.yaml

@@ -5,14 +5,14 @@ targetType:
   - 'action_ratio'
   - 'convert_ratio'
 
-onlineDB:
+productDB:
   host: 139.186.27.96
   username: data
   password: hcst@2021
   port: 3390
   database: jeecg-boot
 
-testDB:
+devDB:
   host: 139.186.165.84
   username: hcst
   password: hcst@2020
@@ -62,12 +62,25 @@ bayesDim:
         fieldName: 'city'
 
 
-materialRule:
+# 活跃素材筛选规则:近3天累计激活个数达到100个
+activeMaterialFilterRule:
   days: 3
   activation: 100
 
+# 计算推荐组合的素材筛选规则:近一个月内累计达到100个
+getBaysCombineMaterialFilterRule:
+  days: 30
+  activation: 100
+
 
 
+filterTargetCombine:
+    actionRatio:
+        sampleSize: 10000
+        improveRatio: 0.2
+    convertRatio:
+        sampleSize: 10000
+        improveRatio: 0.2
 
 
 

+ 13 - 0
config/url.py

@@ -0,0 +1,13 @@
+import os
+
+headers = {'Content-Type': 'application/json'}
+
+os_env = os.getenv('environment', 'unknown')
+product_url = 'http://api.tjyourong.com.cn/jeecg-boot/'
+dev_url = 'http://192.168.1.8:8080/jeecg-boot/'
+
+# 创建计划url
+create_campaign_url = dev_url if os_env == 'dev' else product_url + 'kuaishou/create/campaignCreate'
+
+# 创建组和创意url
+create_group_and_creative_url = dev_url if os_env == 'dev' else product_url + 'kuaishou/create/createUnitAndCreative'