Przeglądaj źródła

获取样本量日期信息从配置文件读取

liyuyi@c-top.com.cn 4 lat temu
rodzic
commit
19ed8bf855
2 zmienionych plików z 82 dodań i 77 usunięć
  1. 81 76
      BayesCombine.py
  2. 1 1
      config/config.yaml

+ 81 - 76
BayesCombine.py

@@ -66,11 +66,10 @@ class BayesFeatures(object):
                   sum(activation) activation
            from %s
            where signature = '%s' 
-           and datediff(now(),stat_date)<=30
+           and datediff(now(),stat_date)<= %s
            group by signature,  %s
-           ''' % (self.file_name, self.table, self.signature, self.file_name)
+           ''' % (self.file_name, self.table, self.signature, config['getBaysCombineMaterialFilterRule']['days'], self.file_name)
         df = pd.read_sql(sql, product_engine)
-
         if self.target_type == 'action_ratio':
             # 计算行为率(bclick/aclick)的组合特征值
             df['unbclick'] = df['aclick'] - df['bclick']
@@ -144,8 +143,8 @@ class BayesCombine(object):
         get_sample_and_pct_sql = """
         select sum(aclick) aclick, sum(bclick) bclick, sum(activation) activation  from %s  where
          signature = '%s' 
-        and datediff(now(),stat_date)<=30
-        """ % (config['bayesDim']['age']['table'], self.signature)
+        and datediff(now(),stat_date)<= %s
+        """ % (config['bayesDim']['age']['table'], self.signature, config['getBaysCombineMaterialFilterRule']['days'])
         sample_pct_df = pd.read_sql(get_sample_and_pct_sql, product_engine)
         if self.target_type == 'action_ratio':
             self.sig_pos_pct = sample_pct_df['bclick'].sum() / sample_pct_df['aclick'].sum()
@@ -176,25 +175,29 @@ class BayesCombine(object):
             out_dict['p_value'] = p_value
             out_dict['sample_size'] = self.sample_size
 
-            # TODO 调用人群预估覆盖接口,得到该组合的人群覆盖数 修改advertiser_id 的值 为 account_id
-            # 获取该项目下在投的账号
-            get_acc_sql = """select account_id from ctop_user_allocation where project_id = %s and account_status=0 limit 1""" % \
-                          self.project_id
-            account_df = pd.read_sql(get_acc_sql, product_engine)
-            account_id = account_df['account_id'].values[0]
-            request_data = {'region': city_code_transform(out_dict.get('city')),
-                            'ages_range': age_code_transform(out_dict.get('age')),
-                            'gender': gender_code_transform(out_dict.get('gender')),
-                            'advertiser_id': 9774238}
-
-            request = requests.post(url=estimate_people_number_url,
-                                    headers=headers,
-                                    data=json.dumps(request_data, cls=NpEncoder))
-            response_data = json.loads(request.text)
-            if response_data['code'] == 0:
-                out_dict['crowd_coverage_cnt'] = response_data['data'].get('audience_prediction_num')
-            else:
-                logger.error("人群预估覆盖接口调用报错,请求数据为%s,返回数据为%s" % (str(request_data), str(response_data)))
+            try:
+                # TODO 调用人群预估覆盖接口,得到该组合的人群覆盖数 修改advertiser_id 的值 为 account_id
+                # 获取该项目下在投的账号
+                get_acc_sql = """select account_id from ctop_user_allocation where project_id = %s and account_status=0 limit 1""" % \
+                              self.project_id
+                account_df = pd.read_sql(get_acc_sql, product_engine)
+                account_id = account_df['account_id'].values[0]
+                request_data = {'region': city_code_transform(out_dict.get('city')),
+                                'ages_range': age_code_transform(out_dict.get('age')),
+                                'gender': gender_code_transform(out_dict.get('gender')),
+                                'advertiser_id': 9774238}
+
+                request = requests.post(url=estimate_people_number_url,
+                                        headers=headers,
+                                        data=json.dumps(request_data, cls=NpEncoder))
+                response_data = json.loads(request.text)
+                if response_data['code'] == 0:
+                    out_dict['crowd_coverage_cnt'] = response_data['data'].get('audience_prediction_num')
+                else:
+                    logger.error("人群预估覆盖接口调用报错,请求数据为%s,返回数据为%s" % (str(request_data), str(response_data)))
+            except:
+                logger.error("人群预估覆盖接口调用报错,请求数据为%s,请求地址为%s, 返回数据为%s, 异常信息为%s" %
+                             (str(request_data), estimate_people_number_url, str(response_data), traceback.format_exc()))
 
             out_dict['combine_estimate_prob'] = prob
             out_dict['actual_prob'] = self.actual_prob
@@ -214,55 +217,55 @@ class BayesCombine(object):
 
 
 if __name__ == '__main__':
-    try:
-        # 1、读取配置文件
-        with open('config/config.yaml', mode='r', encoding='utf-8') as f:
-            config = yaml.load(f.read(), Loader=yaml.FullLoader)
-
-        # 1-1 数据库连接引擎,依据开发环境/生产环境 进行切换
-        if os.getenv('LYY_DEV', 'unknown') == 'dev':
-            engine = get_db_engine(config['devDB'])
-        else:
-            engine = get_db_engine(config['productDB'])
-
-        product_engine = get_db_engine(config['productDB'])
-
-        # 2、 参与定向组合的维度
-        target_dim = [key for key in config['bayesDim'].keys() if config['bayesDim'][key]['isOn']]
-
-        # 4、计算贝叶斯组合入库
-        for project_id in config['projectId']:
-            # 4.1 获取指定项目下当前活跃的素材信息, 如近3天内累计激活个数达到50个(开发环境或者生产环境,这部分都读取生产数据库)
-            sql = '''
-               select signature from ctop_kuaishou_report_daily_material 
-               where account_id in (select account_id from ctop_user_allocation where project_id = %s)
-               and datediff(now(),stat_date) <= %s
-               group by signature
-               having sum(activation) >= %s
-               ''' % (project_id,
-                      config['activeMaterialFilterRule']['days'],
-                      config['activeMaterialFilterRule']['activation'])
-            df = pd.read_sql(sql, product_engine)
-            active_signatures = df[~df.signature.isnull()].signature.values
-
-            # 4.2 在素材人群报表筛选里面近一个月累计100个激活
-            sql = """
-            select signature from ctop_kuaishou_audience_report_daily_age_material  
-            where signature in %s
-             and datediff(now(),stat_date) <= %s
-               group by signature
-               having sum(activation) >= %s
-            """ % (tuple(active_signatures),
-                   config['getBaysCombineMaterialFilterRule']['days'],
-                   config['getBaysCombineMaterialFilterRule']['activation'])
-            df = pd.read_sql(sql, product_engine)
-            signature_lst = df['signature'].values
-
-            # 这行代码用于测试
-            # signature_lst = ['0070efb7557b2a04cf3d4a6f243c3cd8', '03b93728f0d82ea7865c3c7cf632bc1b']
-
-            # 4.2 计算指定素材的贝叶斯特征
-            for sig in signature_lst:
+    # 1、读取配置文件
+    with open('config/config.yaml', mode='r', encoding='utf-8') as f:
+        config = yaml.load(f.read(), Loader=yaml.FullLoader)
+
+    # 1-1 数据库连接引擎,依据开发环境/生产环境 进行切换
+    if os.getenv('LYY_DEV', 'unknown') == 'dev':
+        engine = get_db_engine(config['devDB'])
+    else:
+        engine = get_db_engine(config['productDB'])
+
+    product_engine = get_db_engine(config['productDB'])
+
+    # 2、 参与定向组合的维度
+    target_dim = [key for key in config['bayesDim'].keys() if config['bayesDim'][key]['isOn']]
+
+    # 4、计算贝叶斯组合入库
+    for project_id in config['projectId']:
+        # 4.1 获取指定项目下当前活跃的素材信息, 如近3天内累计激活个数达到50个(开发环境或者生产环境,这部分都读取生产数据库)
+        sql = '''
+           select signature from ctop_kuaishou_report_daily_material 
+           where account_id in (select account_id from ctop_user_allocation where project_id = %s)
+           and datediff(now(),stat_date) <= %s
+           group by signature
+           having sum(activation) >= %s
+           ''' % (project_id,
+                  config['activeMaterialFilterRule']['days'],
+                  config['activeMaterialFilterRule']['activation'])
+        df = pd.read_sql(sql, product_engine)
+        active_signatures = df[~df.signature.isnull()].signature.values
+
+        # 4.2 在素材人群报表筛选里面近一个月累计100个激活
+        sql = """
+        select signature from ctop_kuaishou_audience_report_daily_age_material  
+        where signature in %s
+         and datediff(now(),stat_date) <= %s
+           group by signature
+           having sum(activation) >= %s
+        """ % (tuple(active_signatures),
+               config['getBaysCombineMaterialFilterRule']['days'],
+               config['getBaysCombineMaterialFilterRule']['activation'])
+        df = pd.read_sql(sql, product_engine)
+        signature_lst = df['signature'].values
+
+        # 这行代码用于测试
+        # signature_lst = ['4a56bc00ce51420f3a460ab51c6f5110']
+
+        # 4.2 计算指定素材的贝叶斯特征
+        for sig in signature_lst:
+            try:
                 for t_type in config['targetType']:
                     bayes_feature_lst = []
                     for dimension in target_dim:
@@ -276,6 +279,8 @@ if __name__ == '__main__':
                     cls = BayesCombine(sig, project_id, t_type, bayes_feature_lst)
                     cls.get_bayes_estimate()
                     cls.write_to_db()
-            logger.info('project_id is %s 完成贝叶斯组合计算!' % project_id)
-    except:
-        logger.error("traceback is %s" % (traceback.format_exc()))
+                logger.info('project_id=%s, signature= %s 完成贝叶斯组合计算!' % (project_id, sig))
+            except:
+                logger.error('project_id=%s, signature=%s 贝叶斯组合计算出错,异常信息为%s!' % (project_id, sig, traceback.format_exc()))
+
+        logger.info('project_id is %s 完成贝叶斯组合计算!' % project_id)

+ 1 - 1
config/config.yaml

@@ -68,7 +68,7 @@ activeMaterialFilterRule:
   days: 7
   activation: 100
 
-# 计算推荐组合的素材筛选规则:近个月内累计达到100个
+# 计算推荐组合的素材筛选规则:近个月内累计达到100个
 getBaysCombineMaterialFilterRule:
   days: 60
   activation: 100