|
@@ -1,26 +1,28 @@
|
|
|
import datetime
|
|
|
import json
|
|
|
-import logging
|
|
|
+import os
|
|
|
+import sys
|
|
|
import traceback
|
|
|
-import uuid
|
|
|
from itertools import product
|
|
|
|
|
|
import pandas as pd
|
|
|
import requests
|
|
|
import yaml
|
|
|
-from concurrent_log import ConcurrentTimedRotatingFileHandler
|
|
|
+from loguru import logger
|
|
|
+
|
|
|
+curr_path = os.path.abspath(os.path.dirname(__file__))
|
|
|
+project_root_path = curr_path[:curr_path.find("ai_target") + len("ai_target")]
|
|
|
+sys.path.append(project_root_path)
|
|
|
|
|
|
from config.url_and_db import estimate_people_number_url, headers, jeecg_db, jeecg_product_db, application_product_db
|
|
|
from utils.BaseClass import NpEncoder
|
|
|
-from utils.commonFunc import get_db_engine, age_code_transform, gender_code_transform
|
|
|
+from utils.commonFunc import age_code_transform, gender_code_transform
|
|
|
|
|
|
-log_formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%Y/%m/%d %I:%M:%S %p')
|
|
|
-log_handler = ConcurrentTimedRotatingFileHandler("/data/pythonProject/ai_target/logs/BayesCombine.log", when="midnight", backupCount=100)
|
|
|
-log_handler.setFormatter(log_formatter)
|
|
|
-logger = logging.getLogger('bayes_combine_logger')
|
|
|
-logger.addHandler(log_handler)
|
|
|
-logger.setLevel(logging.DEBUG)
|
|
|
-print('id of bayes_combine_logger %s' % id(logger))
|
|
|
+logger.remove() # 删去 import logger之后自动产生的handler,不删除的话会出现重复输出的现象
|
|
|
+logger.add("/data/pythonProject/ai_target/logs/bayes_combine.{time:YYYY-MM-DD}.log",
|
|
|
+ rotation="00:00",
|
|
|
+ format="{time:YYYY-MM-DD HH:mm:ss,SSS} [{process}] [{thread}] {level} {file} {line} - {message}",
|
|
|
+ level="INFO")
|
|
|
|
|
|
|
|
|
class GetSingleDimFeatures(object):
|
|
@@ -129,102 +131,6 @@ class GetCompositeDimFeatures(object):
|
|
|
index=False)
|
|
|
|
|
|
|
|
|
-# class BayesCombine(object):
|
|
|
-# """
|
|
|
-# 依据特征值,计算多维度的组合预估值
|
|
|
-# """
|
|
|
-#
|
|
|
-# def __init__(self, signature, project_id, target_type, dim_features):
|
|
|
-# self.signature = signature
|
|
|
-# self.project_id = project_id
|
|
|
-# self.target_type = target_type
|
|
|
-# self.dim_features = dim_features
|
|
|
-# self.bayes_combine_df = pd.DataFrame()
|
|
|
-# self.actual_prob = None
|
|
|
-# self.sample_size = None
|
|
|
-# self.sig_pos_pct = None
|
|
|
-# self.sig_neg_pct = None
|
|
|
-#
|
|
|
-# def get_bayes_estimate(self):
|
|
|
-# dim_combine_lst = [[key for key in fea.keys()] for fea in self.dim_features]
|
|
|
-# self.dim_features = reduce(update_dict, self.dim_features)
|
|
|
-#
|
|
|
-# # 通过 年龄对应的表: ctop_kuaishou_audience_daily_report_by_signature_age 来获取样本量,和正负比例值(其他表的数据可能会丢失导致不全)
|
|
|
-# get_sample_and_pct_sql = """
|
|
|
-# select sum(aclick) aclick, sum(bclick) bclick, sum(activation) activation from %s where
|
|
|
-# signature = '%s'
|
|
|
-# """ % (config['bayesDim']['age']['table'], self.signature)
|
|
|
-# sample_pct_df = pd.read_sql(get_sample_and_pct_sql, product_engine)
|
|
|
-# if self.target_type == 'action_ratio':
|
|
|
-# self.sig_pos_pct = sample_pct_df['bclick'].sum() / sample_pct_df['aclick'].sum()
|
|
|
-# self.sig_neg_pct = 1 - self.sig_pos_pct
|
|
|
-# self.actual_prob = self.sig_pos_pct
|
|
|
-# self.sample_size = int(sample_pct_df.aclick.sum())
|
|
|
-# elif self.target_type == 'convert_ratio':
|
|
|
-# self.sig_pos_pct = sample_pct_df['activation'].sum() / sample_pct_df['bclick'].sum()
|
|
|
-# self.sig_neg_pct = 1 - self.sig_pos_pct
|
|
|
-# self.actual_prob = self.sig_pos_pct
|
|
|
-# self.sample_size = int(sample_pct_df.bclick.sum())
|
|
|
-#
|
|
|
-# for ele in product(*dim_combine_lst):
|
|
|
-# pos_pct_lst = [self.dim_features[key]['pos_pct'] for key in ele]
|
|
|
-# prob_pos = reduce(lambda x, y: x * y, pos_pct_lst)
|
|
|
-#
|
|
|
-# neg_pct_lst = [self.dim_features[key]['neg_pct'] for key in ele]
|
|
|
-# prob_neg = reduce(lambda x, y: x * y, neg_pct_lst)
|
|
|
-#
|
|
|
-# prob = (prob_pos * self.sig_pos_pct) / (prob_neg * self.sig_neg_pct)
|
|
|
-# out_dict = dict(zip([e.split('_')[0] for e in ele], [e.split('_')[-1] for e in ele]))
|
|
|
-#
|
|
|
-# # 计算该组合的概率值 与 实际投放的概率值 是否存在显著性差异:显著好、 显著差
|
|
|
-# count = np.array([self.sample_size * prob, self.sample_size * self.actual_prob])
|
|
|
-# nobs = np.array([self.sample_size, self.sample_size])
|
|
|
-# z_score, p_value = proportions_ztest(count=count, nobs=nobs, value=None, alternative='two-sided', prop_var=False)
|
|
|
-# out_dict['z_score'] = z_score
|
|
|
-# out_dict['p_value'] = p_value
|
|
|
-# out_dict['sample_size'] = self.sample_size
|
|
|
-#
|
|
|
-# try:
|
|
|
-# # 获取该项目下在投的账号
|
|
|
-# get_acc_sql = """select account_id from ctop_user_allocation where project_id = %s and account_status=0 limit 1""" % \
|
|
|
-# self.project_id
|
|
|
-# account_df = pd.read_sql(get_acc_sql, product_engine)
|
|
|
-# account_id = account_df['account_id'].values[0]
|
|
|
-# account_id = 9774238
|
|
|
-# request_data = {'region': city_code_transform(out_dict.get('city')),
|
|
|
-# 'ages_range': age_code_transform(out_dict.get('age')),
|
|
|
-# 'gender': gender_code_transform(out_dict.get('gender')),
|
|
|
-# 'advertiser_id': account_id}
|
|
|
-#
|
|
|
-# request = requests.post(url=estimate_people_number_url,
|
|
|
-# headers=headers,
|
|
|
-# data=json.dumps(request_data, cls=NpEncoder))
|
|
|
-# response_data = json.loads(request.text)
|
|
|
-# if response_data['code'] == 0:
|
|
|
-# out_dict['crowd_coverage_cnt'] = response_data['data'].get('audience_prediction_num')
|
|
|
-# else:
|
|
|
-# logger.error("人群预估覆盖接口调用报错,请求数据为%s,返回数据为%s" % (str(request_data), str(response_data)))
|
|
|
-# except:
|
|
|
-# logger.error("人群预估覆盖接口调用报错,请求数据为%s,请求地址为%s, 返回数据为%s, 异常信息为%s" %
|
|
|
-# (str(request_data), estimate_people_number_url, str(response_data), traceback.format_exc()))
|
|
|
-#
|
|
|
-# out_dict['combine_estimate_prob'] = prob
|
|
|
-# out_dict['actual_prob'] = self.actual_prob
|
|
|
-# out_dict['signature'] = self.signature
|
|
|
-# out_dict['project_id'] = self.project_id
|
|
|
-# out_dict['target_type'] = self.target_type
|
|
|
-# out_dict['stat_date'] = str(datetime.now().date())
|
|
|
-#
|
|
|
-# combine = pd.DataFrame([out_dict])
|
|
|
-# self.bayes_combine_df = self.bayes_combine_df.append(combine, ignore_index=True)
|
|
|
-#
|
|
|
-# def write_to_db(self):
|
|
|
-# self.bayes_combine_df.to_sql(name="ctop_ai_kuaishou_signature_recommended_target_combine",
|
|
|
-# con=engine,
|
|
|
-# if_exists='append',
|
|
|
-# index=False)
|
|
|
-
|
|
|
-
|
|
|
if __name__ == '__main__':
|
|
|
# 1、读取配置文件
|
|
|
with open('/data/pythonProject/ai_target/config/config.yaml', mode='r', encoding='utf-8') as f:
|