liyuyi@c-top.com.cn 4 سال پیش
والد
کامیت
879b6c8555
1فایلهای تغییر یافته به همراه75 افزوده شده و 59 حذف شده
  1. 75 59
      BayesCombine.py

+ 75 - 59
BayesCombine.py

@@ -7,6 +7,18 @@ from functools import reduce
 from itertools import product
 from datetime import datetime
 from statsmodels.stats.proportion import proportions_ztest
+import traceback
+
+import logging
+from concurrent_log import ConcurrentTimedRotatingFileHandler
+
+log_formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%Y/%m/%d %I:%M:%S %p')
+log_handler = ConcurrentTimedRotatingFileHandler("logs/BayesCombine.log", when="midnight", backupCount=100)
+log_handler.setFormatter(log_formatter)
+logger = logging.getLogger('bayes_combine_logger')
+logger.addHandler(log_handler)
+logger.setLevel(logging.DEBUG)
+print('id of bayes_combine_logger %s' % id(logger))
 
 
 class BayesFeatures(object):
@@ -190,62 +202,66 @@ class BayesCombine(object):
 
 
 if __name__ == '__main__':
-    # 1、读取配置文件
-    with open('config/config.yaml', mode='r', encoding='utf-8') as f:
-        config = yaml.load(f.read(), Loader=yaml.FullLoader)
-
-    # 1-1 数据库连接引擎,依据开发环境/生产环境 进行切换
-    if os.getenv('LYY_DEV', 'unknown') == 'dev':
-        engine = get_db_engine(config['devDB'])
-    else:
-        engine = get_db_engine(config['productDB'])
-
-    # 2、 参与定向组合的维度
-    target_dim = [key for key in config['bayesDim'].keys() if config['bayesDim'][key]['isOn']]
-
-    # 4、计算贝叶斯组合入库
-    for project_id in config['projectId']:
-        # 4.1 获取指定项目下当前活跃的素材信息, 如近3天内累计激活个数达到50个(开发环境或者生产环境,这部分都读取生产数据库)
-        product_engine = get_db_engine(config['productDB'])
-        sql = '''
-           select signature from ctop_kuaishou_report_daily_material 
-           where account_id in (select account_id from ctop_user_allocation where project_id = %s)
-           and datediff(now(),stat_date) <= %s
-           group by signature
-           having sum(activation) >= %s
-           ''' % (project_id,
-                  config['activeMaterialFilterRule']['days'],
-                  config['activeMaterialFilterRule']['activation'])
-        df = pd.read_sql(sql, product_engine)
-        active_signatures = df[~df.signature.isnull()].signature.values
-
-        # 4.2 在素材人群报表筛选里面近一个月累计100个激活
-        # TODO 等生产的素材人群表存在之后,修改表名,添加 days
-        sql = """
-        select signature from ctop_kuaishou_audience_daily_report_by_signature_age  
-        where signature in %s
-           group by signature
-           having sum(activation) >= %s
-        """ % (tuple(active_signatures),
-               config['getBaysCombineMaterialFilterRule']['activation'])
-        df = pd.read_sql(sql, engine)
-        signature_lst = df['signature'].values
-
-        # TODO 这行代码用于测试
-        signature_lst = ['0070efb7557b2a04cf3d4a6f243c3cd8', '03b93728f0d82ea7865c3c7cf632bc1b']
-
-        # 4.2 计算指定素材的贝叶斯特征
-        for sig in signature_lst:
-            for t_type in config['targetType']:
-                bayes_feature_lst = []
-                for dimension in target_dim:
-                    cls = BayesFeatures(sig, t_type, dimension, config['bayesDim'][dimension])
-                    # 计算滑窗组合
-                    cls.get_window_combine()
-                    cls.get_bayes_feature()
-                    bayes_feature_lst.append(cls.bayes_feature)
-
-                # 依据特征值,计算多维度的组合预估值
-                cls = BayesCombine(sig, project_id, t_type, bayes_feature_lst)
-                cls.get_bayes_estimate()
-                cls.write_to_db()
+    try:
+        # 1、读取配置文件
+        with open('config/config.yaml', mode='r', encoding='utf-8') as f:
+            config = yaml.load(f.read(), Loader=yaml.FullLoader)
+
+        # 1-1 数据库连接引擎,依据开发环境/生产环境 进行切换
+        if os.getenv('LYY_DEV', 'unknown') == 'dev':
+            engine = get_db_engine(config['devDB'])
+        else:
+            engine = get_db_engine(config['productDB'])
+
+        # 2、 参与定向组合的维度
+        target_dim = [key for key in config['bayesDim'].keys() if config['bayesDim'][key]['isOn']]
+
+        # 4、计算贝叶斯组合入库
+        for project_id in config['projectId']:
+            # 4.1 获取指定项目下当前活跃的素材信息, 如近3天内累计激活个数达到50个(开发环境或者生产环境,这部分都读取生产数据库)
+            product_engine = get_db_engine(config['productDB'])
+            sql = '''
+               select signature from ctop_kuaishou_report_daily_material 
+               where account_id in (select account_id from ctop_user_allocation where project_id = %s)
+               and datediff(now(),stat_date) <= %s
+               group by signature
+               having sum(activation) >= %s
+               ''' % (project_id,
+                      config['activeMaterialFilterRule']['days'],
+                      config['activeMaterialFilterRule']['activation'])
+            df = pd.read_sql(sql, product_engine)
+            active_signatures = df[~df.signature.isnull()].signature.values
+
+            # 4.2 在素材人群报表筛选里面近一个月累计100个激活
+            # TODO 等生产的素材人群表存在之后,修改表名,添加 days
+            sql = """
+            select signature from ctop_kuaishou_audience_daily_report_by_signature_age  
+            where signature in %s
+               group by signature
+               having sum(activation) >= %s
+            """ % (tuple(active_signatures),
+                   config['getBaysCombineMaterialFilterRule']['activation'])
+            df = pd.read_sql(sql, engine)
+            signature_lst = df['signature'].values
+
+            # TODO 这行代码用于测试
+            signature_lst = ['0070efb7557b2a04cf3d4a6f243c3cd8', '03b93728f0d82ea7865c3c7cf632bc1b']
+
+            # 4.2 计算指定素材的贝叶斯特征
+            for sig in signature_lst:
+                for t_type in config['targetType']:
+                    bayes_feature_lst = []
+                    for dimension in target_dim:
+                        cls = BayesFeatures(sig, t_type, dimension, config['bayesDim'][dimension])
+                        # 计算滑窗组合
+                        cls.get_window_combine()
+                        cls.get_bayes_feature()
+                        bayes_feature_lst.append(cls.bayes_feature)
+
+                    # 依据特征值,计算多维度的组合预估值
+                    cls = BayesCombine(sig, project_id, t_type, bayes_feature_lst)
+                    cls.get_bayes_estimate()
+                    cls.write_to_db()
+            logger.info('project_id is %s 完成贝叶斯组合计算!' % project_id)
+    except:
+        logger.error("traceback is %s" % (traceback.format_exc()))