Parcourir la source

线上数据库表更换为内网的数据库表

liyuyi@c-top.com.cn il y a 3 ans
Parent
commit
0e93a5c763
4 fichiers modifiés avec 73 ajouts et 60 suppressions
  1. 1 1
      .idea/ai_target.iml
  2. 18 12
      BayesCombine.py
  3. 52 44
      config/config.yaml
  4. 2 3
      config/url_and_db.py

+ 1 - 1
.idea/ai_target.iml

@@ -2,7 +2,7 @@
 <module type="PYTHON_MODULE" version="4">
   <component name="NewModuleRootManager">
     <content url="file://$MODULE_DIR$" />
-    <orderEntry type="jdk" jdkName="ai_target_test_env" jdkType="Python SDK" />
+    <orderEntry type="jdk" jdkName="Remote Python 3.8.10 (sftp://root@192.168.0.195:22/data/Miniconda3/envs/ai_target/bin/python3.8)" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
   <component name="PyDocumentationSettings">

+ 18 - 12
BayesCombine.py

@@ -14,7 +14,7 @@ curr_path = os.path.abspath(os.path.dirname(__file__))
 project_root_path = curr_path[:curr_path.find("ai_target") + len("ai_target")]
 sys.path.append(project_root_path)
 
-from config.url_and_db import estimate_people_number_url, headers, jeecg_db, jeecg_product_db, application_product_db
+from config.url_and_db import estimate_people_number_url, headers, jeecg_db, jeecg_product_db, application_product_db,application_inner_db
 from utils.BaseClass import NpEncoder
 from utils.commonFunc import age_code_transform, gender_code_transform
 
@@ -30,7 +30,8 @@ class GetSingleDimFeatures(object):
     计算素材单维度滑窗组合转化占比情况
     """
 
-    def __init__(self, signature, dim, dim_config):
+    def __init__(self, signature, dim, dim_config, project_id):
+        self.project_id = project_id
         self.signature = signature
         self.dim = dim
         self.dim_config = dim_config
@@ -60,8 +61,9 @@ class GetSingleDimFeatures(object):
     def get_features(self):
         # 1、获取素材在指定维度下的人群数据
         get_audience_sql = f"select signature, {self.filed_name}, sum(activation) activation  " \
-                           f"from {self.table} where signature = '{self.signature}' group by signature, {self.filed_name}"
-        audience_df = pd.read_sql(get_audience_sql, jeecg_product_db)
+                           f"from {self.table} where signature = '{self.signature}' and project_id = {self.project_id} " \
+                           f"group by signature, {self.filed_name}"
+        audience_df = pd.read_sql(get_audience_sql, application_inner_db)
         for combine in self.window_combine:
             if combine != ['不限']:
                 self.features['|'.join(combine)] = round(audience_df[audience_df[self.filed_name].isin(combine)].activation.sum() / \
@@ -142,20 +144,24 @@ if __name__ == '__main__':
         logger.info(f"开始获取项目{project_id}的当前活跃素材")
         # 4.1 获取指定项目下当前活跃的素材信息
         # 如近60天内累计激活个数达到100个(开发环境或者生产环境,这部分都读取生产数据库)
-        start_date = datetime.datetime.now().date() + datetime.timedelta(days=-config['activeMaterialFilterRule']['days'])
+        days = config['activeMaterialFilterRule']['days']
+        activation_sum = config['activeMaterialFilterRule']['activation']
+        pay_first_pct = config['activeMaterialFilterRule']['event_pay_first_day_pct']
         sql = f"select signature from kuaishou_material_video_report_daily_dw where project_id = {project_id} " \
-              f"and stat_date >= {start_date.year * 10000 + start_date.month * 100 + start_date.day} group by signature " \
-              f"having sum(activation) >= {config['activeMaterialFilterRule']['activation']} "  \
-              f"and sum(event_pay_first_day) / sum(activation) >= {config['activeMaterialFilterRule']['event_pay_first_day_pct']}"
-        df = pd.read_sql(sql, application_product_db)
+              f"and stat_date >= date_format(date_sub(now(), interval {days} DAY),'%%Y%%m%%d') group by signature " \
+              f"having sum(activation) >= {activation_sum} " \
+              f"and sum(event_pay_first_day) / sum(activation) >= {pay_first_pct}"
+
+        df = pd.read_sql(sql, application_inner_db)
         active_signatures = list(df[~df.signature.isnull()].signature.values)
         active_signatures = active_signatures * 2 if len(active_signatures) == 1 else active_signatures
 
         # 4.2 在素材人群报表筛选里面累计100个激活
         logger.info(f"开始获取项目{project_id}的活跃素材的人群分析报表")
-        sql = f"select signature from ctop_kuaishou_audience_report_daily_age_material where signature in {tuple(active_signatures)} " \
+        sql = f"select signature from kuaishou_unit_audience_age_report_daily_d where signature in {tuple(active_signatures)} " \
+              f"and project_id = {project_id} " \
               f"group by signature having sum(activation) >= {config['getBaysCombineMaterialFilterRule']['activation']}"
-        df = pd.read_sql(sql, jeecg_product_db)
+        df = pd.read_sql(sql, application_inner_db)
         signature_lst = df['signature'].values
 
         # 这行代码用于测试
@@ -166,7 +172,7 @@ if __name__ == '__main__':
             try:
                 feature_lst = []
                 for dimension in target_dim:
-                    get_single_fea_ins = GetSingleDimFeatures(sig, dimension, config['bayesDim'][dimension])
+                    get_single_fea_ins = GetSingleDimFeatures(sig, dimension, config['bayesDim'][dimension], project_id)
                     # 计算滑窗组合
                     get_single_fea_ins.get_window_combine()
                     get_single_fea_ins.get_features()

+ 52 - 44
config/config.yaml

@@ -27,44 +27,52 @@ jeecg_dev_db:
 
 
 
+application_inner_db:
+  host: 192.168.0.184
+  username: hcst
+  password: hcst@2021
+  port: 3390
+  database: application
+
+
 
 bayesDim:
-    age:
-        windowSize: 3
-        isOn: True
-        Lst:
-          - '18-23岁'
-          - '24-30岁'
-          - '31-40岁'
-          - '41-49岁'
-          - '50+岁'
-        table: 'ctop_kuaishou_audience_report_daily_age_material'
-        fieldName: 'age_segment'
-        ratio_threshold: 0.8
-        ratio_diff_threshold: 0.1
-    gender:
-        windowSize: 1
-        isOn: True
-        Lst:
-            - '男'
-            - '女'
-        table: 'ctop_kuaishou_audience_report_daily_gender_material'
-        fieldName: 'gender'
-        ratio_threshold: 0.7
-        ratio_diff_threshold: 0.1
-    city:
-        windowSize: 4
-        isOn: False
-        Lst:
-            - '一线城市'
-            - '新一线城市'
-            - '二线城市'
-            - '三线城市'
-            - '四线城市'
-            - '五线城市'
-        table: 'ctop_kuaishou_audience_report_daily_city_material'
-        fieldName: 'city'
-        ratio_threshold: 0.7
+  age:
+    windowSize: 3
+    isOn: True
+    Lst:
+      - '18-23岁'
+      - '24-30岁'
+      - '31-40岁'
+      - '41-49岁'
+      - '50+岁'
+    table: 'kuaishou_unit_audience_age_report_daily_d'
+    fieldName: 'age_segment'
+    ratio_threshold: 0.8
+    ratio_diff_threshold: 0.1
+  gender:
+    windowSize: 1
+    isOn: True
+    Lst:
+      - '男'
+      - '女'
+    table: 'kuaishou_unit_audience_gender_report_daily_d'
+    fieldName: 'gender'
+    ratio_threshold: 0.7
+    ratio_diff_threshold: 0.1
+  city:
+    windowSize: 4
+    isOn: False
+    Lst:
+      - '一线城市'
+      - '新一线城市'
+      - '二线城市'
+      - '三线城市'
+      - '四线城市'
+      - '五线城市'
+    table: 'kuaishou_unit_audience_city_report_daily_d'
+    fieldName: 'city'
+    ratio_threshold: 0.7
 
 
 # 活跃素材筛选规则:近60天累计激活个数达到100个
@@ -73,7 +81,7 @@ activeMaterialFilterRule:
   activation: 100
   event_pay_first_day_pct: 0.35
 
-# 计算推荐组合的素材筛选规则:近两个月内累计达到100个
+# 计算推荐组合的素材筛选规则:激活个数累计达到100个
 getBaysCombineMaterialFilterRule:
   days: 60
   activation: 100
@@ -81,22 +89,22 @@ getBaysCombineMaterialFilterRule:
 
 
 filterTargetCombine:
-    combineCnt: 20
+  combineCnt: 20
 
 
 mac_ip_config:
   # 开发环境
   '5254003fa716':
-      ai_target_url: "http://139.186.165.84:31012/"
-      yourong_url: "http://192.168.0.195:8806/jeecg-boot/"
+    ai_target_url: "http://139.186.165.84:31012/"
+    yourong_url: "http://192.168.0.195:8806/jeecg-boot/"
   # 测试环境
   '00163e9162d9':
-      ai_target_url: "http://192.168.0.195:31012/"
-      yourong_url: "http://192.168.0.195:8806/jeecg-boot/"
+    ai_target_url: "http://192.168.0.195:31012/"
+    yourong_url: "http://192.168.0.195:8806/jeecg-boot/"
   # 生产环境
   '52540076ce6d':
-      ai_target_url: "http://139.186.27.96:31012/"
-      yourong_url: "http://api.tjyourong.com.cn/jeecg-boot/"
+    ai_target_url: "http://139.186.27.96:31012/"
+    yourong_url: "http://api.tjyourong.com.cn/jeecg-boot/"
 
 
 

+ 2 - 3
config/url_and_db.py

@@ -1,13 +1,13 @@
 import uuid
 import yaml
 from utils.commonFunc import get_db_engine
+
 headers = {'Content-Type': 'application/json'}
 
 with open('/data/pythonProject/ai_target/config/config.yaml', mode='r', encoding='utf-8') as f:
     config = yaml.load(f.read(), Loader=yaml.FullLoader)
     mac_ip_config = config['mac_ip_config']
 
-
 # 数据库连接引擎,依据开发环境/测试环境/生产环境 进行切换
 mac = uuid.UUID(int=uuid.getnode()).hex[-12:]
 if mac in ['5254003fa716', '00163e9162d9']:
@@ -19,7 +19,7 @@ else:
 
 jeecg_product_db = get_db_engine(config['jeecg_product_db'])
 application_product_db = get_db_engine(config['application_product_db'])
-
+application_inner_db = get_db_engine(config['application_inner_db'])
 
 # 创建计划url
 create_campaign_url = mac_ip_config[mac]['yourong_url'] + 'kuaishou/create/campaignCreate'
@@ -55,4 +55,3 @@ create_group_and_creative_url = mac_ip_config[mac]['yourong_url'] + 'kuaishou/cr
 
 # 定向人群预估查询
 estimate_people_number_url = mac_ip_config[mac]['yourong_url'] + 'estimatePeopleNumber/getCount'
-