liyuyi@c-top.com.cn il y a 4 ans
commit
55f32a2d93

+ 19 - 0
.idea/dataSources.local.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="dataSourceStorageLocal">
+    <data-source name="OnLine" uuid="ce40759c-9f24-450b-942c-5e6978faeda3">
+      <database-info product="MySQL" version="5.7.25-TiDB-v5.2.0" jdbc-version="4.2" driver-name="MariaDB Connector/J" driver-version="2.7.3" dbms="MYSQL" exact-version="5.7.25" exact-driver-version="2.7">
+        <extra-name-characters>#@</extra-name-characters>
+        <identifier-quote-string>`</identifier-quote-string>
+      </database-info>
+      <case-sensitivity plain-identifiers="mixed" quoted-identifiers="mixed" />
+      <secret-storage>master_key</secret-storage>
+      <user-name>data</user-name>
+      <schema-mapping>
+        <introspection-scope>
+          <node kind="schema" qname="@" />
+        </introspection-scope>
+      </schema-mapping>
+    </data-source>
+  </component>
+</project>

+ 11 - 0
.idea/dataSources.xml

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="DataSourceManagerImpl" format="xml" multifile-model="true">
+    <data-source source="LOCAL" name="OnLine" uuid="ce40759c-9f24-450b-942c-5e6978faeda3">
+      <driver-ref>mariadb</driver-ref>
+      <synchronize>true</synchronize>
+      <jdbc-driver>org.mariadb.jdbc.Driver</jdbc-driver>
+      <jdbc-url>jdbc:mariadb://139.186.27.96:3390/jeecg-boot</jdbc-url>
+    </data-source>
+  </component>
+</project>

Fichier diff supprimé car celui-ci est trop grand
+ 58413 - 0
.idea/dataSources/ce40759c-9f24-450b-942c-5e6978faeda3.xml


+ 22 - 0
.idea/deployment.xml

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="PublishConfigData" autoUpload="On explicit save action" serverName="similar_material_remote_root@139.186.165.84" remoteFilesAllowedToDisappearOnAutoupload="false">
+    <serverData>
+      <paths name="ai_target_remote_root@139.186.165.84:22 password">
+        <serverdata>
+          <mappings>
+            <mapping deploy="/data/pythonProject/ai_target" local="$PROJECT_DIR$/../ai_target" web="/" />
+          </mappings>
+        </serverdata>
+      </paths>
+      <paths name="similar_material_remote_root@139.186.165.84">
+        <serverdata>
+          <mappings>
+            <mapping deploy="/data/pythonProject/similar_material" local="$PROJECT_DIR$" web="/" />
+          </mappings>
+        </serverdata>
+      </paths>
+    </serverData>
+    <option name="myAutoUpload" value="ON_EXPLICIT_SAVE" />
+  </component>
+</project>

+ 19 - 0
.idea/inspectionProfiles/Project_Default.xml

@@ -0,0 +1,19 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="DuplicatedCode" enabled="true" level="WEAK WARNING" enabled_by_default="true">
+      <Languages>
+        <language minSize="120" name="Python" />
+      </Languages>
+    </inspection_tool>
+    <inspection_tool class="PyPep8Inspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
+      <option name="ignoredErrors">
+        <list>
+          <option value="E131" />
+        </list>
+      </option>
+    </inspection_tool>
+    <inspection_tool class="SqlDialectInspection" enabled="false" level="WARNING" enabled_by_default="false" />
+    <inspection_tool class="SqlResolveInspection" enabled="false" level="ERROR" enabled_by_default="false" />
+  </profile>
+</component>

+ 6 - 0
.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

+ 8 - 0
.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/similar_material.iml" filepath="$PROJECT_DIR$/.idea/similar_material.iml" />
+    </modules>
+  </component>
+</project>

+ 8 - 0
.idea/similar_material.iml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="jdk" jdkName="Remote Python 3.8.11 (sftp://root@139.186.165.84:22/data/Miniconda3/envs/similar_material/bin/python)" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

+ 208 - 0
.idea/workspace.xml

@@ -0,0 +1,208 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ChangeListManager">
+    <list default="true" id="31704e56-aca5-4e2d-bea1-d331132222c4" name="Default Changelist" comment="" />
+    <option name="SHOW_DIALOG" value="false" />
+    <option name="HIGHLIGHT_CONFLICTS" value="true" />
+    <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
+    <option name="LAST_RESOLUTION" value="IGNORE" />
+  </component>
+  <component name="JupyterTrust" id="ca2d42e4-d722-44a7-b416-c78a02b293a2" />
+  <component name="ProjectId" id="1xtcY2zgL9PgwlsfQTItYSWSoLL" />
+  <component name="ProjectViewState">
+    <option name="hideEmptyMiddlePackages" value="true" />
+    <option name="showLibraryContents" value="true" />
+    <option name="showMembers" value="true" />
+  </component>
+  <component name="PropertiesComponent">
+    <property name="DatabaseDriversLRU" value="mariadb" />
+    <property name="RunOnceActivity.OpenProjectViewOnStart" value="true" />
+    <property name="WebServerToolWindowFactoryState" value="true" />
+    <property name="WebServerToolWindowPanel.toolwindow.highlight.mappings" value="true" />
+    <property name="WebServerToolWindowPanel.toolwindow.highlight.symlinks" value="true" />
+    <property name="WebServerToolWindowPanel.toolwindow.show.date" value="false" />
+    <property name="WebServerToolWindowPanel.toolwindow.show.permissions" value="false" />
+    <property name="WebServerToolWindowPanel.toolwindow.show.size" value="false" />
+    <property name="last_opened_file_path" value="$PROJECT_DIR$" />
+    <property name="node.js.detected.package.eslint" value="true" />
+    <property name="node.js.detected.package.tslint" value="true" />
+    <property name="node.js.path.for.package.eslint" value="project" />
+    <property name="node.js.path.for.package.tslint" value="project" />
+    <property name="node.js.selected.package.eslint" value="(autodetect)" />
+    <property name="node.js.selected.package.tslint" value="(autodetect)" />
+    <property name="settings.editor.selected.configurable" value="com.jetbrains.python.configuration.PyActiveSdkModuleConfigurable" />
+  </component>
+  <component name="RunManager">
+    <configuration name="historical_material" type="PythonConfigurationType" factoryName="Python" temporary="true" nameIsGenerated="true">
+      <module name="similar_material" />
+      <option name="INTERPRETER_OPTIONS" value="" />
+      <option name="PARENT_ENVS" value="true" />
+      <envs>
+        <env name="PYTHONUNBUFFERED" value="1" />
+        <env name="LYY_DEV" value="dev" />
+      </envs>
+      <option name="SDK_HOME" value="" />
+      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
+      <option name="IS_MODULE_SDK" value="true" />
+      <option name="ADD_CONTENT_ROOTS" value="true" />
+      <option name="ADD_SOURCE_ROOTS" value="true" />
+      <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
+      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/historical_material.py" />
+      <option name="PARAMETERS" value="" />
+      <option name="SHOW_COMMAND_LINE" value="false" />
+      <option name="EMULATE_TERMINAL" value="false" />
+      <option name="MODULE_MODE" value="false" />
+      <option name="REDIRECT_INPUT" value="false" />
+      <option name="INPUT_FILE" value="" />
+      <method v="2" />
+    </configuration>
+    <recent_temporary>
+      <list>
+        <item itemvalue="Python.historical_material" />
+      </list>
+    </recent_temporary>
+  </component>
+  <component name="SpellCheckerSettings" RuntimeDictionaries="0" Folders="0" CustomDictionaries="0" DefaultDictionary="application-level" UseSingleDictionary="true" transferred="true" />
+  <component name="TaskManager">
+    <task active="true" id="Default" summary="Default task">
+      <changelist id="31704e56-aca5-4e2d-bea1-d331132222c4" name="Default Changelist" comment="" />
+      <created>1631177869104</created>
+      <option name="number" value="Default" />
+      <option name="presentableId" value="Default" />
+      <updated>1631177869104</updated>
+      <workItem from="1631177870267" duration="1370000" />
+      <workItem from="1631516407803" duration="507000" />
+      <workItem from="1631518475799" duration="418000" />
+      <workItem from="1631518912011" duration="191000" />
+      <workItem from="1631519119181" duration="6843000" />
+      <workItem from="1631584529196" duration="124000" />
+      <workItem from="1631601848067" duration="64000" />
+      <workItem from="1631695590002" duration="1530000" />
+      <workItem from="1631757779061" duration="17549000" />
+      <workItem from="1631846187097" duration="1382000" />
+      <workItem from="1631861458465" duration="38000" />
+      <workItem from="1631861519848" duration="283000" />
+      <workItem from="1631861811445" duration="1237000" />
+    </task>
+    <servers />
+  </component>
+  <component name="TypeScriptGeneratedFilesManager">
+    <option name="version" value="3" />
+  </component>
+  <component name="WindowStateProjectService">
+    <state x="1348" y="345" width="1404" height="849" key="#Deployment" timestamp="1631695991505">
+      <screen x="0" y="39" width="3440" height="1401" />
+    </state>
+    <state x="1073" y="361" key="#Deployment/0.30.3440.1410@0.30.3440.1410" timestamp="1631184746519" />
+    <state x="1348" y="345" width="1404" height="849" key="#Deployment/0.39.3440.1401@0.39.3440.1401" timestamp="1631695991505" />
+    <state x="1561" y="335" key="#Options" timestamp="1631695996481">
+      <screen x="0" y="39" width="3440" height="1401" />
+    </state>
+    <state x="1238" y="398" key="#Options/0.30.3440.1410@0.30.3440.1410" timestamp="1631184627720" />
+    <state x="1561" y="335" key="#Options/0.39.3440.1401@0.39.3440.1401" timestamp="1631695996481" />
+    <state x="1331" y="719" width="1431" height="699" key="#com.intellij.execution.impl.EditConfigurationsDialog" timestamp="1631521915391">
+      <screen x="0" y="39" width="3440" height="1401" />
+    </state>
+    <state x="1331" y="719" width="1431" height="699" key="#com.intellij.execution.impl.EditConfigurationsDialog/0.39.3440.1401@0.39.3440.1401" timestamp="1631521915391" />
+    <state x="1260" y="650" width="1118" height="734" key="#xdebugger.evaluate" timestamp="1631778697501">
+      <screen x="0" y="39" width="3440" height="1401" />
+    </state>
+    <state x="1260" y="650" width="1118" height="734" key="#xdebugger.evaluate/0.39.3440.1401@0.39.3440.1401" timestamp="1631778697501" />
+    <state x="1809" y="461" width="824" height="601" key="EnvironmentVariablesDialog" timestamp="1631521912488">
+      <screen x="0" y="39" width="3440" height="1401" />
+    </state>
+    <state x="1809" y="461" width="824" height="601" key="EnvironmentVariablesDialog/0.39.3440.1401@0.39.3440.1401" timestamp="1631521912488" />
+    <state x="1374" y="473" key="FileChooserDialogImpl" timestamp="1631861493770">
+      <screen x="0" y="30" width="3440" height="1410" />
+    </state>
+    <state x="1374" y="473" key="FileChooserDialogImpl/0.30.3440.1410@0.30.3440.1410" timestamp="1631861493770" />
+    <state x="1374" y="479" key="FileChooserDialogImpl/0.39.3440.1401@0.39.3440.1401" timestamp="1631695773220" />
+    <state width="1478" height="451" key="GridCell.Tab.0.bottom" timestamp="1631790340398">
+      <screen x="0" y="39" width="3440" height="1401" />
+    </state>
+    <state width="1478" height="451" key="GridCell.Tab.0.bottom/0.39.3440.1401@0.39.3440.1401" timestamp="1631790340398" />
+    <state width="1478" height="451" key="GridCell.Tab.0.center" timestamp="1631790340397">
+      <screen x="0" y="39" width="3440" height="1401" />
+    </state>
+    <state width="1478" height="451" key="GridCell.Tab.0.center/0.39.3440.1401@0.39.3440.1401" timestamp="1631790340397" />
+    <state width="1478" height="451" key="GridCell.Tab.0.left" timestamp="1631790340397">
+      <screen x="0" y="39" width="3440" height="1401" />
+    </state>
+    <state width="1478" height="451" key="GridCell.Tab.0.left/0.39.3440.1401@0.39.3440.1401" timestamp="1631790340397" />
+    <state width="1478" height="451" key="GridCell.Tab.0.right" timestamp="1631790340398">
+      <screen x="0" y="39" width="3440" height="1401" />
+    </state>
+    <state width="1478" height="451" key="GridCell.Tab.0.right/0.39.3440.1401@0.39.3440.1401" timestamp="1631790340398" />
+    <state width="1647" height="234" key="GridCell.Tab.1.bottom" timestamp="1631790340388">
+      <screen x="0" y="39" width="3440" height="1401" />
+    </state>
+    <state width="1647" height="234" key="GridCell.Tab.1.bottom/0.39.3440.1401@0.39.3440.1401" timestamp="1631790340388" />
+    <state width="1647" height="234" key="GridCell.Tab.1.center" timestamp="1631790340388">
+      <screen x="0" y="39" width="3440" height="1401" />
+    </state>
+    <state width="1647" height="234" key="GridCell.Tab.1.center/0.39.3440.1401@0.39.3440.1401" timestamp="1631790340388" />
+    <state width="1647" height="234" key="GridCell.Tab.1.left" timestamp="1631790340388">
+      <screen x="0" y="39" width="3440" height="1401" />
+    </state>
+    <state width="1647" height="234" key="GridCell.Tab.1.left/0.39.3440.1401@0.39.3440.1401" timestamp="1631790340388" />
+    <state width="1647" height="234" key="GridCell.Tab.1.right" timestamp="1631790340388">
+      <screen x="0" y="39" width="3440" height="1401" />
+    </state>
+    <state width="1647" height="234" key="GridCell.Tab.1.right/0.39.3440.1401@0.39.3440.1401" timestamp="1631790340388" />
+    <state x="1055" y="341" width="1655" height="883" key="SettingsEditor" timestamp="1631695873267">
+      <screen x="0" y="39" width="3440" height="1401" />
+    </state>
+    <state x="1055" y="341" width="1655" height="883" key="SettingsEditor/0.39.3440.1401@0.39.3440.1401" timestamp="1631695873267" />
+    <state width="1730" height="696" key="XDebugger.FullValuePopup" timestamp="1631758279917">
+      <screen x="0" y="39" width="3440" height="1401" />
+    </state>
+    <state width="1730" height="696" key="XDebugger.FullValuePopup/0.39.3440.1401@0.39.3440.1401" timestamp="1631758279917" />
+    <state x="1643" y="642" key="com.jetbrains.plugins.remotesdk.ui.RemoteMappingsConfiguration" timestamp="1631695750382">
+      <screen x="0" y="39" width="3440" height="1401" />
+    </state>
+    <state x="1643" y="642" key="com.jetbrains.plugins.remotesdk.ui.RemoteMappingsConfiguration/0.39.3440.1401@0.39.3440.1401" timestamp="1631695750382" />
+    <state x="1651" y="304" width="768" height="667" key="ftp.server.browser" timestamp="1631695810714">
+      <screen x="0" y="39" width="3440" height="1401" />
+    </state>
+    <state x="1651" y="304" width="768" height="667" key="ftp.server.browser/0.39.3440.1401@0.39.3440.1401" timestamp="1631695810714" />
+  </component>
+  <component name="XDebuggerManager">
+    <breakpoint-manager>
+      <breakpoints>
+        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
+          <url>file://$PROJECT_DIR$/historical_material.py</url>
+          <line>201</line>
+          <option name="timeStamp" value="24" />
+        </line-breakpoint>
+        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
+          <url>file://$PROJECT_DIR$/historical_material.py</url>
+          <line>186</line>
+          <option name="timeStamp" value="25" />
+        </line-breakpoint>
+        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
+          <url>file://$PROJECT_DIR$/historical_material.py</url>
+          <line>202</line>
+          <option name="timeStamp" value="26" />
+        </line-breakpoint>
+        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
+          <url>file://$PROJECT_DIR$/historical_material.py</url>
+          <line>173</line>
+          <option name="timeStamp" value="28" />
+        </line-breakpoint>
+        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
+          <url>file://$PROJECT_DIR$/historical_material.py</url>
+          <line>177</line>
+          <option name="timeStamp" value="29" />
+        </line-breakpoint>
+        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
+          <url>file://$PROJECT_DIR$/historical_material.py</url>
+          <line>97</line>
+          <option name="timeStamp" value="33" />
+        </line-breakpoint>
+      </breakpoints>
+    </breakpoint-manager>
+  </component>
+  <component name="com.intellij.coverage.CoverageDataManagerImpl">
+    <SUITE FILE_PATH="coverage/similar_material$historical_material.coverage" NAME="historical_material Coverage Results" MODIFIED="1631787705424" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
+  </component>
+</project>

+ 18 - 0
common_func.py

@@ -0,0 +1,18 @@
+from sqlalchemy import create_engine
+from urllib import parse
+
+
+def get_db_engine(db_info):
+    db_con_str = 'mysql+pymysql://%s:%s@%s:%d/%s' % \
+                 (db_info['username'],
+                  parse.quote_plus(db_info['password']),
+                  db_info['host'],
+                  db_info['port'],
+                  db_info['database'])
+    engine = create_engine(db_con_str, connect_args={'charset': 'utf8'})
+    return engine
+
+
+
+
+

+ 69 - 0
config/config.yaml

@@ -0,0 +1,69 @@
+projectId:
+  - 458
+
+# 生产数据库
+productDB:
+  host: 139.186.27.96
+  username: data
+  password: hcst@2021
+  port: 3390
+  database: jeecg-boot
+
+# 开发测试数据库
+devDB:
+  host: 139.186.165.84
+  username: hcst
+  password: hcst@2020
+  port: 3306
+  database: jeecg-boot
+
+# 本地数据库
+localDB:
+  host: 192.168.1.193
+  username: root
+  password: root@123
+  port: 3306
+  database: mysql
+
+
+bayesDim:
+    age:
+        windowSize: 3
+        isOn: True
+        Lst:
+          - '18-23岁'
+          - '24-30岁'
+          - '31-40岁'
+          - '41-49岁'
+          - '50+岁'
+        table: 'ctop_kuaishou_audience_report_daily_age_material'
+        fieldName: 'age_segment'
+    gender:
+        windowSize: 1
+        isOn: True
+        Lst:
+            - '男'
+            - '女'
+        table: 'ctop_kuaishou_audience_report_daily_gender_material'
+        fieldName: 'gender'
+    city:
+        windowSize: 4
+        isOn: False
+        Lst:
+            - '一线城市'
+            - '新一线城市'
+            - '二线城市'
+            - '三线城市'
+            - '四线城市'
+            - '五线城市'
+        table: 'ctop_kuaishou_audience_report_daily_city_material'
+        fieldName: 'city'
+
+
+# k: 返回最多几个相邻素材
+# distanceThreshold: 小于等于该距离阈值,才判定为相似素材
+nearestNeighbors:
+  k: 30
+  distanceThreshold: 20
+
+

BIN
distances.pkl


BIN
final_feature_df.pkl


+ 239 - 0
historical_material.py

@@ -0,0 +1,239 @@
+import os
+import yaml
+from common_func import get_db_engine
+import pandas as pd
+from sklearn.preprocessing import StandardScaler
+from sklearn.neighbors import NearestNeighbors
+import numpy as np
+import sys
+import pickle
+import logging
+from concurrent_log import ConcurrentTimedRotatingFileHandler
+
+log_formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%Y/%m/%d %I:%M:%S %p')
+log_handler = ConcurrentTimedRotatingFileHandler("/data/pythonProject/similar_material/logs/historical_material.log", when="midnight",
+                                                 backupCount=100)
+log_handler.setFormatter(log_formatter)
+logger = logging.getLogger('historical_material_logger')
+logger.addHandler(log_handler)
+logger.setLevel(logging.DEBUG)
+logger.info("historical_material_logger started!")
+print('id of historical_material_logger %s' % id(logger))
+
+
+def scaler_feature(data, axis=0):
+    """
+    :param data: pd.DataFrame
+    :param axis: 0-横向归一化 ,  1-纵向归一化
+    :return: z = (x - u) / s
+    """
+    scaler = StandardScaler()
+    scaler_data = None
+    if axis == 0:
+        scaler_data = scaler.fit_transform(data.T).T
+    if axis == 1:
+        scaler_data = scaler.fit_transform(data)
+
+    scaler_df = pd.DataFrame(scaler_data)
+    scaler_df.index = data.index
+    scaler_df.columns = [col + '_scaler' for col in data.columns]
+    return scaler_df
+
+
+def get_segment_features(table, signature, dim_col, project_id, col_prefix):
+    # 1 获取原始数据
+    get_org_data_sql = """
+    select signature, aclick, bclick, activation, charge, %s from %s
+    where signature in %s and project_id = %s
+    """ % (dim_col, table, tuple(signature), project_id)
+    org_df = pd.read_sql(get_org_data_sql, read_engine)
+
+    # 2 计算各分段的 行为率、激活率、激活成本,并进行横向归一化
+    g = org_df.groupby(['signature', dim_col]).agg({'activation': sum, 'aclick': sum, 'bclick': sum, 'charge': sum}).unstack()
+    # 该维度下有哪些分段值:如 gender维度下有:男、女、其他 , 需要去掉 '其他',因为业务含义不明确,没法解释模型效果
+    segment_cols = g['charge'].columns
+    if '其他' in segment_cols:
+        segment_cols.remove('其他')
+
+    segment_fea_df = pd.DataFrame()
+    for col in segment_cols:
+        segment_fea_df[col + '_bclick_aclick_pct'] = g[('bclick', col)] / g[('aclick', col)]
+        segment_fea_df[col + '_activation_bclick_pct'] = g[('activation', col)] / g[('bclick', col)]
+        segment_fea_df[col + '_cost'] = g[('charge', col)] / g[('activation', col)]
+
+    # 3 计算累计激活中各分段的激活个数占比,并进行横向归一化
+    for col in segment_cols:
+        segment_fea_df[col + '_activation_pct'] = g[('activation', col)] / (g[[('activation', item) for item in segment_cols]].sum(axis=1))
+
+    # 4 对 np.nan 和 np.inf 进行处理
+    # 4-1 cost
+    # np.inf : 分子>0/分母=0, 表示有消耗但是没有激活数,填充为 999
+    # np.nan :分子=0/分母=0, 表示没有消耗也没有激活数,填充为 -1
+    cost_cols = [col for col in segment_fea_df.columns if 'cost' in col]
+    segment_fea_df[cost_cols] = segment_fea_df[cost_cols].replace({np.nan: -1, np.inf: 999})
+
+    # 4-2 占比类
+    # np.inf :分子>0/分母=0 ,如行为率:表示有点击但没有曝光,属于数据bug情况。需要剔除该素材并写入日志
+    # np.nan :分子=0/分母=0 ,如激活率:表示没有激活也没有点击,填充为 0
+    pct_cols = [col for col in segment_fea_df.columns if 'pct' in col]
+    segment_fea_df[pct_cols] = segment_fea_df[pct_cols].replace({np.nan: 0})
+
+    inf_df = np.isinf(segment_fea_df[pct_cols]).sum(axis=1)
+    remove_sig_lst = inf_df[inf_df >= 1].index
+    # TODO 添加日志 记录数据存在问题的素材
+    if len(remove_sig_lst):
+        print(remove_sig_lst)
+
+    segment_fea_df = segment_fea_df[~segment_fea_df.index.isin(remove_sig_lst)]
+
+    # 5 对特征进行横向归一化
+    segment_fea_scaler_df = pd.DataFrame()
+    for col in ['bclick_aclick_pct', 'activation_bclick_pct', 'cost', 'activation_pct']:
+        tmp_df = segment_fea_df.filter(like=col)
+        scaler_df = scaler_feature(tmp_df, axis=0)
+        segment_fea_scaler_df = pd.concat([segment_fea_scaler_df, scaler_df], axis=1)
+
+    # 6 col_prefix and index
+    segment_fea_scaler_df.columns = [col_prefix + col for col in segment_fea_scaler_df.columns]
+    segment_fea_scaler_df.index = g.index
+    return segment_fea_scaler_df
+
+
+def get_signature_features(signature, project_id):
+    """
+    计算素材整体的 行为率、激活率、激活成本
+    :param project_id:
+    :param signature:
+    :return:
+    """
+    get_data_sql = """ select signature, sum(aclick) aclick, sum(bclick) bclick, sum(activation) activation, sum(charge) charge
+    from ctop_kuaishou_audience_report_daily_age_material
+    where signature in %s and project_id = %s
+    group by signature
+    """ % (tuple(signature), project_id)
+    sig_org_df = pd.read_sql(get_data_sql, read_engine)
+
+    sig_fea_df = pd.DataFrame()
+    sig_fea_df['cost'] = sig_org_df['charge'] / sig_org_df['activation']
+    sig_fea_df['bclick_aclick_pct'] = sig_org_df['bclick'] / sig_org_df['aclick']
+    sig_fea_df['activation_bclick_pct'] = sig_org_df['activation'] / sig_org_df['bclick']
+
+    # 对特征进行纵向的归一化
+    sig_fea_df.index = sig_org_df['signature'].values
+    sig_fea_scaler_df = scaler_feature(sig_fea_df, axis=1)
+
+    return sig_fea_scaler_df
+
+
+if __name__ == '__main__':
+    # 1、读取配置文件
+    with open('config/config.yaml', mode='r', encoding='utf-8') as f:
+        config = yaml.load(f.read(), Loader=yaml.FullLoader)
+
+    # 1-1 数据库连接引擎,依据开发环境/生产环境 进行切换
+    if os.getenv('LYY_DEV', 'unknown') == 'dev':
+        write_engine = get_db_engine(config['devDB'])
+    else:
+        write_engine = get_db_engine(config['productDB'])
+
+    read_engine = get_db_engine(config['productDB'])
+
+    # 2、获取历史所有达标的素材
+    project_id = 458
+    get_sig_sql = """select signature, sum(activation) activation
+             from ctop_kuaishou_audience_report_daily_age_material
+             where project_id = %s
+             group by signature
+            having sum(activation) >= 100
+            """ % project_id
+    sig_df = pd.read_sql(get_sig_sql, read_engine)
+    sig_df = sig_df[~sig_df.signature.isnull()]
+    sig_lst = list(sig_df.signature.values)
+    print('sig_lst:', len(sig_lst))
+
+    # TODO for test
+    # sig_lst = ['cefd07cf7951ba9674d43c1d9c112f37',
+    #            '6f11859acaf4f1dd501706f8d50995d8',
+    #            'bc8fe6ae97db794acaf050acff4386b9',
+    #            '59a34e41bfcf321a931a0b204a8609c9',
+    #            'c67604889af726d49401f40283865bb0',
+    #            '75237de4427a9ff6da994976bf516f78',
+    #            '0ab2a8816b1b34638767862123fc74a6',
+    #            '177a0f359eb90623b7ecaf1c8d0b9db4',
+    #            'd3c9bbd98d07afb7be9fe8b1b40133d3']
+
+    if sig_lst is None:
+        # TODO 加入 logger
+        sys.exit(0)
+    if len(sig_lst) == 1:
+        # TODO 加入 logger
+        sys.exit(1)
+
+    # 2、构造特征
+    # 2-1 获取素材整体的特征情况:行为率、激活率、激活成本
+    sig_feature_df = get_signature_features(sig_lst, project_id)
+
+    # 2-2 获取各维度(age,gender)各分段(男、女等)的行为率、激活率、激活成本、激活占比,并进行纵向归一化
+    # get_segment_features(table, signature, dim_col, project_id)
+    table_dim_tuple = [('ctop_kuaishou_audience_report_daily_age_material', 'age_segment', 'age_'),
+                       ('ctop_kuaishou_audience_report_daily_gender_material', 'gender', 'gender_')]
+
+    segment_feature_df = pd.DataFrame()
+    for table, dim_col, col_prefix in table_dim_tuple:
+        tmp_df = get_segment_features(table, sig_lst, dim_col, project_id, col_prefix)
+        segment_feature_df = pd.concat([segment_feature_df, tmp_df], axis=1)
+
+    # 2-3 素材整体的特征 和 各维度各分段的特征 进行合并,得到最终的特征DataFrame
+    final_feature_df = pd.concat([sig_feature_df, segment_feature_df], axis=1)
+    # 部分素材在 age 或者 gender 维度上没有数据,特征concat时,会导致空值
+    # 去掉这部分的素材,防止在调用 NearestNeighbors 时,报错
+    null_df = final_feature_df.isnull().sum(axis=1)
+    exist_null_fea_sig_lst = null_df[null_df >= 1].index
+    # TODO 添加日志 记录存在数据缺失的素材
+    if len(exist_null_fea_sig_lst) > 0:
+        print(exist_null_fea_sig_lst)
+
+    final_feature_df = final_feature_df[~final_feature_df.index.isin(exist_null_fea_sig_lst)]
+
+    with open('final_feature_df.pkl', 'wb') as f:
+        pickle.dump(final_feature_df, f)
+
+    # 3 Finding the Nearest Neighbors
+    k = config['nearestNeighbors']['k']
+    distance_threshold = config['nearestNeighbors']['distanceThreshold']
+    X = final_feature_df.values
+    nbrs = NearestNeighbors(n_neighbors=k, algorithm='ball_tree').fit(X)
+    distances, indices = nbrs.kneighbors(X)
+
+    with open('distances.pkl', 'wb') as f:
+        pickle.dump(distances, f)
+
+    with open('indices.pkl', 'wb') as f:
+        pickle.dump(indices, f)
+
+    # 3-1 两点之间的距离超过阈值时,不作为相邻素材
+    # mask 中,为 True 的元素替换为 NaN
+    mask = distances > distance_threshold
+    indices_mask_df = pd.DataFrame(indices).mask(mask)
+    indices_mask_df = indices_mask_df.astype(dtype=pd.Int64Dtype())
+
+    # 3-2 把 index 替换为具体的 signatures
+    sig_lst = final_feature_df.index.values
+    indices_mask_df.index = sig_lst
+
+    indices_mask_df = indices_mask_df.applymap(lambda x: sig_lst[x] if not pd.isnull(x) else x)
+    res_lst = []
+    for index, row in indices_mask_df.iterrows():
+        others = [item for item in row if (not pd.isnull(item)) and (item != index)]
+        res_lst.append([index] + others)
+
+    res_df = pd.DataFrame(res_lst)
+    res_df.columns = ['self'] + ['N' + str(i + 1) for i in range(res_df.shape[1] - 1)]
+    columns = ['self'] + ['N' + str(i + 1) for i in range(k)]
+    res_df = res_df.reindex(columns=columns)
+
+    with open('res_df.pkl', 'wb') as f:
+        pickle.dump(res_df, f)
+
+    # 4 TODO 结果入库
+    pass

+ 2 - 0
incremental_material.py

@@ -0,0 +1,2 @@
+if __name__ == '__main__':
+    pass

BIN
indices.pkl


+ 7 - 0
main.py

@@ -0,0 +1,7 @@
+
+if __name__ == '__main__':
+    # 1、获取
+    pass
+
+
+

BIN
res_df.pkl