liyuyi@c-top.com.cn 5 年之前
当前提交
1c89781a8c

+ 8 - 0
.idea/.gitignore

@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Datasource local storage ignored files
+/../../../:\script_similarity\.idea/dataSources/
+/dataSources.local.xml
+# Editor-based HTTP Client requests
+/httpRequests/

+ 6 - 0
.idea/encodings.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Encoding">
+    <file url="PROJECT" charset="GBK" />
+  </component>
+</project>

+ 6 - 0
.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

+ 4 - 0
.idea/misc.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.7 (simhash)" project-jdk-type="Python SDK" />
+</project>

+ 8 - 0
.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/script_similarity.iml" filepath="$PROJECT_DIR$/.idea/script_similarity.iml" />
+    </modules>
+  </component>
+</project>

+ 8 - 0
.idea/script_similarity.iml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

+ 8 - 0
.idea/simHash.iml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

+ 6 - 0
.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

+ 17 - 0
config.py

@@ -0,0 +1,17 @@
+# 数据库连接信息 --- 本地环境
+from sqlalchemy import create_engine
+con_str = "mysql+pymysql://%s:%s@%s:%d/%s" % \
+                ("root", "root@123", "127.0.0.1", 3306, "mysql")
+engine = create_engine(con_str, connect_args={'charset': 'UTF8MB4'})
+
+# 数据库连接信息 --- 测试环境
+
+
+# 数据库连接信息 --- 线上环境
+
+
+# 提取的分词个数
+topK = 20
+
+# 相似度的阈值
+distance_threshold = 22

文件差异内容过多而无法显示
+ 6894 - 0
logs/simhash_model/model_service.log


+ 124 - 0
logs/simhash_model/model_service.log.2020-09-30

@@ -0,0 +1,124 @@
+09/30/2020 05:15:10 PM INFO simhash similarity service started at 2020-09-30 17:15:10
+09/30/2020 05:15:10 PM ERROR Traceback (most recent call last):
+  File "E:/script_similarity/main.py", line 42, in create_application
+    with open('simhash_model/doc/pre_remove_words.txt','r',encoding='utf-8') as f:
+FileNotFoundError: [Errno 2] No such file or directory: 'simhash_model/doc/pre_remove_words.txt'
+
+09/30/2020 05:15:41 PM INFO simhash similarity service started at 2020-09-30 17:15:41
+09/30/2020 05:15:41 PM ERROR Traceback (most recent call last):
+  File "E:/script_similarity/main.py", line 42, in create_application
+    with open('doc/pre_remove_words.txt','r',encoding='utf-8') as f:
+FileNotFoundError: [Errno 2] No such file or directory: 'doc/pre_remove_words.txt'
+
+09/30/2020 05:16:17 PM INFO simhash similarity service started at 2020-09-30 17:16:17
+09/30/2020 05:16:17 PM ERROR Traceback (most recent call last):
+  File "E:/script_similarity/main.py", line 42, in create_application
+    with open('./simhash_model/doc/pre_remove_words.txt','r',encoding='utf-8') as f:
+FileNotFoundError: [Errno 2] No such file or directory: './simhash_model/doc/pre_remove_words.txt'
+
+09/30/2020 05:16:31 PM INFO simhash similarity service started at 2020-09-30 17:16:31
+09/30/2020 05:16:31 PM ERROR Traceback (most recent call last):
+  File "E:/script_similarity/main.py", line 42, in create_application
+    with open('/simhash_model/doc/pre_remove_words.txt','r',encoding='utf-8') as f:
+FileNotFoundError: [Errno 2] No such file or directory: '/simhash_model/doc/pre_remove_words.txt'
+
+09/30/2020 05:17:10 PM INFO simhash similarity service started at 2020-09-30 17:17:10
+09/30/2020 05:17:10 PM ERROR Traceback (most recent call last):
+  File "E:/script_similarity/main.py", line 42, in create_application
+    with open('E:/script_similarity/simhash_model/doc/pre_remove_words.txt','r',encoding='utf-8') as f:
+FileNotFoundError: [Errno 2] No such file or directory: 'E:/script_similarity/simhash_model/doc/pre_remove_words.txt'
+
+09/30/2020 05:17:56 PM INFO simhash similarity service started at 2020-09-30 17:17:56
+09/30/2020 05:17:56 PM ERROR Traceback (most recent call last):
+  File "E:/script_similarity/main.py", line 45, in create_application
+    with open('simhash_model/doc/stop_words.txt','r',encoding='utf-8') as f:
+FileNotFoundError: [Errno 2] No such file or directory: 'simhash_model/doc/stop_words.txt'
+
+09/30/2020 05:18:29 PM INFO simhash similarity service started at 2020-09-30 17:18:29
+09/30/2020 05:18:29 PM INFO simhash similarity service loaded successfully at 2020-09-30 17:18:29
+09/30/2020 05:19:08 PM INFO simhash similarity service started at 2020-09-30 17:19:08
+09/30/2020 05:19:08 PM INFO simhash similarity service started successfully at 2020-09-30 17:19:08
+09/30/2020 05:53:47 PM INFO simhash similarity service started at 2020-09-30 17:53:47
+09/30/2020 05:53:47 PM INFO simhash similarity service started successfully at 2020-09-30 17:53:47
+09/30/2020 05:55:36 PM INFO simhash similarity service started at 2020-09-30 17:55:36
+09/30/2020 05:55:36 PM INFO simhash similarity service started successfully at 2020-09-30 17:55:36
+09/30/2020 05:55:42 PM INFO ********************************************  NEW REQUEST ********************************************
+09/30/2020 05:56:39 PM INFO simhash similarity service started at 2020-09-30 17:56:39
+09/30/2020 05:56:39 PM INFO simhash similarity service started successfully at 2020-09-30 17:56:39
+09/30/2020 05:56:45 PM INFO ********************************************  NEW REQUEST ********************************************
+09/30/2020 05:56:59 PM INFO simhash similarity service started at 2020-09-30 17:56:59
+09/30/2020 05:56:59 PM INFO simhash similarity service started successfully at 2020-09-30 17:56:59
+09/30/2020 05:57:13 PM INFO ********************************************  NEW REQUEST ********************************************
+09/30/2020 05:57:13 PM INFO the md5 of request script is 123
+09/30/2020 05:57:13 PM ERROR Traceback (most recent call last):
+  File "D:\anaconda3\envs\simhash\lib\site-packages\sqlalchemy\engine\base.py", line 1267, in _execute_context
+    cursor, statement, context
+  File "D:\anaconda3\envs\simhash\lib\site-packages\sqlalchemy\engine\default.py", line 596, in do_execute_no_params
+    cursor.execute(statement)
+  File "D:\anaconda3\envs\simhash\lib\site-packages\pymysql\cursors.py", line 170, in execute
+    result = self._query(query)
+  File "D:\anaconda3\envs\simhash\lib\site-packages\pymysql\cursors.py", line 328, in _query
+    conn.query(q)
+  File "D:\anaconda3\envs\simhash\lib\site-packages\pymysql\connections.py", line 517, in query
+    self._affected_rows = self._read_query_result(unbuffered=unbuffered)
+  File "D:\anaconda3\envs\simhash\lib\site-packages\pymysql\connections.py", line 732, in _read_query_result
+    result.read()
+  File "D:\anaconda3\envs\simhash\lib\site-packages\pymysql\connections.py", line 1075, in read
+    first_packet = self.connection._read_packet()
+  File "D:\anaconda3\envs\simhash\lib\site-packages\pymysql\connections.py", line 684, in _read_packet
+    packet.check_error()
+  File "D:\anaconda3\envs\simhash\lib\site-packages\pymysql\protocol.py", line 220, in check_error
+    err.raise_mysql_exception(self._data)
+  File "D:\anaconda3\envs\simhash\lib\site-packages\pymysql\err.py", line 109, in raise_mysql_exception
+    raise errorclass(errno, errval)
+pymysql.err.InternalError: (1054, "Unknown column 'self.script_md5' in 'where clause'")
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "E:\script_similarity\simhash_model\simhash_distance_handler.py", line 39, in post
+    inst.parse_content()
+  File "E:\script_similarity\simhash_model\simhash_distance_handler.py", line 87, in parse_content
+    df = pd.read_sql(sql, engine)
+  File "D:\anaconda3\envs\simhash\lib\site-packages\pandas\io\sql.py", line 515, in read_sql
+    chunksize=chunksize,
+  File "D:\anaconda3\envs\simhash\lib\site-packages\pandas\io\sql.py", line 1295, in read_query
+    result = self.execute(*args)
+  File "D:\anaconda3\envs\simhash\lib\site-packages\pandas\io\sql.py", line 1162, in execute
+    *args, **kwargs
+  File "D:\anaconda3\envs\simhash\lib\site-packages\sqlalchemy\engine\base.py", line 2237, in execute
+    return connection.execute(statement, *multiparams, **params)
+  File "D:\anaconda3\envs\simhash\lib\site-packages\sqlalchemy\engine\base.py", line 1003, in execute
+    return self._execute_text(object_, multiparams, params)
+  File "D:\anaconda3\envs\simhash\lib\site-packages\sqlalchemy\engine\base.py", line 1178, in _execute_text
+    parameters,
+  File "D:\anaconda3\envs\simhash\lib\site-packages\sqlalchemy\engine\base.py", line 1317, in _execute_context
+    e, statement, parameters, cursor, context
+  File "D:\anaconda3\envs\simhash\lib\site-packages\sqlalchemy\engine\base.py", line 1511, in _handle_dbapi_exception
+    sqlalchemy_exception, with_traceback=exc_info[2], from_=e
+  File "D:\anaconda3\envs\simhash\lib\site-packages\sqlalchemy\util\compat.py", line 182, in raise_
+    raise exception
+  File "D:\anaconda3\envs\simhash\lib\site-packages\sqlalchemy\engine\base.py", line 1267, in _execute_context
+    cursor, statement, context
+  File "D:\anaconda3\envs\simhash\lib\site-packages\sqlalchemy\engine\default.py", line 596, in do_execute_no_params
+    cursor.execute(statement)
+  File "D:\anaconda3\envs\simhash\lib\site-packages\pymysql\cursors.py", line 170, in execute
+    result = self._query(query)
+  File "D:\anaconda3\envs\simhash\lib\site-packages\pymysql\cursors.py", line 328, in _query
+    conn.query(q)
+  File "D:\anaconda3\envs\simhash\lib\site-packages\pymysql\connections.py", line 517, in query
+    self._affected_rows = self._read_query_result(unbuffered=unbuffered)
+  File "D:\anaconda3\envs\simhash\lib\site-packages\pymysql\connections.py", line 732, in _read_query_result
+    result.read()
+  File "D:\anaconda3\envs\simhash\lib\site-packages\pymysql\connections.py", line 1075, in read
+    first_packet = self.connection._read_packet()
+  File "D:\anaconda3\envs\simhash\lib\site-packages\pymysql\connections.py", line 684, in _read_packet
+    packet.check_error()
+  File "D:\anaconda3\envs\simhash\lib\site-packages\pymysql\protocol.py", line 220, in check_error
+    err.raise_mysql_exception(self._data)
+  File "D:\anaconda3\envs\simhash\lib\site-packages\pymysql\err.py", line 109, in raise_mysql_exception
+    raise errorclass(errno, errval)
+sqlalchemy.exc.InternalError: (pymysql.err.InternalError) (1054, "Unknown column 'self.script_md5' in 'where clause'")
+[SQL: select * from ctop_script_file where id = self.script_md5 ]
+(Background on this error at: http://sqlalche.me/e/13/2j85)
+

文件差异内容过多而无法显示
+ 2057 - 0
logs/simhash_model/model_service.log.2020-10-09


+ 65 - 0
main.py

@@ -0,0 +1,65 @@
+import tornado.ioloop
+import tornado.web
+import argparse
+import json
+import pickle
+import tornado.options
+import tornado.log
+import time
+import traceback
+from logging.handlers import TimedRotatingFileHandler
+import logging
+
+from simhash_model import simhash_distance_handler
+
+
+def create_application():
+    # 日志的格式
+    log_formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s',
+                                      '%m/%d/%Y %I:%M:%S %p')
+
+    # 创建日志对象
+    simhash_log_handler = TimedRotatingFileHandler("logs/simhash_model/model_service.log",
+                                                   when="midnight",
+                                                   backupCount=100)
+
+    simhash_log_handler.setFormatter(log_formatter)
+    simhash_logger = logging.getLogger('simhash_logger')
+    simhash_logger.addHandler(simhash_log_handler)
+    simhash_logger.setLevel(logging.DEBUG)
+
+    print("simhash_logger's id is {}".format(id(simhash_logger)))
+    msg = "simhash similarity service started at {t}".\
+           format(t=time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
+    simhash_logger.info(msg)
+
+    try:
+        with open('simhash_model/doc/pre_remove_word.txt', 'r', encoding='utf-8') as f:
+            pre_remove_words = f.readlines()
+        pre_remove_words = [item.rstrip('\n') for item in pre_remove_words]
+
+        msg = "simhash similarity service started successfully at {t}".format(
+            t=time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
+        simhash_logger.info(msg)
+    except Exception:
+        msg = ''.join(traceback.format_exc())
+        simhash_logger.error(msg)
+        raise Exception(msg)
+
+    application = tornado.web.Application([
+        (r"/simhash_diatance_calc/(\w+)",
+         simhash_distance_handler.OnlineSimHashSimilarity,
+         {"pre_remove_words": pre_remove_words,
+          "logger": simhash_logger}),
+    ])
+
+    return application
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-p', '--port', type=int, default=31011)
+    args = parser.parse_args()
+    application = create_application()
+    application.listen(args.port)
+    tornado.ioloop.IOLoop.instance().start()

+ 52 - 0
simhash_model/doc/etl_stopword.txt

@@ -0,0 +1,52 @@
+20
+APP
+OS
+一个
+上面
+下方
+下载
+会员
+余额
+你们
+免费
+哎呀
+妈妈
+姐夫
+字幕
+室内
+室外
+小孩
+小朋友
+小猪
+小马
+屏幕
+录屏
+影视
+微信
+怎么
+我们
+我用
+我要
+手机
+打开
+提现
+游戏
+点击
+玩游戏
+现在
+电影
+画面
+红包
+花钱
+视频
+资源
+赚钱
+赶快
+赶紧
+软件
+这个
+这么
+配音
+链接
+镜头
+领取

+ 60 - 0
simhash_model/doc/pre_remove_word.txt

@@ -0,0 +1,60 @@
+男1
+女1
+男2
+女2
+万年历
+爱奇艺随刻
+来啊捕鱼
+夸克
+疯狂收租婆
+红柚(听书)
+红柚
+魔帝归来
+火山极速版
+火山极速
+爱上消消消
+好运到万年历
+清理管家
+闲鱼
+大赢家斗地主
+抖音极速版
+经典消消消
+极速安全管家
+来啊 捕鱼
+西瓜视频
+红柚免费小说
+水果大亨
+易车
+西瓜
+爱奇艺
+支付宝
+王者荣耀
+女一
+女二
+女三
+男一
+男二
+男三
+一男
+两男
+一女
+两女
+一 女
+两 女
+一老
+一小孩
+男主
+女主
+老公
+老婆
+群演
+道具
+\d+\.*\d*
+会议室
+室内
+拍摄画面
+录屏画面
+配音
+领到
+赠送
+福利

二进制
simhash_model/file_cache/脚本模板-时间+客户+功能点.docx


+ 228 - 0
simhash_model/simhash_distance_handler.py

@@ -0,0 +1,228 @@
+import tornado.web
+import pandas as pd
+import json
+import numpy as np
+import datetime
+from config import *
+import traceback
+import hashlib
+import jieba
+import jieba.analyse
+from operator import itemgetter
+import requests
+import docx
+import re
+
+# 设置停用词--分词之后去掉的词
+jieba.analyse.set_stop_words('./simhash_model/doc/etl_stopword.txt')
+
+
+class OnlineSimHashSimilarity(tornado.web.RequestHandler):
+    def initialize(self, pre_remove_words, logger):
+        self.pre_remove_words = pre_remove_words
+        self.logger = logger
+
+    def post(self, script_md5):
+        res = {
+                'script_md5': script_md5,
+                'similartiry_script': None
+               }
+
+        self.logger.info("**************************************  NEW REQUEST ***************************************")
+        self.logger.info("the md5 of request script is %s" % script_md5)
+
+        try:
+            inst = ParseContentAndGetsimhash(script_md5, self.pre_remove_words)
+            # 提取台词
+            inst.parse_content()
+            self.logger.info("the parsed content is %s", str(inst.content))
+
+            # 计算 simhash
+            inst.get_simhash()
+            self.logger.info("the simhash is %s", str(inst.finger_print))
+            self.logger.info("the keyword and weight is %s", str(inst.keyword_weight))
+
+            # simhash 写入数据库
+            inst.write_db()
+            self.logger.info("write to database has completed !")
+
+            # 与数据库里所有的剧本进行海明距离计算,过滤出与之相识度高的剧本
+            simi_inst = GetSimilarityScripts(inst.file_name, inst.script_md5, inst.finger_print)
+            if simi_inst.similarity_scripts:
+                res['similartiry_script'] = simi_inst.similarity_scripts
+                self.logger.info("there are {} similarity script: {} !".
+                                 format(len(eval(simi_inst.similarity_scripts)),
+                                        simi_inst.similarity_scripts))
+
+                # 相似度的剧本信息写入数据库
+                simi_inst.write_db()
+                self.logger.info("SimilarityScripts write to database has completed !")
+            else:
+                self.logger.info("no similarity script!")
+        except Exception:
+            self.logger.error(traceback.format_exc())
+
+        # 返回接口结果
+        result_str = json.dumps(res)
+        self.write(result_str)
+        self.flush()
+
+
+class ParseContentAndGetsimhash(object):
+    """
+    1、根据 script_md5 从数据库库里查找文档的下载地址,下载文档;
+    2、解析文档中的台词;
+    3、计算台词的 simhash 码。
+    """
+    def __init__(self, script_md5, pre_remove_words):
+        self.script_md5 = script_md5
+        self.pre_remove_words = pre_remove_words
+        self.file_name = None
+        self.content = None
+        self.finger_print = None
+        self.keyword_weight = None
+
+    def parse_content(self):
+        # 通过url将文件下载到本地
+        sql = """select * from ctop_script_file where id = '%s' """ % self.script_md5
+        df = pd.read_sql(sql, engine)
+        download_url = df['download_url'].values[0]
+        self.file_name = df['file_name'].values[0]
+        r = requests.get(download_url)
+        with open('./simhash_model/file_cache/%s' % self.file_name, 'wb') as code:
+            code.write(r.content)
+
+        # 解析本地的文件,提取台词
+        # TODO: 台词的提取解析,以每段的第一个冒号进行分割,冒号之前包含'录屏、 场景、演员'等关键字,则不提取冒号之后的内容。
+        doc = docx.Document('./simhash_model/file_cache/%s' % self.file_name)
+        script_list = [para.text for para in doc.paragraphs]
+        self.content = ''.join(script_list)
+
+        # 台词预处理,去除产品名称/角色名等
+        for word in self.pre_remove_words:
+            pattern = re.compile(word)
+            self.content = re.sub(pattern, " ", self.content)
+
+    def get_simhash(self):
+        sim = SimHash(self.content)
+        self.finger_print = sim.finger_print
+        self.keyword_weight = sim.keyword_weight
+
+    def write_db(self):
+        res_dict = {
+            "id": self.script_md5,
+            "file_name": self.file_name,
+            "finger_print": self.finger_print,
+            "create_time": datetime.datetime.now()
+            }
+        res_df = pd.DataFrame.from_dict(res_dict, orient='index').T
+        print("id of engine", id(engine))
+        res_df.to_sql(name='ctop_script_fingerprint_v3', con=engine, if_exists='append', index=False)
+
+
+class SimHash(object):
+    def __init__(self, content):
+        self.content = content
+        self.finger_print = None
+        self.keyword_weight = None
+        self.sim_hash()
+
+    def sim_hash(self):
+        """
+        计算文档的simHash指纹
+        :return:64位的01字符串
+        """
+        seg_list = jieba.cut(self.content, cut_all=False)  # 精确模式
+
+        # 如果 topK<=30,则提取 前30个权重的关键词,并按照keyword进行排序
+        if topK <= 30:
+            keyword_weight = jieba.analyse.extract_tags("|".join(seg_list), 30, withWeight=True)
+        else:
+            keyword_weight = jieba.analyse.extract_tags("|".join(seg_list), topK, withWeight=True)
+
+        # 如果没有关键信息,则直接返回,finger_print 为 None
+        if len(keyword_weight) == 0:
+            return
+
+        # sort by weight then by keyword
+        sorted_keyword_weight = sorted(keyword_weight, key=itemgetter(1, 0), reverse=True)[:topK]
+        self.keyword_weight = sorted_keyword_weight
+
+        weight_hash_list = []
+        for keyword, weight in sorted_keyword_weight:
+            weight = int(10 * weight)  # 与使用原始的weight的区别?
+
+            # 获取单词的哈希码
+            str_hash = self.build_in_hash(keyword)
+            weight_hash = [weight if b == '1' else -weight for b in str_hash]
+            weight_hash_list.append(weight_hash)
+        weight_sum = np.sum(np.array(weight_hash_list), axis=0)
+
+        self.finger_print = ''.join(['1' if i > 0 else '0' for i in weight_sum])
+
+    def build_in_hash(self, keyword):
+        """
+        使用 hashlib.md5 计算关键词的哈希码 (整个词语直接调用该函数,不用挨个单字调用)
+        :return:64位的二进制字符串
+        """
+        truncate_mask = 2 ** 64 - 1
+        bitstring_format = '0{}b'.format(64)
+        h = int(hashlib.md5(keyword.encode('utf-8')).hexdigest(), 16)  # 16进制转为10进制
+        h_bits = format(h & truncate_mask, bitstring_format)  # 截取为64位的二进制字符串
+        return h_bits
+
+
+class GetSimilarityScripts(object):
+    """
+    获取数据库里的所有 simhash,并与之进行海明距离计算,返回距离小于阈值的脚本对象。
+    如果存在高相似度的文档,则将相关信息写入数据库
+    """
+    def __init__(self, file_name, script_md5, finger_print):
+        self.file_name = file_name
+        self.script_md5 = script_md5
+        self.finger_print = finger_print
+        self.similarity_scripts = ""
+
+        self.get_similarity_scripts()
+
+    def get_similarity_scripts(self):
+        sql = """select id, file_name, finger_print from ctop_script_fingerprint_v3"""
+        df = pd.read_sql(sql, engine)
+        df['distance'] = df['finger_print'].apply(lambda x: self.hamming_dis(x))
+        sim_df = df[df['distance'] == distance_threshold]
+        sim_df.reset_index(drop=True, inplace=True)
+
+        # 列表3元组形式:"[('name1', 'md5', 1), ('name2', 'md5', 4), ('name3', 'md5', 9)]"
+        if len(sim_df) > 0:
+            sim_list = []
+            for i in range(len(sim_df)):
+                sim_list.append((sim_df.loc[i, 'file_name'],
+                                sim_df.loc[i, 'id'],
+                                sim_df.loc[i, 'distance']))
+            self.similarity_scripts = str(sim_list)
+        else:
+            pass
+
+    def hamming_dis(self, another_finger_print):
+        # 如果其中一个为空,则距离返回为空
+        if not self.finger_print or not another_finger_print:
+            return
+
+        h1 = '0b' + self.finger_print
+        h2 = '0b' + another_finger_print
+        n = int(h1, 2) ^ int(h2, 2)
+        cnt = 0
+        while n:
+            n &= (n - 1)
+            cnt += 1
+        return cnt
+
+    def write_db(self):
+        res_dict = {
+            "id": self.script_md5,
+            "file_name": self.file_name,
+            "similarity": self.similarity_scripts,
+            "create_time": datetime.datetime.now()
+        }
+        res_df = pd.DataFrame.from_dict(res_dict, orient='index').T
+        res_df.to_sql(name='ctop_script_similarity', con=engine, if_exists='append', index=False)