simhash_distance_handler.py 8.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228
  1. import tornado.web
  2. import pandas as pd
  3. import json
  4. import numpy as np
  5. import datetime
  6. from config import *
  7. import traceback
  8. import hashlib
  9. import jieba
  10. import jieba.analyse
  11. from operator import itemgetter
  12. import requests
  13. import docx
  14. import re
  15. # 设置停用词--分词之后去掉的词
  16. jieba.analyse.set_stop_words('./simhash_model/doc/etl_stopword.txt')
  17. class OnlineSimHashSimilarity(tornado.web.RequestHandler):
  18. def initialize(self, pre_remove_words, logger):
  19. self.pre_remove_words = pre_remove_words
  20. self.logger = logger
  21. def post(self, script_md5):
  22. res = {
  23. 'script_md5': script_md5,
  24. 'similartiry_script': None
  25. }
  26. self.logger.info("************************************** NEW REQUEST ***************************************")
  27. self.logger.info("the md5 of request script is %s" % script_md5)
  28. try:
  29. inst = ParseContentAndGetsimhash(script_md5, self.pre_remove_words)
  30. # 提取台词
  31. inst.parse_content()
  32. self.logger.info("the parsed content is %s", str(inst.content))
  33. # 计算 simhash
  34. inst.get_simhash()
  35. self.logger.info("the simhash is %s", str(inst.finger_print))
  36. self.logger.info("the keyword and weight is %s", str(inst.keyword_weight))
  37. # simhash 写入数据库
  38. inst.write_db()
  39. self.logger.info("write to database has completed !")
  40. # 与数据库里所有的剧本进行海明距离计算,过滤出与之相识度高的剧本
  41. simi_inst = GetSimilarityScripts(inst.file_name, inst.script_md5, inst.finger_print)
  42. if simi_inst.similarity_scripts:
  43. res['similartiry_script'] = simi_inst.similarity_scripts
  44. self.logger.info("there are {} similarity script: {} !".
  45. format(len(eval(simi_inst.similarity_scripts)),
  46. simi_inst.similarity_scripts))
  47. # 相似度的剧本信息写入数据库
  48. simi_inst.write_db()
  49. self.logger.info("SimilarityScripts write to database has completed !")
  50. else:
  51. self.logger.info("no similarity script!")
  52. except Exception:
  53. self.logger.error(traceback.format_exc())
  54. # 返回接口结果
  55. result_str = json.dumps(res)
  56. self.write(result_str)
  57. self.flush()
  58. class ParseContentAndGetsimhash(object):
  59. """
  60. 1、根据 script_md5 从数据库库里查找文档的下载地址,下载文档;
  61. 2、解析文档中的台词;
  62. 3、计算台词的 simhash 码。
  63. """
  64. def __init__(self, script_md5, pre_remove_words):
  65. self.script_md5 = script_md5
  66. self.pre_remove_words = pre_remove_words
  67. self.file_name = None
  68. self.content = None
  69. self.finger_print = None
  70. self.keyword_weight = None
  71. def parse_content(self):
  72. # 通过url将文件下载到本地
  73. sql = """select * from ctop_script_file where id = '%s' """ % self.script_md5
  74. df = pd.read_sql(sql, engine)
  75. download_url = df['download_url'].values[0]
  76. self.file_name = df['file_name'].values[0]
  77. r = requests.get(download_url)
  78. with open('./simhash_model/file_cache/%s' % self.file_name, 'wb') as code:
  79. code.write(r.content)
  80. # 解析本地的文件,提取台词
  81. # TODO: 台词的提取解析,以每段的第一个冒号进行分割,冒号之前包含'录屏、 场景、演员'等关键字,则不提取冒号之后的内容。
  82. doc = docx.Document('./simhash_model/file_cache/%s' % self.file_name)
  83. script_list = [para.text for para in doc.paragraphs]
  84. self.content = ''.join(script_list)
  85. # 台词预处理,去除产品名称/角色名等
  86. for word in self.pre_remove_words:
  87. pattern = re.compile(word)
  88. self.content = re.sub(pattern, " ", self.content)
  89. def get_simhash(self):
  90. sim = SimHash(self.content)
  91. self.finger_print = sim.finger_print
  92. self.keyword_weight = sim.keyword_weight
  93. def write_db(self):
  94. res_dict = {
  95. "id": self.script_md5,
  96. "file_name": self.file_name,
  97. "finger_print": self.finger_print,
  98. "create_time": datetime.datetime.now()
  99. }
  100. res_df = pd.DataFrame.from_dict(res_dict, orient='index').T
  101. print("id of engine", id(engine))
  102. res_df.to_sql(name='ctop_script_fingerprint_v3', con=engine, if_exists='append', index=False)
  103. class SimHash(object):
  104. def __init__(self, content):
  105. self.content = content
  106. self.finger_print = None
  107. self.keyword_weight = None
  108. self.sim_hash()
  109. def sim_hash(self):
  110. """
  111. 计算文档的simHash指纹
  112. :return:64位的01字符串
  113. """
  114. seg_list = jieba.cut(self.content, cut_all=False) # 精确模式
  115. # 如果 topK<=30,则提取 前30个权重的关键词,并按照keyword进行排序
  116. if topK <= 30:
  117. keyword_weight = jieba.analyse.extract_tags("|".join(seg_list), 30, withWeight=True)
  118. else:
  119. keyword_weight = jieba.analyse.extract_tags("|".join(seg_list), topK, withWeight=True)
  120. # 如果没有关键信息,则直接返回,finger_print 为 None
  121. if len(keyword_weight) == 0:
  122. return
  123. # sort by weight then by keyword
  124. sorted_keyword_weight = sorted(keyword_weight, key=itemgetter(1, 0), reverse=True)[:topK]
  125. self.keyword_weight = sorted_keyword_weight
  126. weight_hash_list = []
  127. for keyword, weight in sorted_keyword_weight:
  128. weight = int(10 * weight) # 与使用原始的weight的区别?
  129. # 获取单词的哈希码
  130. str_hash = self.build_in_hash(keyword)
  131. weight_hash = [weight if b == '1' else -weight for b in str_hash]
  132. weight_hash_list.append(weight_hash)
  133. weight_sum = np.sum(np.array(weight_hash_list), axis=0)
  134. self.finger_print = ''.join(['1' if i > 0 else '0' for i in weight_sum])
  135. def build_in_hash(self, keyword):
  136. """
  137. 使用 hashlib.md5 计算关键词的哈希码 (整个词语直接调用该函数,不用挨个单字调用)
  138. :return:64位的二进制字符串
  139. """
  140. truncate_mask = 2 ** 64 - 1
  141. bitstring_format = '0{}b'.format(64)
  142. h = int(hashlib.md5(keyword.encode('utf-8')).hexdigest(), 16) # 16进制转为10进制
  143. h_bits = format(h & truncate_mask, bitstring_format) # 截取为64位的二进制字符串
  144. return h_bits
  145. class GetSimilarityScripts(object):
  146. """
  147. 获取数据库里的所有 simhash,并与之进行海明距离计算,返回距离小于阈值的脚本对象。
  148. 如果存在高相似度的文档,则将相关信息写入数据库
  149. """
  150. def __init__(self, file_name, script_md5, finger_print):
  151. self.file_name = file_name
  152. self.script_md5 = script_md5
  153. self.finger_print = finger_print
  154. self.similarity_scripts = ""
  155. self.get_similarity_scripts()
  156. def get_similarity_scripts(self):
  157. sql = """select id, file_name, finger_print from ctop_script_fingerprint_v3"""
  158. df = pd.read_sql(sql, engine)
  159. df['distance'] = df['finger_print'].apply(lambda x: self.hamming_dis(x))
  160. sim_df = df[df['distance'] == distance_threshold]
  161. sim_df.reset_index(drop=True, inplace=True)
  162. # 列表3元组形式:"[('name1', 'md5', 1), ('name2', 'md5', 4), ('name3', 'md5', 9)]"
  163. if len(sim_df) > 0:
  164. sim_list = []
  165. for i in range(len(sim_df)):
  166. sim_list.append((sim_df.loc[i, 'file_name'],
  167. sim_df.loc[i, 'id'],
  168. sim_df.loc[i, 'distance']))
  169. self.similarity_scripts = str(sim_list)
  170. else:
  171. pass
  172. def hamming_dis(self, another_finger_print):
  173. # 如果其中一个为空,则距离返回为空
  174. if not self.finger_print or not another_finger_print:
  175. return
  176. h1 = '0b' + self.finger_print
  177. h2 = '0b' + another_finger_print
  178. n = int(h1, 2) ^ int(h2, 2)
  179. cnt = 0
  180. while n:
  181. n &= (n - 1)
  182. cnt += 1
  183. return cnt
  184. def write_db(self):
  185. res_dict = {
  186. "id": self.script_md5,
  187. "file_name": self.file_name,
  188. "similarity": self.similarity_scripts,
  189. "create_time": datetime.datetime.now()
  190. }
  191. res_df = pd.DataFrame.from_dict(res_dict, orient='index').T
  192. res_df.to_sql(name='ctop_script_similarity', con=engine, if_exists='append', index=False)