liyuyi@c-top.com.cn 3 år sedan
förälder
incheckning
a3081c05eb

+ 4 - 0
main.py

@@ -8,6 +8,9 @@ from loguru import logger
 from routers import get_related_words
 from routers import script_config
 from routers import tengxunyun_server
+from script_score.lstm_network import SentimentNet
+from script_score.evaluate_script import router as evaluate_script_router
+
 
 logger.remove()  # 删去 import logger之后自动产生的handler,不删除的话会出现重复输出的现象
 logger.add("/data/pythonProject/video_to_word/logs/main_server.{time:YYYY-MM-DD}.log",
@@ -44,6 +47,7 @@ app.add_middleware(
 app.include_router(get_related_words.router)
 app.include_router(tengxunyun_server.router)
 app.include_router(script_config.router)
+app.include_router(evaluate_script_router)
 
 if __name__ == '__main__':
     uvicorn.run(app='main:app', host="0.0.0.0", port=31013, reload=True, debug=True)

+ 106 - 0
script_score/evaluate_script.py

@@ -0,0 +1,106 @@
+import random
+
+import jieba
+import numpy as np
+import os
+import pickle
+import sys
+import torch
+from fastapi import APIRouter
+from loguru import logger
+import traceback
+
+curr_path = os.path.abspath(os.path.dirname(__file__))
+project_root_path = curr_path[:curr_path.find("video_to_word") + len("video_to_word")]
+sys.path.append(project_root_path)
+
+random_num = random.random()
+print("OUT random_num", random_num)
+
+with open('/data/pythonProject/video_to_word/script_score/pkl/vocab.pkl', 'rb') as f:
+    vocab = pickle.load(f)
+
+with open('/data/pythonProject/video_to_word/script_score/pkl/word_to_idx.pkl', 'rb') as f:
+    word_to_idx = pickle.load(f)
+
+log_score_min = -19.266664505004883
+log_score_max = 0
+
+
+def split_by_jieba(x):
+    seg_list = jieba.cut(x)
+    seg_list = ','.join(seg_list)
+    seg_list = seg_list.split(",")
+    seg_list = [v for v in seg_list if v != ',']
+    return seg_list
+
+
+def encode_samples(tokenized_samples):
+    features = []
+    for sample in tokenized_samples:
+        feature = []
+        for token in sample[0]:
+            if token in word_to_idx:
+                feature.append(word_to_idx[token])
+            else:
+                feature.append(0)
+        features.append(feature)
+    return features
+
+
+def pad_samples(features, maxlen=113, PAD=0):
+    padded_features = []
+    for feature in features:
+        if len(feature) >= maxlen:
+            padded_feature = feature[:maxlen]
+        else:
+            padded_feature = feature
+            while len(padded_feature) < maxlen:
+                padded_feature.append(PAD)
+        padded_features.append(padded_feature)
+    return padded_features
+
+
+def log_and_map_min_max_score(score):
+    # log 变换
+    log_score = np.log(score)
+    # 映射到 0-100 分
+    out_score = 0 + (100 - 0) / (log_score_max - log_score_min) * (log_score - log_score_min)
+    out_score = round(out_score, 2)
+    # 映射到 A/B/C 等级 (A:[95,100], B:[75,95), C:[0,75))
+    score_level = 'C' if out_score < 75 else ('B' if out_score < 95 else 'A')
+    return out_score, score_level
+
+
+router = APIRouter(tags=['evaluate_script_server'])
+
+
+@logger.catch()
+@router.post("/evaluate_script", description="脚本质量评级", summary="脚本质量评级")
+def evaluate_script(script):
+    response = {'code': 0, "success": True, "result": [], "message": "脚本评分完成"}
+    try:
+        script_split_lst = split_by_jieba(script)
+        feature = torch.tensor(pad_samples(encode_samples([[script_split_lst]])))
+
+        from script_score.lstm_network import SentimentNet
+        net = torch.load('/data/pythonProject/video_to_word/script_score/pkl/epoch52_test_auc_0.790_train_auc_0.815.pth',
+                         map_location='cpu')
+
+        with torch.no_grad():
+            score = net(feature)
+
+        score, level = log_and_map_min_max_score(score[0][1].item())
+
+        response["result"] = {"script_level": level, "script_score": score}
+        return response
+    except:
+        response["code"] = -1
+        response["success"] = False
+        response["message"] = traceback.format_exc()
+        return response
+
+
+if __name__ == '__main__':
+    text = "十六块二十六块,只要二十六块包邮到家,这么大一件派克服,现在不要两百,不要一百二十六块就给你包邮到家,真的太划算了,咱们工厂现在为了扩大销售渠道,所以特地拿出一批货在桃树上做活动,这款派克服寒气版型,特别时尚,抽绳收腰设计,修身显瘦,加绒内里还保暖毛领,精致又洋气,喜欢的朋友赶紧点击视频下方链接进入操作即可就可以购买啦。"
+    print(evaluate_script(text))

+ 31 - 0
script_score/lstm_network.py

@@ -0,0 +1,31 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+class SentimentNet(nn.Module):
+    def __init__(self, vocab_size, embed_size, num_hiddens, num_layers, bidirectional, weight, labels,
+                 use_gpu, **kwargs):
+        super(SentimentNet, self).__init__(**kwargs)
+        self.num_hiddens = num_hiddens
+        self.num_layers = num_layers
+        self.bidirectional = bidirectional
+        self.embedding = nn.Embedding.from_pretrained(weight)
+        self.embedding.weight.requires_grad = False
+        self.encoder = nn.LSTM(input_size=embed_size,
+                               hidden_size=self.num_hiddens,
+                               num_layers=num_layers,
+                               bidirectional=self.bidirectional,
+                               dropout=0.1)
+        if self.bidirectional:
+            self.decoder = nn.Linear(num_hiddens * 4, labels)
+        else:
+            self.decoder = nn.Linear(num_hiddens * 2, labels)
+
+    def forward(self, inputs):
+        embeddings = self.embedding(inputs)
+        states, hidden = self.encoder(embeddings.permute([1, 0, 2]))
+        encoding = torch.cat([states[0], states[-1]], dim=1)
+        outputs = self.decoder(encoding)
+        outputs = F.softmax(outputs)
+        return outputs

BIN
script_score/pkl/epoch52_test_auc_0.790_train_auc_0.815.pth


BIN
script_score/pkl/vocab.pkl


BIN
script_score/pkl/word_to_idx.pkl