重构 preprocess.py，增加两个情感分析模型的支持，优化情感得分计算逻辑，增强错误处理和日志打印

preprocess.py

@@ -1,6 +1,9 @@
 import re
 import sys
 import os
+import trace
+import traceback
+from typing import final
 import numpy as np
 from collections import defaultdict
 import pandas as pd
@@ -19,7 +22,7 @@ import akshare as ak
 
 from gensim.models import Word2Vec
 from transformers import pipeline, AutoTokenizer, AutoModelForSequenceClassification
-
+from transformers import BertTokenizer, BertForSequenceClassification
 
 
 
@@ -47,13 +50,12 @@ print("Is NPL GPU used Preprocessing.py:", spacy.prefer_gpu())
 
 
 # 使用合适的模型和tokenizer
-model_name = "ProsusAI/finbert"  # 选择合适的预训练模型
-tokenizer = AutoTokenizer.from_pretrained(model_name)
-sa_model = AutoModelForSequenceClassification.from_pretrained(model_name)
+tokenizer_one = AutoTokenizer.from_pretrained("ProsusAI/finbert")
+sa_model_one = AutoModelForSequenceClassification.from_pretrained("ProsusAI/finbert")
 
-# 初始化情感分析器
-sentiment_analyzer = pipeline('sentiment-analysis', model=sa_model, tokenizer=tokenizer)
 
+tokenizer_two = BertTokenizer.from_pretrained('yiyanghkust/finbert-tone')
+sa_model_two = BertForSequenceClassification.from_pretrained('yiyanghkust/finbert-tone',num_labels=3)
 
 
 
@@ -177,16 +179,28 @@ def process_entities(entities):
 def process_pos_tags(pos_tags):
     pos_counts = defaultdict(int)
     try:
+        # 确保 pos_tags 不为空且是有效的标记
+        if not pos_tags or not isinstance(pos_tags, (list, tuple)):
+            print(f"Invalid POS tags: {pos_tags}")
+            return np.zeros(1), []
+
+        # 安全地处理每个 POS 标记
         for pos in pos_tags:
-            pos_counts[pos[1]] += 1  # 使用POS标签（如NN, VB等）
+            if isinstance(pos, str) and pos:  # 确保是非空字符串
+                pos_counts[pos] += 1
+            elif isinstance(pos, (list, tuple)) and len(pos) > 1:  # 如果是元组/列表，取第二个元素
+                pos_counts[pos[1]] += 1
         
         # 将字典转化为有序的数组
         pos_types = sorted(pos_counts.keys())
+        if not pos_types:  # 如果没有有效的类型，返回零向量
+            print(f"No valid POS tags found: {pos_tags}")
+            return np.zeros(1), []
+        
         counts = np.array([pos_counts[pos] for pos in pos_types])
     except Exception as e:
-        print(f"Error in process_pos_tags: {str(e)}")
-        counts = np.zeros(len(pos_tags))
-        pos_types = []
+        print(f"Error in process_pos_tags: {str(e)} for POS tags: {pos_tags}")
+        return np.zeros(1), []
 
     return counts, pos_types
 
@@ -211,14 +225,69 @@ def get_document_vector(words, model = word2vec_model):
 # 函数：获取情感得分
 def get_sentiment_score(text):
     try:
-        # 直接将原始文本传递给 sentiment_analyzer，它会自动处理 tokenization
-        result = sentiment_analyzer(text, truncation=True, max_length=512)[0]
-        score = result['score'] if result['label'] == 'positive' else -result['score']
+        import torch
+
+        # 获取第一个模型的结果 (ProsusAI/finbert)
+        # result_one = sentiment_analyzer_one(text, truncation=True, max_length=512)[0]
+        # 获取模型输出
+        with torch.no_grad():
+            outputs_one = sa_model_one(**tokenizer_one(text, return_tensors="pt", truncation=False))
+            predictions_one = torch.nn.functional.softmax(outputs_one.logits, dim=-1)
+
+
+            outputs_two = sa_model_two(**tokenizer_two(text, return_tensors="pt", truncation=False))
+            predictions_two = torch.nn.functional.softmax(outputs_two.logits, dim=-1)
+
+        # 获取所有标签的概率
+        scores_one = predictions_one[0].tolist()
+        scores_two = predictions_two[0].tolist()
+
+        # 获取标签映射
+        # labels_one = sa_model_one.config.id2label
+        # labels_two = sa_model_two.config.id2label
+
+        # 打印所有标签的概率
+        score_one_positive = scores_one[0]
+        score_one_negative = scores_one[1]
+        score_one_neutral = scores_one[2]
+
+
+        final_score_one = 0.0
+        final_score_one += score_one_positive
+        final_score_one -= score_one_negative
+        if score_one_positive > score_one_negative:
+            final_score_one += score_one_neutral
+        else:
+            final_score_one -= score_one_neutral
+
+        final_score_one = max(-1.0, min(1.0, final_score_one))
+
+        score_two_neutral = scores_two[0]
+        score_two_positive = scores_two[1]
+        score_two_negative = scores_two[2]
+
+        final_score_two = 0.0
+        final_score_two += score_two_positive
+        final_score_two -= score_two_negative
+        if score_two_positive > score_two_negative:
+            final_score_two += score_two_neutral
+        else:
+            final_score_two -= score_two_neutral
+        
+
+        # 将两个模型的得分组合（加权平均）
+        final_score = np.average([final_score_one, final_score_two], weights=[0.3, 0.7])
+        
+        # 确保最终得分在 [-1, 1] 范围内
+        final_score = np.clip(final_score, -1.0, 1.0)
+        
+        return final_score
+
     except Exception as e:
         print(f"Error in get_sentiment_score for text: {text[:50]}... Error: {str(e)}")
-        score = 0.0
-
-    return score
+        traceback.print_exc()
+        return 0.0
+    
 
 
 
@@ -386,7 +455,7 @@ def lemmatized_entry(entry):
 nlp = spacy.load("en_core_web_md")
 
 # 检查是否使用 GPU
-print("Is NPL GPU used Lemmatized:", spacy.prefer_gpu())
+# print("Is NPL GPU used Lemmatized:", spacy.prefer_gpu())
 
 
 
@@ -521,7 +590,7 @@ def lemmatize_text(text):
 nlp = spacy.load("en_core_web_md")
 
 # 检查是否使用 GPU
-print("Is NPL GPU used Enchance_text.py:", spacy.prefer_gpu())
+# print("Is NPL GPU used Enchance_text.py:", spacy.prefer_gpu())