Update app.py

app.py

@@ -13,7 +13,7 @@ from tqdm import tqdm
 from torch import nn
 from transformers import BertModel
 
-from transformers import AutoTokenizer, AutoModelForSequenceClassification
+from transformers import AutoTokenizer
 
 import argparse
 
@@ -39,37 +39,37 @@ def get_sent_labeldata():
     idx2emo = {v : k[1] for k, v in emo2idx.items()}
     return emo2idx, idx2emo
 
-def load_model():
+# def load_model():
     
-    class BertClassifier(nn.Module):
+#     class BertClassifier(nn.Module):
 
-        def __init__(self, dropout = 0.3):
-            super(BertClassifier, self).__init__()
+#         def __init__(self, dropout = 0.3):
+#             super(BertClassifier, self).__init__()
 
-            self.bert= BertModel.from_pretrained('bert-base-multilingual-cased')
-            self.dropout = nn.Dropout(dropout)
-            self.linear = nn.Linear(768, 6)
-            self.relu = nn.ReLU()
+#             self.bert= BertModel.from_pretrained('bert-base-multilingual-cased')
+#             self.dropout = nn.Dropout(dropout)
+#             self.linear = nn.Linear(768, 6)
+#             self.relu = nn.ReLU()
 
-        def forward(self, input_id, mask):
-            _, pooled_output = self.bert(input_ids = input_id, attention_mask = mask, return_dict = False)
-            dropout_output = self.dropout(pooled_output)
-            linear_output = self.linear(dropout_output)
-            final_layer= self.relu(linear_output)
+#         def forward(self, input_id, mask):
+#             _, pooled_output = self.bert(input_ids = input_id, attention_mask = mask, return_dict = False)
+#             dropout_output = self.dropout(pooled_output)
+#             linear_output = self.linear(dropout_output)
+#             final_layer= self.relu(linear_output)
 
-            return final_layer
+#             return final_layer
 
 
-    tokenizer = AutoTokenizer.from_pretrained('bert-base-multilingual-cased')
-    device = 'cuda' if torch.cuda.is_available() else 'cpu'
-    cls_model = BertClassifier()
-    criterion = nn.CrossEntropyLoss()
-    model_name = 'bert-base-multilingual-cased'
-    PATH = './model' + '/' + model_name + '_' + '2023102410'
-    print(PATH)
-    cls_model = torch.load(PATH)
-    #cls_model.load_state_dict(torch.load(PATH))
-    return tokenizer, cls_model
+#     tokenizer = AutoTokenizer.from_pretrained('bert-base-multilingual-cased')
+#     device = 'cuda' if torch.cuda.is_available() else 'cpu'
+#     cls_model = BertClassifier()
+#     criterion = nn.CrossEntropyLoss()
+#     model_name = 'bert-base-multilingual-cased'
+#     PATH = './model' + '/' + model_name + '_' + '2023102410'
+#     print(PATH)
+#     cls_model = torch.load(PATH)
+#     #cls_model.load_state_dict(torch.load(PATH))
+#     return tokenizer, cls_model
 
 
 class myDataset_for_infer(torch.utils.data.Dataset):
@@ -80,7 +80,7 @@ class myDataset_for_infer(torch.utils.data.Dataset):
         return len(self.X)
 
     def __getitem__(self,idx):
-        sentences =  tokenizer(self.X[idx], return_tensors = 'pt', padding = 'max_length', max_length = 128, truncation = True)
+        sentences =  tokenizer(self.X[idx], return_tensors = 'pt', padding = 'max_length', max_length = 96, truncation = True)
         return sentences
     
     
@@ -100,33 +100,32 @@ def infer_data(model, main_feeling_keyword):
             mask = infer_input['attention_mask'].to(device)
             input_id = infer_input['input_ids'].squeeze(1).to(device)
 
-            output = model(input_id, mask)
-            result = np.argmax(F.softmax(output, dim=0).cpu(), axis=1).numpy()
+            output = clsmodel(input_id, mask)
+            result = np.argmax(output.logits, axis=1).numpy()
             result_list.extend(result)
     return result_list
 
-def get_word_emotion_pair(cls_model, origin_essay_sentence):
-
-    from konlpy.tag import Okt
+def get_word_emotion_pair(cls_model, origin_essay_sentence, idx2emo):
 
-    okt = Okt()
-    #text = '나는 왜 엄마만 미워했을까'
-    def get_noun(text):
-      noun_list = [k for k, v  in okt.pos(text) if (v == 'Noun' and len(k) > 1)]
-      return noun_list
-    def get_adj(text):
-      adj_list = [k for k, v  in okt.pos(text) if (v == 'Adjective') and (len(k) > 1)]
-      return adj_list
-    def get_verb(text):
-      verb_list = [k for k, v  in okt.pos(text) if (v == 'Verb') and (len(k) > 1)]
-      return verb_list
+    import re
+    def get_noun(sent):
+        return [re.sub(r'[을를]+', '', vocab) for (vocab, pos) in nlp(sent) if len(vocab) > 1 and pos == 'NOUN']
+    def get_adj(sent):
+        return [re.sub(r'[을를]+', '', vocab) for (vocab, pos) in nlp(sent) if len(vocab) > 1 and pos == 'ADJ']
+    def get_verb(sent):
+        return [re.sub(r'[을를]+', '', vocab) for (vocab, pos) in nlp(sent) if len(vocab) > 1 and pos == 'VERB']
 
     result_list = infer_data(cls_model, origin_essay_sentence)
     final_result = pd.DataFrame(data = {'text': origin_essay_sentence , 'label' : result_list})
     final_result['emotion'] = final_result['label'].map(idx2emo)
+    
+    nlp=lambda x:[(x[t["start"]:t["end"]],t["entity_group"]) for t in pipeline(x)]
+    #essay_sent_pos = [nlp(i) for i in tqdm(essay_sent)]
+    #final_result['text_pos'] = essay_sent_pos
     final_result['noun_list'] = final_result['text'].map(get_noun)
     final_result['adj_list'] = final_result['text'].map(get_adj)
     final_result['verb_list'] = final_result['text'].map(get_verb)
+    
     final_result['title'] = 'none'
     file_made_dt = datetime.datetime.now()
     file_made_dt_str = datetime.datetime.strftime(file_made_dt, '%Y%m%d_%H%M%d')
@@ -136,7 +135,6 @@ def get_word_emotion_pair(cls_model, origin_essay_sentence):
     return final_result, file_made_dt_str
 
 
-
 def get_essay_base_analysis(file_made_dt_str):
     essay1 = pd.read_csv(f"./result/{file_name_dt}/essay_result.csv")
     essay1['noun_list_len'] = essay1['noun_list'].apply(lambda x : len(x))
@@ -213,28 +211,21 @@ def get_essay_base_analysis(file_made_dt_str):
 
 
 from transformers import pipeline
-model_name = 'AlexKay/xlm-roberta-large-qa-multilingual-finedtuned-ru'
+#model_name = 'AlexKay/xlm-roberta-large-qa-multilingual-finedtuned-ru'
+model_name = 'monologg/koelectra-base-v2-finetuned-korquad'
 question_answerer = pipeline("question-answering", model=model_name)
 
-class BertClassifier(nn.Module):
-
-    def __init__(self, dropout = 0.3):
-        super(BertClassifier, self).__init__()
-
-        self.bert= BertModel.from_pretrained('bert-base-multilingual-cased')
-        self.dropout = nn.Dropout(dropout)
-        self.linear = nn.Linear(768, 6)
-        self.relu = nn.ReLU()
+from transformers import AutoTokenizer,AutoModelForTokenClassification,TokenClassificationPipeline
+tokenizer=AutoTokenizer.from_pretrained("KoichiYasuoka/roberta-large-korean-upos")
+posmodel=AutoModelForTokenClassification.from_pretrained("KoichiYasuoka/roberta-large-korean-upos")
 
-    def forward(self, input_id, mask):
-        _, pooled_output = self.bert(input_ids = input_id, attention_mask = mask, return_dict = False)
-        dropout_output = self.dropout(pooled_output)
-        linear_output = self.linear(dropout_output)
-        final_layer= self.relu(linear_output)
+pipeline=TokenClassificationPipeline(tokenizer=tokenizer,model=posmodel,aggregation_strategy="simple")
+nlp=lambda x:[(x[t["start"]:t["end"]],t["entity_group"]) for t in pipeline(x)]
 
-        return final_layer
+from transformers import AutoModelForSequenceClassification
+device = 'cuda' if torch.cuda.is_available() else 'cpu'
+tokenizer = AutoTokenizer.from_pretrained('bert-base-multilingual-cased')
 
-        
 def all_process(origin_essay):
     essay_sent =split_essay_to_sentence(origin_essay)
     row_dict = {}
@@ -243,11 +234,12 @@ def all_process(origin_essay):
         answer = question_answerer(question=question, context=row)
         row_dict[row] = answer
     emo2idx, idx2emo = get_sent_labeldata()
-    #tokenizer, cls_model = load_model()
     tokenizer = AutoTokenizer.from_pretrained('bert-base-multilingual-cased')
-    cls_model = AutoModelForSequenceClassification.from_pretrained('bert-base-multilingual-cased')
+    #cls_model = AutoModelForSequenceClassification.from_pretrain ed('seriouspark/bert-base-multilingual-cased-finetuning-sentimental-6label')
+    cls_model = AutoModelForSequenceClassification.from_pretrained('bert-base-multilingual-cased', num_labels = 6)
+    
     final_result, file_name_dt = get_word_emotion_pair(cls_model, essay_sent)
-    all_result, adj_result, noun_result, essay_summary, file_made_dt_str = get_essay_base_analysis(file_name_dt)
+    all_result, adj_\result, noun_result, essay_summary, file_made_dt_str = get_essay_base_analysis(file_name_dt)
     
     summary_result = pd.concat([adj_result, noun_result]).fillna(0).sort_values(by = 'total', ascending = False).fillna(0).reset_index()[:30]
     with open(f'./result/{file_name_dt}/summary.json','w') as f:
@@ -280,4 +272,4 @@ iface = gr.Interface(
    inputs = gr.Textbox(lines=2, placeholder= '당신의 글을 넣어보세요'),
    outputs = outputs,
 )
-iface.launch(share =True)
+iface.launch(share=True)