put inference model

app.py

@@ -1,15 +1,281 @@
-import gradio as gr
+import datetime 
+import numpy as np
+import pandas as pd
+import re
+import json
+import os
+import glob
+
+import torch
+import torch.nn.functional as F
+from torch.optim import Adam
+from tqdm.notebook import tqdm
+from torch import nn
+from transformers import BertModel
+
+from transformers import AutoTokenizer
+
+import argparse
+
+def split_essay_to_sentence(origin_essay):
+    origin_essay_sentence = sum([[a.strip() for a in i.split('.')] for i in origin_essay.split('\n')], [])
+    essay_sent = [a for a in origin_essay_sentence if len(a) > 0]
+    return essay_sent
+
+def get_first_extraction(text_sentence):
+    row_dict = {}
+    for row in tqdm(text_sentence):
+        question = 'what is the feeling?'
+        answer = question_answerer(question=question, context=row)
+        row_dict[row] = answer
+    return row_dict
+
+
+def get_sent_labeldata():
+    label =pd.read_csv('./rawdata/sentimental_label.csv', encoding = 'cp949', header = None)
+    label[1] = label[1].apply(lambda x : re.findall(r'[가-힣]+', x)[0])
+    label_dict =label[label.index % 10 == 0].set_index(0).to_dict()[1]
+    emo2idx = {v : k for k, v in enumerate(label_dict.items())}
+    idx2emo = {v : k[1] for k, v in emo2idx.items()}
+    return emo2idx, idx2emo
+
+def load_model():
+    
+    class BertClassifier(nn.Module):
+
+        def __init__(self, dropout = 0.3):
+            super(BertClassifier, self).__init__()
+
+            self.bert= BertModel.from_pretrained('bert-base-multilingual-cased')
+            self.dropout = nn.Dropout(dropout)
+            self.linear = nn.Linear(768, 6)
+            self.relu = nn.ReLU()
+
+        def forward(self, input_id, mask):
+            _, pooled_output = self.bert(input_ids = input_id, attention_mask = mask, return_dict = False)
+            dropout_output = self.dropout(pooled_output)
+            linear_output = self.linear(dropout_output)
+            final_layer= self.relu(linear_output)
+
+            return final_layer
+
+
+    tokenizer = AutoTokenizer.from_pretrained('bert-base-multilingual-cased')
+    device = 'cuda' if torch.cuda.is_available() else 'cpu'
+    cls_model = BertClassifier()
+    criterion = nn.CrossEntropyLoss()
+    model_name = 'bert-base-multilingual-cased'
+    PATH = './model' + '/' + model_name + '_' + '2023102410'
+    print(PATH)
+    cls_model = torch.load(PATH)
+    #cls_model.load_state_dict(torch.load(PATH))
+    return tokenizer, cls_model
+
+
+class myDataset_for_infer(torch.utils.data.Dataset):
+    def __init__(self, X):
+        self.X = X
+
+    def __len__(self):
+        return len(self.X)
+
+    def __getitem__(self,idx):
+        sentences =  tokenizer(self.X[idx], return_tensors = 'pt', padding = 'max_length', max_length = 128, truncation = True)
+        return sentences
+    
+    
+def infer_data(model, main_feeling_keyword):
+    #ds = myDataset_for_infer()
+    df_infer = myDataset_for_infer(main_feeling_keyword)
+
+    infer_dataloader = torch.utils.data.DataLoader(df_infer, batch_size= 16)
+    device = 'cuda' if torch.cuda.is_available() else 'cpu'
+
+    if device == 'cuda':
+        model = model.cuda()
+
+    result_list = []
+    with torch.no_grad():
+        for idx, infer_input in tqdm(enumerate(infer_dataloader)):
+            mask = infer_input['attention_mask'].to(device)
+            input_id = infer_input['input_ids'].squeeze(1).to(device)
+
+            output = model(input_id, mask)
+            result = np.argmax(F.softmax(output, dim=0).cpu(), axis=1).numpy()
+            result_list.extend(result)
+    return result_list
+
+def get_word_emotion_pair(cls_model, origin_essay_sentence):
 
+    from konlpy.tag import Okt
 
+    okt = Okt()
+    #text = '나는 왜 엄마만 미워했을까'
+    def get_noun(text):
+      noun_list = [k for k, v  in okt.pos(text) if (v == 'Noun' and len(k) > 1)]
+      return noun_list
+    def get_adj(text):
+      adj_list = [k for k, v  in okt.pos(text) if (v == 'Adjective') and (len(k) > 1)]
+      return adj_list
+    def get_verb(text):
+      verb_list = [k for k, v  in okt.pos(text) if (v == 'Verb') and (len(k) > 1)]
+      return verb_list
 
-def greet(name):
-   return "Hello" + name + '!!'
+    result_list = infer_data(cls_model, origin_essay_sentence)
+    final_result = pd.DataFrame(data = {'text': origin_essay_sentence , 'label' : result_list})
+    final_result['emotion'] = final_result['label'].map(idx2emo)
+    final_result['noun_list'] = final_result['text'].map(get_noun)
+    final_result['adj_list'] = final_result['text'].map(get_adj)
+    final_result['verb_list'] = final_result['text'].map(get_verb)
+    final_result['title'] = 'none'
+    file_made_dt = datetime.datetime.now()
+    file_made_dt_str = datetime.datetime.strftime(file_made_dt, '%Y%m%d_%H%M%d')
+    os.makedirs(f'./result/{file_made_dt_str}/', exist_ok = True)
+    final_result.to_csv(f"./result/{file_made_dt_str}/essay_result.csv", index = False)
 
-title = "moogeul-moogeul"
+    return final_result, file_made_dt_str
 
+
+
+def get_essay_base_analysis(file_made_dt_str):
+    essay1 = pd.read_csv(f"./result/{file_name_dt}/essay_result.csv")
+    essay1['noun_list_len'] = essay1['noun_list'].apply(lambda x : len(x))
+    essay1['noun_list_uniqlen'] = essay1['noun_list'].apply(lambda x : len(set(x)))
+    essay1['adj_list_len'] = essay1['adj_list'].apply(lambda x : len(x))
+    essay1['adj_list_uniqlen'] = essay1['adj_list'].apply(lambda x : len(set(x)))
+    essay1['vocab_all'] = essay1[['noun_list','adj_list']].apply(lambda x : sum((eval(x[0]),eval(x[1])), []), axis=1)
+    essay1['vocab_cnt'] = essay1['vocab_all'].apply(lambda x : len(x))
+    essay1['vocab_unique_cnt'] = essay1['vocab_all'].apply(lambda x : len(set(x)))
+    essay1['noun_list'] = essay1['noun_list'].apply(lambda x : eval(x))
+    essay1['adj_list'] = essay1['adj_list'].apply(lambda x : eval(x))
+    d = essay1.groupby('title')[['noun_list','adj_list']].sum([]).reset_index()
+    d['noun_cnt'] = d['noun_list'].apply(lambda x : len(set(x)))
+    d['adj_cnt'] = d['adj_list'].apply(lambda x : len(set(x)))
+
+    # 문장 기준 최고 감정
+    essay_summary =essay1.groupby(['title'])['emotion'].value_counts().unstack(level =1)
+
+    emo_vocab_dict = {}
+    for k, v in essay1[['emotion','noun_list']].values:
+      for vocab in v:
+        if (k, 'noun', vocab) not in emo_vocab_dict:
+          emo_vocab_dict[(k, 'noun', vocab)] = 0
+
+        emo_vocab_dict[(k, 'noun', vocab)] += 1
+
+    for k, v in essay1[['emotion','adj_list']].values:
+      for vocab in v:
+        if (k, 'adj', vocab) not in emo_vocab_dict:
+          emo_vocab_dict[(k, 'adj', vocab)] = 0
+
+        emo_vocab_dict[(k, 'adj', vocab)] += 1
+    vocab_emo_cnt_dict = {}
+    for k, v in essay1[['emotion','noun_list']].values:
+      for vocab in v:
+        if (vocab, 'noun') not in vocab_emo_cnt_dict:
+          vocab_emo_cnt_dict[('noun', vocab)] = {}
+        if k not in vocab_emo_cnt_dict[( 'noun', vocab)]:
+          vocab_emo_cnt_dict[( 'noun', vocab)][k] = 0
+
+        vocab_emo_cnt_dict[('noun', vocab)][k] += 1
+
+    for k, v in essay1[['emotion','adj_list']].values:
+      for vocab in v:
+        if ('adj', vocab) not in vocab_emo_cnt_dict:
+          vocab_emo_cnt_dict[( 'adj', vocab)] = {}
+        if k not in vocab_emo_cnt_dict[( 'adj', vocab)]:
+          vocab_emo_cnt_dict[( 'adj', vocab)][k] = 0
+
+        vocab_emo_cnt_dict[('adj', vocab)][k] += 1
+
+    vocab_emo_cnt_df = pd.DataFrame(vocab_emo_cnt_dict).T
+    vocab_emo_cnt_df['total'] = vocab_emo_cnt_df.sum(axis=1)
+    # 단어별 최고 감정 및 감정 개수
+    all_result=vocab_emo_cnt_df.sort_values(by = 'total', ascending = False)
+
+    # 단어별 최고 감정 및 감정 개수 , 형용사 포함 시
+    adj_result=vocab_emo_cnt_df.sort_values(by = 'total', ascending = False)
+
+    # 명사만 사용 시
+    noun_result=vocab_emo_cnt_df[vocab_emo_cnt_df.index.get_level_values(0) == 'noun'].sort_values(by = 'total', ascending = False)
+
+    final_file_name = f"essay_all_vocab_result.csv"
+    adj_file_name = f"essay_adj_vocab_result.csv"
+    noun_file_name = f"essay_noun_vocab_result.csv"
+    
+    os.makedirs(f'./result/{file_made_dt_str}/', exist_ok = True)
+    
+    final_result.to_csv(f"./result/{file_made_dt_str}/essay_all_vocab_result.csv", index = False)
+    adj_result.to_csv(f"./result/{file_made_dt_str}/essay_adj_vocab_result.csv", index = False)
+    noun_result.to_csv(f"./result/{file_made_dt_str}/essay_noun_vocab_result.csv", index = False)
+    
+    return final_result, adj_result, noun_result, essay_summary, file_made_dt_str
+
+
+from transformers import pipeline
+model_name = 'AlexKay/xlm-roberta-large-qa-multilingual-finedtuned-ru'
+question_answerer = pipeline("question-answering", model=model_name)
+
+class BertClassifier(nn.Module):
+
+    def __init__(self, dropout = 0.3):
+        super(BertClassifier, self).__init__()
+
+        self.bert= BertModel.from_pretrained('bert-base-multilingual-cased')
+        self.dropout = nn.Dropout(dropout)
+        self.linear = nn.Linear(768, 6)
+        self.relu = nn.ReLU()
+
+    def forward(self, input_id, mask):
+        _, pooled_output = self.bert(input_ids = input_id, attention_mask = mask, return_dict = False)
+        dropout_output = self.dropout(pooled_output)
+        linear_output = self.linear(dropout_output)
+        final_layer= self.relu(linear_output)
+
+        return final_layer
+
+        
+def all_process(origin_essay):
+    essay_sent =split_essay_to_sentence(origin_essay)
+    row_dict = {}
+    for row in tqdm(essay_sent):
+        question = 'what is the feeling?'
+        answer = question_answerer(question=question, context=row)
+        row_dict[row] = answer
+    emo2idx, idx2emo = get_sent_labeldata()
+    tokenizer, cls_model = load_model()
+    final_result, file_name_dt = get_word_emotion_pair(cls_model, essay_sent)
+    all_result, adj_result, noun_result, essay_summary, file_made_dt_str = get_essay_base_analysis(file_name_dt)
+    
+    summary_result = pd.concat([adj_result, noun_result]).fillna(0).sort_values(by = 'total', ascending = False).fillna(0).reset_index()[:30]
+    with open(f'./result/{file_name_dt}/summary.json','w') as f:
+        json.dump( essay_summary.to_json(),f)
+    with open(f'./result/{file_made_dt_str}/all_result.json','w') as f:
+        json.dump( all_result.to_json(),f)    
+    with open(f'./result/{file_made_dt_str}/adj_result.json','w') as f:
+        json.dump( adj_result.to_json(),f)  
+    with open(f'./result/{file_made_dt_str}/noun_result.json','w') as f:
+        json.dump( noun_result.to_json(),f)  
+    return essay_summary
+
+import gradio as gr
+outputs = [gr.Dataframe(row_count = (6, "dynamic"),
+                        col_count=(2, "dynamic"),
+                        label="Essay Summary based on Words")
+                        #headers=['type','word','슬픔', '분노', '기쁨', '불안', '상처', '당황', 'total'])
+    
+                        ]
+    
+    
+                        #row_count = (10, "dynamic"),
+                        #col_count=(9, "dynamic"),
+                        #label="Results",
+                        #headers=['type','word','슬픔', '분노', '기쁨', '불안', '상처', '당황', 'total'])
+          #]
+        
 iface = gr.Interface(
-   fn=greet,
+   fn=all_process,
    inputs = gr.Textbox(lines=2, placeholder= '당신의 글을 넣어보세요'),
-   outputs = gr.Textbox(lines=10, placeholder = '글 속에 있는 무의식을 찾아드립니다.')
+   outputs = outputs,
 )
-iface.launch(share =True)
+iface.launch(share =True)