Update Model/MultimodelNER/VLSP2021/train_umt_2021.py

Model/MultimodelNER/VLSP2021/train_umt_2021.py

@@ -1,351 +1,351 @@
-import os
-import sys
-
-os.environ["CUDA_VISIBLE_DEVICES"] = "0"
-import argparse
-
-import logging
-import random
-import numpy as np
-import torch
-import torch.nn.functional as F
-from transformers import AutoTokenizer, BertConfig
-from Model.MultimodelNER.UMT import UMT
-from Model.MultimodelNER import resnet as resnet
-from Model.MultimodelNER.resnet_utils import myResnet
-from Model.MultimodelNER.VLSP2021.dataset_roberta import convert_mm_examples_to_features, MNERProcessor_2021
-from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler,
-                              TensorDataset)
-from pytorch_pretrained_bert.optimization import BertAdam, warmup_linear
-from Model.MultimodelNER.ner_evaluate import evaluate_each_class,evaluate
-from seqeval.metrics import classification_report
-from tqdm import tqdm, trange
-import json
-from Model.MultimodelNER.predict import convert_mm_examples_to_features_predict, get_test_examples_predict
-from Model.MultimodelNER.Ner_processing import *
-CONFIG_NAME = 'bert_config.json'
-WEIGHTS_NAME = 'pytorch_model.bin'
-
-logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
-                    datefmt='%m/%d/%Y %H:%M:%S',
-                    level=logging.INFO)
-logger = logging.getLogger(__name__)
-parser = argparse.ArgumentParser()
-## Required parameters
-parser.add_argument("--negative_rate",
-                    default=16,
-                    type=int,
-                    help="the negative samples rate")
-
-parser.add_argument('--lamb',
-                    default=0.62,
-                    type=float)
-
-parser.add_argument('--temp',
-                    type=float,
-                    default=0.179,
-                    help="parameter for CL training")
-
-parser.add_argument('--temp_lamb',
-                    type=float,
-                    default=0.7,
-                    help="parameter for CL training")
-
-parser.add_argument("--data_dir",
-                    default='./data/twitter2017',
-                    type=str,
-
-                    help="The input data dir. Should contain the .tsv files (or other data files) for the task.")
-parser.add_argument("--bert_model", default='vinai/phobert-base-v2', type=str)
-parser.add_argument("--task_name",
-                    default='sonba',
-                    type=str,
-
-                    help="The name of the task to train.")
-parser.add_argument("--output_dir",
-                    default='E:/demo_datn/pythonProject1/Model/MultimodelNER/VLSP2021/best_model/',
-                    type=str,
-                    help="The output directory where the model predictions and checkpoints will be written.")
-
-## Other parameters
-parser.add_argument("--cache_dir",
-                    default="",
-                    type=str,
-                    help="Where do you want to store the pre-trained models downloaded from s3")
-
-parser.add_argument("--max_seq_length",
-                    default=128,
-                    type=int,
-                    help="The maximum total input sequence length after WordPiece tokenization. \n"
-                         "Sequences longer than this will be truncated, and sequences shorter \n"
-                         "than this will be padded.")
-
-parser.add_argument("--do_train",
-                    action='store_true',
-                    help="Whether to run training.")
-
-parser.add_argument("--do_eval",
-                    action='store_true',
-                    help="Whether to run eval on the dev set.")
-
-parser.add_argument("--do_lower_case",
-                    action='store_true',
-                    help="Set this flag if you are using an uncased model.")
-
-parser.add_argument("--train_batch_size",
-                    default=64,
-                    type=int,
-                    help="Total batch size for training.")
-
-parser.add_argument("--eval_batch_size",
-                    default=16,
-                    type=int,
-                    help="Total batch size for eval.")
-
-parser.add_argument("--learning_rate",
-                    default=5e-5,
-                    type=float,
-                    help="The initial learning rate for Adam.")
-
-parser.add_argument("--num_train_epochs",
-                    default=12.0,
-                    type=float,
-                    help="Total number of training epochs to perform.")
-
-parser.add_argument("--warmup_proportion",
-                    default=0.1,
-                    type=float,
-                    help="Proportion of training to perform linear learning rate warmup for. "
-                         "E.g., 0.1 = 10%% of training.")
-
-parser.add_argument("--no_cuda",
-                    action='store_true',
-                    help="Whether not to use CUDA when available")
-
-parser.add_argument("--local_rank",
-                    type=int,
-                    default=-1,
-                    help="local_rank for distributed training on gpus")
-
-parser.add_argument('--seed',
-                    type=int,
-                    default=37,
-                    help="random seed for initialization")
-
-parser.add_argument('--gradient_accumulation_steps',
-                    type=int,
-                    default=1,
-                    help="Number of updates steps to accumulate before performing a backward/update pass.")
-
-parser.add_argument('--fp16',
-                    action='store_true',
-                    help="Whether to use 16-bit float precision instead of 32-bit")
-
-parser.add_argument('--loss_scale',
-                    type=float, default=0,
-                    help="Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n"
-                         "0 (default value): dynamic loss scaling.\n"
-                         "Positive power of 2: static loss scaling value.\n")
-
-parser.add_argument('--mm_model', default='MTCCMBert', help='model name')  # 'MTCCMBert', 'NMMTCCMBert'
-parser.add_argument('--layer_num1', type=int, default=1, help='number of txt2img layer')
-parser.add_argument('--layer_num2', type=int, default=1, help='number of img2txt layer')
-parser.add_argument('--layer_num3', type=int, default=1, help='number of txt2txt layer')
-parser.add_argument('--fine_tune_cnn', action='store_true', help='fine tune pre-trained CNN if True')
-parser.add_argument('--resnet_root', default='E:/demo_datn/pythonProject1/Model/Resnet/', help='path the pre-trained cnn models')
-parser.add_argument('--crop_size', type=int, default=224, help='crop size of image')
-parser.add_argument('--path_image', default='E:/demo_datn/pythonProject1/Model/MultimodelNER/VLSP2021/Image', help='path to images')
-# parser.add_argument('--mm_model', default='TomBert', help='model name') #
-parser.add_argument('--server_ip', type=str, default='', help="Can be used for distant debugging.")
-parser.add_argument('--server_port', type=str, default='', help="Can be used for distant debugging.")
-args = parser.parse_args()
-
-
-
-processors = {
-    "twitter2015": MNERProcessor_2021,
-    "twitter2017": MNERProcessor_2021,
-    "sonba": MNERProcessor_2021
-}
-
-
-
-random.seed(args.seed)
-np.random.seed(args.seed)
-torch.manual_seed(args.seed)
-
-
-task_name = args.task_name.lower()
-
-
-
-processor = processors[task_name]()
-label_list = processor.get_labels()
-auxlabel_list = processor.get_auxlabels()
-num_labels = len(label_list) + 1  # label 0 corresponds to padding, label in label_list starts from 1
-auxnum_labels = len(auxlabel_list) + 1  # label 0 corresponds to padding, label in label_list starts from 1
-
-start_label_id = processor.get_start_label_id()
-stop_label_id = processor.get_stop_label_id()
-
-# ''' initialization of our conversion matrix, in our implementation, it is a 7*12 matrix initialized as follows:
-trans_matrix = np.zeros((auxnum_labels, num_labels), dtype=float)
-trans_matrix[0, 0] = 1  # pad to pad
-trans_matrix[1, 1] = 1  # O to O
-trans_matrix[2, 2] = 0.25  # B to B-MISC
-trans_matrix[2, 4] = 0.25  # B to B-PER
-trans_matrix[2, 6] = 0.25  # B to B-ORG
-trans_matrix[2, 8] = 0.25  # B to B-LOC
-trans_matrix[3, 3] = 0.25  # I to I-MISC
-trans_matrix[3, 5] = 0.25  # I to I-PER
-trans_matrix[3, 7] = 0.25  # I to I-ORG
-trans_matrix[3, 9] = 0.25  # I to I-LOC
-trans_matrix[4, 10] = 1  # X to X
-trans_matrix[5, 11] = 1  # [CLS] to [CLS]
-trans_matrix[6, 12] = 1  # [SEP] to [SEP]
-'''
-trans_matrix = np.zeros((num_labels, auxnum_labels), dtype=float)
-trans_matrix[0,0]=1 # pad to pad
-trans_matrix[1,1]=1
-trans_matrix[2,2]=1
-trans_matrix[4,2]=1
-trans_matrix[6,2]=1
-trans_matrix[8,2]=1
-trans_matrix[3,3]=1
-trans_matrix[5,3]=1
-trans_matrix[7,3]=1
-trans_matrix[9,3]=1
-trans_matrix[10,4]=1
-trans_matrix[11,5]=1
-trans_matrix[12,6]=1
-'''
-device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-
-tokenizer = AutoTokenizer.from_pretrained(args.bert_model, do_lower_case=args.do_lower_case)
-
-
-
-net = getattr(resnet, 'resnet152')()
-net.load_state_dict(torch.load(os.path.join(args.resnet_root, 'resnet152.pth')))
-encoder = myResnet(net, args.fine_tune_cnn, device)
-
-
-output_model_file = os.path.join(args.output_dir, WEIGHTS_NAME)
-# output_config_file = os.path.join(args.output_dir, CONFIG_NAME)
-output_encoder_file = os.path.join(args.output_dir, "pytorch_encoder.bin")
-
-temp = args.temp
-temp_lamb = args.temp_lamb
-lamb = args.lamb
-negative_rate = args.negative_rate
-# # loadmodel
-#     model = UMT.from_pretrained(args.bert_model,
-#                                 cache_dir=args.cache_dir, layer_num1=args.layer_num1,
-#                                 layer_num2=args.layer_num2,
-#                                 layer_num3=args.layer_num3,
-#                                 num_labels_=num_labels, auxnum_labels=auxnum_labels)
-#     model.load_state_dict(torch.load(output_model_file,map_location=torch.device('cpu')))
-#     model.to(device)
-#     encoder_state_dict = torch.load(output_encoder_file,map_location=torch.device('cpu'))
-#     encoder.load_state_dict(encoder_state_dict)
-#     encoder.to(device)
-#     print(model)
-
-def load_model(output_model_file, output_encoder_file,encoder,num_labels,auxnum_labels):
-    model = UMT.from_pretrained(args.bert_model,
-                                cache_dir=args.cache_dir, layer_num1=args.layer_num1,
-                                layer_num2=args.layer_num2,
-                                layer_num3=args.layer_num3,
-                                num_labels_=num_labels, auxnum_labels=auxnum_labels)
-    model.load_state_dict(torch.load(output_model_file, map_location=torch.device('cpu')))
-    model.to(device)
-    encoder_state_dict = torch.load(output_encoder_file, map_location=torch.device('cpu'))
-    encoder.load_state_dict(encoder_state_dict)
-    encoder.to(device)
-    return model, encoder
-
-model_umt,encoder_umt=load_model(output_model_file, output_encoder_file,encoder,num_labels,auxnum_labels)
-#
-#     # sentence = 'Thương biết_mấy những Thuận, những Liên, những Luận, Xuân, Nghĩa mỗi người một hoàn_cảnh nhưng đều rất giống nhau: rất ham học, rất cố_gắng để đạt mức hiểu biết cao nhất.'
-#     # image_path = '/kaggle/working/data/014715.jpg'
-#     # # crop_size = 224'
-path_image='E:\demo_datn\pythonProject1\Model\MultimodelNER\VLSP2021\Image'
-trans_matrix = np.zeros((auxnum_labels,num_labels), dtype=float)
-trans_matrix[0,0]=1 # pad to pad
-trans_matrix[1,1]=1 # O to O
-trans_matrix[2,2]=0.25 # B to B-MISC
-trans_matrix[2,4]=0.25 # B to B-PER
-trans_matrix[2,6]=0.25 # B to B-ORG
-trans_matrix[2,8]=0.25 # B to B-LOC
-trans_matrix[3,3]=0.25 # I to I-MISC
-trans_matrix[3,5]=0.25 # I to I-PER
-trans_matrix[3,7]=0.25 # I to I-ORG
-trans_matrix[3,9]=0.25 # I to I-LOC
-trans_matrix[4,10]=1   # X to X
-trans_matrix[5,11]=1   # [CLS] to [CLS]
-trans_matrix[6,12]=1   # [SE
-def predict(model_umt, encoder_umt, eval_examples, tokenizer, device,path_image,trans_matrix):
-
-    features = convert_mm_examples_to_features_predict(eval_examples, 256, tokenizer, 224,path_image)
-
-    input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
-    input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long)
-    added_input_mask = torch.tensor([f.added_input_mask for f in features], dtype=torch.long)
-    segment_ids = torch.tensor([f.segment_ids for f in features], dtype=torch.long)
-    img_feats = torch.stack([f.img_feat for f in features])
-    print(img_feats)
-    eval_data = TensorDataset(input_ids, input_mask, added_input_mask, segment_ids, img_feats)
-    eval_sampler = SequentialSampler(eval_data)
-    eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=16)
-
-    model_umt.eval()
-    encoder_umt.eval()
-
-    y_pred = []
-    label_map = {i: label for i, label in enumerate(label_list, 1)}
-    label_map[0] = "<pad>"
-
-    for input_ids, input_mask, added_input_mask, segment_ids, img_feats in tqdm(eval_dataloader, desc="Evaluating"):
-        input_ids = input_ids.to(device)
-        input_mask = input_mask.to(device)
-        added_input_mask = added_input_mask.to(device)
-        segment_ids = segment_ids.to(device)
-        img_feats = img_feats.to(device)
-
-        with torch.no_grad():
-            imgs_f, img_mean, img_att = encoder_umt(img_feats)
-            predicted_label_seq_ids = model_umt(input_ids, segment_ids, input_mask, added_input_mask, img_att,
-                                                trans_matrix)
-
-        logits = predicted_label_seq_ids
-        input_mask = input_mask.to('cpu').numpy()
-
-        for i, mask in enumerate(input_mask):
-            temp_1 = []
-            for j, m in enumerate(mask):
-                if j == 0:
-                    continue
-                if m:
-                    if label_map[logits[i][j]] not in ["<pad>", "<s>", "</s>", "X"]:
-                        temp_1.append(label_map[logits[i][j]])
-                else:
-                    break
-            y_pred.append(temp_1)
-
-    a = eval_examples[0].text_a.split(" ")
-
-    return y_pred, a
-
-# eval_examples = get_test_examples_predict('E:/demo_datn/pythonProject1/Model/MultimodelNER/VLSP2021/Filetxt/')
-# y_pred, a = predict(model_umt, encoder_umt, eval_examples, tokenizer, device,path_image,trans_matrix)
-# print(y_pred)
-# print(a)
-# formatted_output = format_predictions(a, y_pred[0])
-#
-# final= process_predictions(formatted_output)
-# final2= combine_entities(final)
-# print(final2)
-# final3= remove_B_prefix(final2)
-# final4=combine_i_tags(final3)
-# print(final3)
-
+import os
+import sys
+
+os.environ["CUDA_VISIBLE_DEVICES"] = "0"
+import argparse
+
+import logging
+import random
+import numpy as np
+import torch
+import torch.nn.functional as F
+from transformers import AutoTokenizer, BertConfig
+from Model.MultimodelNER.UMT import UMT
+from Model.MultimodelNER import resnet as resnet
+from Model.MultimodelNER.resnet_utils import myResnet
+from Model.MultimodelNER.VLSP2021.dataset_roberta import convert_mm_examples_to_features, MNERProcessor_2021
+from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler,
+                              TensorDataset)
+from pytorch_pretrained_bert.optimization import BertAdam, warmup_linear
+from Model.MultimodelNER.ner_evaluate import evaluate_each_class,evaluate
+from seqeval.metrics import classification_report
+from tqdm import tqdm, trange
+import json
+from Model.MultimodelNER.predict import convert_mm_examples_to_features_predict, get_test_examples_predict
+from Model.MultimodelNER.Ner_processing import *
+CONFIG_NAME = 'bert_config.json'
+WEIGHTS_NAME = 'pytorch_model.bin'
+
+logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
+                    datefmt='%m/%d/%Y %H:%M:%S',
+                    level=logging.INFO)
+logger = logging.getLogger(__name__)
+parser = argparse.ArgumentParser()
+## Required parameters
+parser.add_argument("--negative_rate",
+                    default=16,
+                    type=int,
+                    help="the negative samples rate")
+
+parser.add_argument('--lamb',
+                    default=0.62,
+                    type=float)
+
+parser.add_argument('--temp',
+                    type=float,
+                    default=0.179,
+                    help="parameter for CL training")
+
+parser.add_argument('--temp_lamb',
+                    type=float,
+                    default=0.7,
+                    help="parameter for CL training")
+
+parser.add_argument("--data_dir",
+                    default='./data/twitter2017',
+                    type=str,
+
+                    help="The input data dir. Should contain the .tsv files (or other data files) for the task.")
+parser.add_argument("--bert_model", default='vinai/phobert-base-v2', type=str)
+parser.add_argument("--task_name",
+                    default='sonba',
+                    type=str,
+
+                    help="The name of the task to train.")
+parser.add_argument("--output_dir",
+                    default='Model/MultimodelNER/VLSP2021/best_model/',
+                    type=str,
+                    help="The output directory where the model predictions and checkpoints will be written.")
+
+## Other parameters
+parser.add_argument("--cache_dir",
+                    default="",
+                    type=str,
+                    help="Where do you want to store the pre-trained models downloaded from s3")
+
+parser.add_argument("--max_seq_length",
+                    default=128,
+                    type=int,
+                    help="The maximum total input sequence length after WordPiece tokenization. \n"
+                         "Sequences longer than this will be truncated, and sequences shorter \n"
+                         "than this will be padded.")
+
+parser.add_argument("--do_train",
+                    action='store_true',
+                    help="Whether to run training.")
+
+parser.add_argument("--do_eval",
+                    action='store_true',
+                    help="Whether to run eval on the dev set.")
+
+parser.add_argument("--do_lower_case",
+                    action='store_true',
+                    help="Set this flag if you are using an uncased model.")
+
+parser.add_argument("--train_batch_size",
+                    default=64,
+                    type=int,
+                    help="Total batch size for training.")
+
+parser.add_argument("--eval_batch_size",
+                    default=16,
+                    type=int,
+                    help="Total batch size for eval.")
+
+parser.add_argument("--learning_rate",
+                    default=5e-5,
+                    type=float,
+                    help="The initial learning rate for Adam.")
+
+parser.add_argument("--num_train_epochs",
+                    default=12.0,
+                    type=float,
+                    help="Total number of training epochs to perform.")
+
+parser.add_argument("--warmup_proportion",
+                    default=0.1,
+                    type=float,
+                    help="Proportion of training to perform linear learning rate warmup for. "
+                         "E.g., 0.1 = 10%% of training.")
+
+parser.add_argument("--no_cuda",
+                    action='store_true',
+                    help="Whether not to use CUDA when available")
+
+parser.add_argument("--local_rank",
+                    type=int,
+                    default=-1,
+                    help="local_rank for distributed training on gpus")
+
+parser.add_argument('--seed',
+                    type=int,
+                    default=37,
+                    help="random seed for initialization")
+
+parser.add_argument('--gradient_accumulation_steps',
+                    type=int,
+                    default=1,
+                    help="Number of updates steps to accumulate before performing a backward/update pass.")
+
+parser.add_argument('--fp16',
+                    action='store_true',
+                    help="Whether to use 16-bit float precision instead of 32-bit")
+
+parser.add_argument('--loss_scale',
+                    type=float, default=0,
+                    help="Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n"
+                         "0 (default value): dynamic loss scaling.\n"
+                         "Positive power of 2: static loss scaling value.\n")
+
+parser.add_argument('--mm_model', default='MTCCMBert', help='model name')  # 'MTCCMBert', 'NMMTCCMBert'
+parser.add_argument('--layer_num1', type=int, default=1, help='number of txt2img layer')
+parser.add_argument('--layer_num2', type=int, default=1, help='number of img2txt layer')
+parser.add_argument('--layer_num3', type=int, default=1, help='number of txt2txt layer')
+parser.add_argument('--fine_tune_cnn', action='store_true', help='fine tune pre-trained CNN if True')
+parser.add_argument('--resnet_root', default='Model/Resnet/', help='path the pre-trained cnn models')
+parser.add_argument('--crop_size', type=int, default=224, help='crop size of image')
+parser.add_argument('--path_image', default='Model/MultimodelNER/VLSP2021/Image', help='path to images')
+# parser.add_argument('--mm_model', default='TomBert', help='model name') #
+parser.add_argument('--server_ip', type=str, default='', help="Can be used for distant debugging.")
+parser.add_argument('--server_port', type=str, default='', help="Can be used for distant debugging.")
+args = parser.parse_args()
+
+
+
+processors = {
+    "twitter2015": MNERProcessor_2021,
+    "twitter2017": MNERProcessor_2021,
+    "sonba": MNERProcessor_2021
+}
+
+
+
+random.seed(args.seed)
+np.random.seed(args.seed)
+torch.manual_seed(args.seed)
+
+
+task_name = args.task_name.lower()
+
+
+
+processor = processors[task_name]()
+label_list = processor.get_labels()
+auxlabel_list = processor.get_auxlabels()
+num_labels = len(label_list) + 1  # label 0 corresponds to padding, label in label_list starts from 1
+auxnum_labels = len(auxlabel_list) + 1  # label 0 corresponds to padding, label in label_list starts from 1
+
+start_label_id = processor.get_start_label_id()
+stop_label_id = processor.get_stop_label_id()
+
+# ''' initialization of our conversion matrix, in our implementation, it is a 7*12 matrix initialized as follows:
+trans_matrix = np.zeros((auxnum_labels, num_labels), dtype=float)
+trans_matrix[0, 0] = 1  # pad to pad
+trans_matrix[1, 1] = 1  # O to O
+trans_matrix[2, 2] = 0.25  # B to B-MISC
+trans_matrix[2, 4] = 0.25  # B to B-PER
+trans_matrix[2, 6] = 0.25  # B to B-ORG
+trans_matrix[2, 8] = 0.25  # B to B-LOC
+trans_matrix[3, 3] = 0.25  # I to I-MISC
+trans_matrix[3, 5] = 0.25  # I to I-PER
+trans_matrix[3, 7] = 0.25  # I to I-ORG
+trans_matrix[3, 9] = 0.25  # I to I-LOC
+trans_matrix[4, 10] = 1  # X to X
+trans_matrix[5, 11] = 1  # [CLS] to [CLS]
+trans_matrix[6, 12] = 1  # [SEP] to [SEP]
+'''
+trans_matrix = np.zeros((num_labels, auxnum_labels), dtype=float)
+trans_matrix[0,0]=1 # pad to pad
+trans_matrix[1,1]=1
+trans_matrix[2,2]=1
+trans_matrix[4,2]=1
+trans_matrix[6,2]=1
+trans_matrix[8,2]=1
+trans_matrix[3,3]=1
+trans_matrix[5,3]=1
+trans_matrix[7,3]=1
+trans_matrix[9,3]=1
+trans_matrix[10,4]=1
+trans_matrix[11,5]=1
+trans_matrix[12,6]=1
+'''
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+tokenizer = AutoTokenizer.from_pretrained(args.bert_model, do_lower_case=args.do_lower_case)
+
+
+
+net = getattr(resnet, 'resnet152')()
+net.load_state_dict(torch.load(os.path.join(args.resnet_root, 'resnet152.pth')))
+encoder = myResnet(net, args.fine_tune_cnn, device)
+
+
+output_model_file = os.path.join(args.output_dir, WEIGHTS_NAME)
+# output_config_file = os.path.join(args.output_dir, CONFIG_NAME)
+output_encoder_file = os.path.join(args.output_dir, "pytorch_encoder.bin")
+
+temp = args.temp
+temp_lamb = args.temp_lamb
+lamb = args.lamb
+negative_rate = args.negative_rate
+# # loadmodel
+#     model = UMT.from_pretrained(args.bert_model,
+#                                 cache_dir=args.cache_dir, layer_num1=args.layer_num1,
+#                                 layer_num2=args.layer_num2,
+#                                 layer_num3=args.layer_num3,
+#                                 num_labels_=num_labels, auxnum_labels=auxnum_labels)
+#     model.load_state_dict(torch.load(output_model_file,map_location=torch.device('cpu')))
+#     model.to(device)
+#     encoder_state_dict = torch.load(output_encoder_file,map_location=torch.device('cpu'))
+#     encoder.load_state_dict(encoder_state_dict)
+#     encoder.to(device)
+#     print(model)
+
+def load_model(output_model_file, output_encoder_file,encoder,num_labels,auxnum_labels):
+    model = UMT.from_pretrained(args.bert_model,
+                                cache_dir=args.cache_dir, layer_num1=args.layer_num1,
+                                layer_num2=args.layer_num2,
+                                layer_num3=args.layer_num3,
+                                num_labels_=num_labels, auxnum_labels=auxnum_labels)
+    model.load_state_dict(torch.load(output_model_file, map_location=torch.device('cpu')))
+    model.to(device)
+    encoder_state_dict = torch.load(output_encoder_file, map_location=torch.device('cpu'))
+    encoder.load_state_dict(encoder_state_dict)
+    encoder.to(device)
+    return model, encoder
+
+model_umt,encoder_umt=load_model(output_model_file, output_encoder_file,encoder,num_labels,auxnum_labels)
+#
+#     # sentence = 'Thương biết_mấy những Thuận, những Liên, những Luận, Xuân, Nghĩa mỗi người một hoàn_cảnh nhưng đều rất giống nhau: rất ham học, rất cố_gắng để đạt mức hiểu biết cao nhất.'
+#     # image_path = '/kaggle/working/data/014715.jpg'
+#     # # crop_size = 224'
+path_image='E:\demo_datn\pythonProject1\Model\MultimodelNER\VLSP2021\Image'
+trans_matrix = np.zeros((auxnum_labels,num_labels), dtype=float)
+trans_matrix[0,0]=1 # pad to pad
+trans_matrix[1,1]=1 # O to O
+trans_matrix[2,2]=0.25 # B to B-MISC
+trans_matrix[2,4]=0.25 # B to B-PER
+trans_matrix[2,6]=0.25 # B to B-ORG
+trans_matrix[2,8]=0.25 # B to B-LOC
+trans_matrix[3,3]=0.25 # I to I-MISC
+trans_matrix[3,5]=0.25 # I to I-PER
+trans_matrix[3,7]=0.25 # I to I-ORG
+trans_matrix[3,9]=0.25 # I to I-LOC
+trans_matrix[4,10]=1   # X to X
+trans_matrix[5,11]=1   # [CLS] to [CLS]
+trans_matrix[6,12]=1   # [SE
+def predict(model_umt, encoder_umt, eval_examples, tokenizer, device,path_image,trans_matrix):
+
+    features = convert_mm_examples_to_features_predict(eval_examples, 256, tokenizer, 224,path_image)
+
+    input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
+    input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long)
+    added_input_mask = torch.tensor([f.added_input_mask for f in features], dtype=torch.long)
+    segment_ids = torch.tensor([f.segment_ids for f in features], dtype=torch.long)
+    img_feats = torch.stack([f.img_feat for f in features])
+    print(img_feats)
+    eval_data = TensorDataset(input_ids, input_mask, added_input_mask, segment_ids, img_feats)
+    eval_sampler = SequentialSampler(eval_data)
+    eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=16)
+
+    model_umt.eval()
+    encoder_umt.eval()
+
+    y_pred = []
+    label_map = {i: label for i, label in enumerate(label_list, 1)}
+    label_map[0] = "<pad>"
+
+    for input_ids, input_mask, added_input_mask, segment_ids, img_feats in tqdm(eval_dataloader, desc="Evaluating"):
+        input_ids = input_ids.to(device)
+        input_mask = input_mask.to(device)
+        added_input_mask = added_input_mask.to(device)
+        segment_ids = segment_ids.to(device)
+        img_feats = img_feats.to(device)
+
+        with torch.no_grad():
+            imgs_f, img_mean, img_att = encoder_umt(img_feats)
+            predicted_label_seq_ids = model_umt(input_ids, segment_ids, input_mask, added_input_mask, img_att,
+                                                trans_matrix)
+
+        logits = predicted_label_seq_ids
+        input_mask = input_mask.to('cpu').numpy()
+
+        for i, mask in enumerate(input_mask):
+            temp_1 = []
+            for j, m in enumerate(mask):
+                if j == 0:
+                    continue
+                if m:
+                    if label_map[logits[i][j]] not in ["<pad>", "<s>", "</s>", "X"]:
+                        temp_1.append(label_map[logits[i][j]])
+                else:
+                    break
+            y_pred.append(temp_1)
+
+    a = eval_examples[0].text_a.split(" ")
+
+    return y_pred, a
+
+# eval_examples = get_test_examples_predict('Model/MultimodelNER/VLSP2021/Filetxt/')
+# y_pred, a = predict(model_umt, encoder_umt, eval_examples, tokenizer, device,path_image,trans_matrix)
+# print(y_pred)
+# print(a)
+# formatted_output = format_predictions(a, y_pred[0])
+#
+# final= process_predictions(formatted_output)
+# final2= combine_entities(final)
+# print(final2)
+# final3= remove_B_prefix(final2)
+# final4=combine_i_tags(final3)
+# print(final3)
+