Spaces:
Running
on
Zero
Running
on
Zero
| import torch | |
| from torch.nn import functional as F | |
| from funasr_detach.models.encoder.abs_encoder import AbsEncoder | |
| from typing import Tuple, Optional | |
| from funasr_detach.models.pooling.statistic_pooling import ( | |
| statistic_pooling, | |
| windowed_statistic_pooling, | |
| ) | |
| from collections import OrderedDict | |
| import logging | |
| import numpy as np | |
| class BasicLayer(torch.nn.Module): | |
| def __init__( | |
| self, in_filters: int, filters: int, stride: int, bn_momentum: float = 0.5 | |
| ): | |
| super().__init__() | |
| self.stride = stride | |
| self.in_filters = in_filters | |
| self.filters = filters | |
| self.bn1 = torch.nn.BatchNorm2d( | |
| in_filters, eps=1e-3, momentum=bn_momentum, affine=True | |
| ) | |
| self.relu1 = torch.nn.ReLU() | |
| self.conv1 = torch.nn.Conv2d(in_filters, filters, 3, stride, bias=False) | |
| self.bn2 = torch.nn.BatchNorm2d( | |
| filters, eps=1e-3, momentum=bn_momentum, affine=True | |
| ) | |
| self.relu2 = torch.nn.ReLU() | |
| self.conv2 = torch.nn.Conv2d(filters, filters, 3, 1, bias=False) | |
| if in_filters != filters or stride > 1: | |
| self.conv_sc = torch.nn.Conv2d(in_filters, filters, 1, stride, bias=False) | |
| self.bn_sc = torch.nn.BatchNorm2d( | |
| filters, eps=1e-3, momentum=bn_momentum, affine=True | |
| ) | |
| def proper_padding(self, x, stride): | |
| # align padding mode to tf.layers.conv2d with padding_mod="same" | |
| if stride == 1: | |
| return F.pad(x, (1, 1, 1, 1), "constant", 0) | |
| elif stride == 2: | |
| h, w = x.size(2), x.size(3) | |
| # (left, right, top, bottom) | |
| return F.pad(x, (w % 2, 1, h % 2, 1), "constant", 0) | |
| def forward(self, xs_pad, ilens): | |
| identity = xs_pad | |
| if self.in_filters != self.filters or self.stride > 1: | |
| identity = self.conv_sc(identity) | |
| identity = self.bn_sc(identity) | |
| xs_pad = self.relu1(self.bn1(xs_pad)) | |
| xs_pad = self.proper_padding(xs_pad, self.stride) | |
| xs_pad = self.conv1(xs_pad) | |
| xs_pad = self.relu2(self.bn2(xs_pad)) | |
| xs_pad = self.proper_padding(xs_pad, 1) | |
| xs_pad = self.conv2(xs_pad) | |
| if self.stride == 2: | |
| ilens = (ilens + 1) // self.stride | |
| return xs_pad + identity, ilens | |
| class BasicBlock(torch.nn.Module): | |
| def __init__(self, in_filters, filters, num_layer, stride, bn_momentum=0.5): | |
| super().__init__() | |
| self.num_layer = num_layer | |
| for i in range(num_layer): | |
| layer = BasicLayer( | |
| in_filters if i == 0 else filters, | |
| filters, | |
| stride if i == 0 else 1, | |
| bn_momentum, | |
| ) | |
| self.add_module("layer_{}".format(i), layer) | |
| def forward(self, xs_pad, ilens): | |
| for i in range(self.num_layer): | |
| xs_pad, ilens = self._modules["layer_{}".format(i)](xs_pad, ilens) | |
| return xs_pad, ilens | |
| class ResNet34(AbsEncoder): | |
| def __init__( | |
| self, | |
| input_size, | |
| use_head_conv=True, | |
| batchnorm_momentum=0.5, | |
| use_head_maxpool=False, | |
| num_nodes_pooling_layer=256, | |
| layers_in_block=(3, 4, 6, 3), | |
| filters_in_block=(32, 64, 128, 256), | |
| ): | |
| super(ResNet34, self).__init__() | |
| self.use_head_conv = use_head_conv | |
| self.use_head_maxpool = use_head_maxpool | |
| self.num_nodes_pooling_layer = num_nodes_pooling_layer | |
| self.layers_in_block = layers_in_block | |
| self.filters_in_block = filters_in_block | |
| self.input_size = input_size | |
| pre_filters = filters_in_block[0] | |
| if use_head_conv: | |
| self.pre_conv = torch.nn.Conv2d( | |
| 1, pre_filters, 3, 1, 1, bias=False, padding_mode="zeros" | |
| ) | |
| self.pre_conv_bn = torch.nn.BatchNorm2d( | |
| pre_filters, eps=1e-3, momentum=batchnorm_momentum | |
| ) | |
| if use_head_maxpool: | |
| self.head_maxpool = torch.nn.MaxPool2d(3, 1, padding=1) | |
| for i in range(len(layers_in_block)): | |
| if i == 0: | |
| in_filters = pre_filters if self.use_head_conv else 1 | |
| else: | |
| in_filters = filters_in_block[i - 1] | |
| block = BasicBlock( | |
| in_filters, | |
| filters=filters_in_block[i], | |
| num_layer=layers_in_block[i], | |
| stride=1 if i == 0 else 2, | |
| bn_momentum=batchnorm_momentum, | |
| ) | |
| self.add_module("block_{}".format(i), block) | |
| self.resnet0_dense = torch.nn.Conv2d( | |
| filters_in_block[-1], num_nodes_pooling_layer, 1 | |
| ) | |
| self.resnet0_bn = torch.nn.BatchNorm2d( | |
| num_nodes_pooling_layer, eps=1e-3, momentum=batchnorm_momentum | |
| ) | |
| self.time_ds_ratio = 8 | |
| def output_size(self) -> int: | |
| return self.num_nodes_pooling_layer | |
| def forward( | |
| self, | |
| xs_pad: torch.Tensor, | |
| ilens: torch.Tensor, | |
| prev_states: torch.Tensor = None, | |
| ) -> Tuple[torch.Tensor, torch.Tensor]: | |
| features = xs_pad | |
| assert ( | |
| features.size(-1) == self.input_size | |
| ), "Dimension of features {} doesn't match the input_size {}.".format( | |
| features.size(-1), self.input_size | |
| ) | |
| features = torch.unsqueeze(features, dim=1) | |
| if self.use_head_conv: | |
| features = self.pre_conv(features) | |
| features = self.pre_conv_bn(features) | |
| features = F.relu(features) | |
| if self.use_head_maxpool: | |
| features = self.head_maxpool(features) | |
| resnet_outs, resnet_out_lens = features, ilens | |
| for i in range(len(self.layers_in_block)): | |
| block = self._modules["block_{}".format(i)] | |
| resnet_outs, resnet_out_lens = block(resnet_outs, resnet_out_lens) | |
| features = self.resnet0_dense(resnet_outs) | |
| features = F.relu(features) | |
| features = self.resnet0_bn(features) | |
| return features, resnet_out_lens | |
| # Note: For training, this implement is not equivalent to tf because of the kernel_regularizer in tf.layers. | |
| # TODO: implement kernel_regularizer in torch with munal loss addition or weigth_decay in the optimizer | |
| class ResNet34_SP_L2Reg(AbsEncoder): | |
| def __init__( | |
| self, | |
| input_size, | |
| use_head_conv=True, | |
| batchnorm_momentum=0.5, | |
| use_head_maxpool=False, | |
| num_nodes_pooling_layer=256, | |
| layers_in_block=(3, 4, 6, 3), | |
| filters_in_block=(32, 64, 128, 256), | |
| tf2torch_tensor_name_prefix_torch="encoder", | |
| tf2torch_tensor_name_prefix_tf="EAND/speech_encoder", | |
| tf_train_steps=720000, | |
| ): | |
| super(ResNet34_SP_L2Reg, self).__init__() | |
| self.use_head_conv = use_head_conv | |
| self.use_head_maxpool = use_head_maxpool | |
| self.num_nodes_pooling_layer = num_nodes_pooling_layer | |
| self.layers_in_block = layers_in_block | |
| self.filters_in_block = filters_in_block | |
| self.input_size = input_size | |
| self.tf2torch_tensor_name_prefix_torch = tf2torch_tensor_name_prefix_torch | |
| self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tf | |
| self.tf_train_steps = tf_train_steps | |
| pre_filters = filters_in_block[0] | |
| if use_head_conv: | |
| self.pre_conv = torch.nn.Conv2d( | |
| 1, pre_filters, 3, 1, 1, bias=False, padding_mode="zeros" | |
| ) | |
| self.pre_conv_bn = torch.nn.BatchNorm2d( | |
| pre_filters, eps=1e-3, momentum=batchnorm_momentum | |
| ) | |
| if use_head_maxpool: | |
| self.head_maxpool = torch.nn.MaxPool2d(3, 1, padding=1) | |
| for i in range(len(layers_in_block)): | |
| if i == 0: | |
| in_filters = pre_filters if self.use_head_conv else 1 | |
| else: | |
| in_filters = filters_in_block[i - 1] | |
| block = BasicBlock( | |
| in_filters, | |
| filters=filters_in_block[i], | |
| num_layer=layers_in_block[i], | |
| stride=1 if i == 0 else 2, | |
| bn_momentum=batchnorm_momentum, | |
| ) | |
| self.add_module("block_{}".format(i), block) | |
| self.resnet0_dense = torch.nn.Conv1d( | |
| filters_in_block[-1] * input_size // 8, num_nodes_pooling_layer, 1 | |
| ) | |
| self.resnet0_bn = torch.nn.BatchNorm1d( | |
| num_nodes_pooling_layer, eps=1e-3, momentum=batchnorm_momentum | |
| ) | |
| self.time_ds_ratio = 8 | |
| def output_size(self) -> int: | |
| return self.num_nodes_pooling_layer | |
| def forward( | |
| self, | |
| xs_pad: torch.Tensor, | |
| ilens: torch.Tensor, | |
| prev_states: torch.Tensor = None, | |
| ) -> Tuple[torch.Tensor, torch.Tensor]: | |
| features = xs_pad | |
| assert ( | |
| features.size(-1) == self.input_size | |
| ), "Dimension of features {} doesn't match the input_size {}.".format( | |
| features.size(-1), self.input_size | |
| ) | |
| features = torch.unsqueeze(features, dim=1) | |
| if self.use_head_conv: | |
| features = self.pre_conv(features) | |
| features = self.pre_conv_bn(features) | |
| features = F.relu(features) | |
| if self.use_head_maxpool: | |
| features = self.head_maxpool(features) | |
| resnet_outs, resnet_out_lens = features, ilens | |
| for i in range(len(self.layers_in_block)): | |
| block = self._modules["block_{}".format(i)] | |
| resnet_outs, resnet_out_lens = block(resnet_outs, resnet_out_lens) | |
| # B, C, T, F | |
| bb, cc, tt, ff = resnet_outs.shape | |
| resnet_outs = torch.reshape(resnet_outs.permute(0, 3, 1, 2), [bb, ff * cc, tt]) | |
| features = self.resnet0_dense(resnet_outs) | |
| features = F.relu(features) | |
| features = self.resnet0_bn(features) | |
| return features, resnet_out_lens | |
| def gen_tf2torch_map_dict(self): | |
| tensor_name_prefix_torch = self.tf2torch_tensor_name_prefix_torch | |
| tensor_name_prefix_tf = self.tf2torch_tensor_name_prefix_tf | |
| train_steps = self.tf_train_steps | |
| map_dict_local = { | |
| # torch: conv1d.weight in "out_channel in_channel kernel_size" | |
| # tf : conv1d.weight in "kernel_size in_channel out_channel" | |
| # torch: linear.weight in "out_channel in_channel" | |
| # tf : dense.weight in "in_channel out_channel" | |
| "{}.pre_conv.weight".format(tensor_name_prefix_torch): { | |
| "name": "{}/pre_conv/kernel".format(tensor_name_prefix_tf), | |
| "squeeze": None, | |
| "transpose": (3, 2, 0, 1), | |
| }, | |
| "{}.pre_conv_bn.bias".format(tensor_name_prefix_torch): { | |
| "name": "{}/pre_conv_bn/beta".format(tensor_name_prefix_tf), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.pre_conv_bn.weight".format(tensor_name_prefix_torch): { | |
| "name": "{}/pre_conv_bn/gamma".format(tensor_name_prefix_tf), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.pre_conv_bn.running_mean".format(tensor_name_prefix_torch): { | |
| "name": "{}/pre_conv_bn/moving_mean".format(tensor_name_prefix_tf), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.pre_conv_bn.running_var".format(tensor_name_prefix_torch): { | |
| "name": "{}/pre_conv_bn/moving_variance".format(tensor_name_prefix_tf), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.pre_conv_bn.num_batches_tracked".format( | |
| tensor_name_prefix_torch | |
| ): train_steps, | |
| } | |
| for layer_idx in range(3): | |
| map_dict_local.update( | |
| { | |
| "{}.resnet{}_dense.weight".format( | |
| tensor_name_prefix_torch, layer_idx | |
| ): { | |
| "name": "{}/resnet{}_dense/kernel".format( | |
| tensor_name_prefix_tf, layer_idx | |
| ), | |
| "squeeze": None, | |
| "transpose": (2, 1, 0) if layer_idx == 0 else (1, 0), | |
| }, | |
| "{}.resnet{}_dense.bias".format( | |
| tensor_name_prefix_torch, layer_idx | |
| ): { | |
| "name": "{}/resnet{}_dense/bias".format( | |
| tensor_name_prefix_tf, layer_idx | |
| ), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.resnet{}_bn.weight".format( | |
| tensor_name_prefix_torch, layer_idx | |
| ): { | |
| "name": "{}/resnet{}_bn/gamma".format( | |
| tensor_name_prefix_tf, layer_idx | |
| ), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.resnet{}_bn.bias".format(tensor_name_prefix_torch, layer_idx): { | |
| "name": "{}/resnet{}_bn/beta".format( | |
| tensor_name_prefix_tf, layer_idx | |
| ), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.resnet{}_bn.running_mean".format( | |
| tensor_name_prefix_torch, layer_idx | |
| ): { | |
| "name": "{}/resnet{}_bn/moving_mean".format( | |
| tensor_name_prefix_tf, layer_idx | |
| ), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.resnet{}_bn.running_var".format( | |
| tensor_name_prefix_torch, layer_idx | |
| ): { | |
| "name": "{}/resnet{}_bn/moving_variance".format( | |
| tensor_name_prefix_tf, layer_idx | |
| ), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.resnet{}_bn.num_batches_tracked".format( | |
| tensor_name_prefix_torch, layer_idx | |
| ): train_steps, | |
| } | |
| ) | |
| for block_idx in range(len(self.layers_in_block)): | |
| for layer_idx in range(self.layers_in_block[block_idx]): | |
| for i in ["1", "2", "_sc"]: | |
| map_dict_local.update( | |
| { | |
| "{}.block_{}.layer_{}.conv{}.weight".format( | |
| tensor_name_prefix_torch, block_idx, layer_idx, i | |
| ): { | |
| "name": "{}/block_{}/layer_{}/conv{}/kernel".format( | |
| tensor_name_prefix_tf, block_idx, layer_idx, i | |
| ), | |
| "squeeze": None, | |
| "transpose": (3, 2, 0, 1), | |
| }, | |
| "{}.block_{}.layer_{}.bn{}.weight".format( | |
| tensor_name_prefix_torch, block_idx, layer_idx, i | |
| ): { | |
| "name": "{}/block_{}/layer_{}/bn{}/gamma".format( | |
| tensor_name_prefix_tf, block_idx, layer_idx, i | |
| ), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.block_{}.layer_{}.bn{}.bias".format( | |
| tensor_name_prefix_torch, block_idx, layer_idx, i | |
| ): { | |
| "name": "{}/block_{}/layer_{}/bn{}/beta".format( | |
| tensor_name_prefix_tf, block_idx, layer_idx, i | |
| ), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.block_{}.layer_{}.bn{}.running_mean".format( | |
| tensor_name_prefix_torch, block_idx, layer_idx, i | |
| ): { | |
| "name": "{}/block_{}/layer_{}/bn{}/moving_mean".format( | |
| tensor_name_prefix_tf, block_idx, layer_idx, i | |
| ), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.block_{}.layer_{}.bn{}.running_var".format( | |
| tensor_name_prefix_torch, block_idx, layer_idx, i | |
| ): { | |
| "name": "{}/block_{}/layer_{}/bn{}/moving_variance".format( | |
| tensor_name_prefix_tf, block_idx, layer_idx, i | |
| ), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.block_{}.layer_{}.bn{}.num_batches_tracked".format( | |
| tensor_name_prefix_torch, block_idx, layer_idx, i | |
| ): train_steps, | |
| } | |
| ) | |
| return map_dict_local | |
| def convert_tf2torch( | |
| self, | |
| var_dict_tf, | |
| var_dict_torch, | |
| ): | |
| map_dict = self.gen_tf2torch_map_dict() | |
| var_dict_torch_update = dict() | |
| for name in sorted(var_dict_torch.keys(), reverse=False): | |
| if name.startswith(self.tf2torch_tensor_name_prefix_torch): | |
| if name in map_dict: | |
| if "num_batches_tracked" not in name: | |
| name_tf = map_dict[name]["name"] | |
| data_tf = var_dict_tf[name_tf] | |
| if map_dict[name]["squeeze"] is not None: | |
| data_tf = np.squeeze( | |
| data_tf, axis=map_dict[name]["squeeze"] | |
| ) | |
| if map_dict[name]["transpose"] is not None: | |
| data_tf = np.transpose(data_tf, map_dict[name]["transpose"]) | |
| data_tf = ( | |
| torch.from_numpy(data_tf).type(torch.float32).to("cpu") | |
| ) | |
| assert ( | |
| var_dict_torch[name].size() == data_tf.size() | |
| ), "{}, {}, {} != {}".format( | |
| name, name_tf, var_dict_torch[name].size(), data_tf.size() | |
| ) | |
| var_dict_torch_update[name] = data_tf | |
| logging.info( | |
| "torch tensor: {}, {}, loading from tf tensor: {}, {}".format( | |
| name, | |
| data_tf.size(), | |
| name_tf, | |
| var_dict_tf[name_tf].shape, | |
| ) | |
| ) | |
| else: | |
| var_dict_torch_update[name] = ( | |
| torch.Tensor(map_dict[name]).type(torch.int64).to("cpu") | |
| ) | |
| logging.info( | |
| "torch tensor: {}, manually assigning to: {}".format( | |
| name, map_dict[name] | |
| ) | |
| ) | |
| else: | |
| logging.warning("{} is missed from tf checkpoint".format(name)) | |
| return var_dict_torch_update | |
| class ResNet34Diar(ResNet34): | |
| def __init__( | |
| self, | |
| input_size, | |
| embedding_node="resnet1_dense", | |
| use_head_conv=True, | |
| batchnorm_momentum=0.5, | |
| use_head_maxpool=False, | |
| num_nodes_pooling_layer=256, | |
| layers_in_block=(3, 4, 6, 3), | |
| filters_in_block=(32, 64, 128, 256), | |
| num_nodes_resnet1=256, | |
| num_nodes_last_layer=256, | |
| pooling_type="window_shift", | |
| pool_size=20, | |
| stride=1, | |
| tf2torch_tensor_name_prefix_torch="encoder", | |
| tf2torch_tensor_name_prefix_tf="seq2seq/speech_encoder", | |
| ): | |
| """ | |
| Author: Speech Lab, Alibaba Group, China | |
| SOND: Speaker Overlap-aware Neural Diarization for Multi-party Meeting Analysis | |
| https://arxiv.org/abs/2211.10243 | |
| """ | |
| super(ResNet34Diar, self).__init__( | |
| input_size, | |
| use_head_conv=use_head_conv, | |
| batchnorm_momentum=batchnorm_momentum, | |
| use_head_maxpool=use_head_maxpool, | |
| num_nodes_pooling_layer=num_nodes_pooling_layer, | |
| layers_in_block=layers_in_block, | |
| filters_in_block=filters_in_block, | |
| ) | |
| self.embedding_node = embedding_node | |
| self.num_nodes_resnet1 = num_nodes_resnet1 | |
| self.num_nodes_last_layer = num_nodes_last_layer | |
| self.pooling_type = pooling_type | |
| self.pool_size = pool_size | |
| self.stride = stride | |
| self.tf2torch_tensor_name_prefix_torch = tf2torch_tensor_name_prefix_torch | |
| self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tf | |
| self.resnet1_dense = torch.nn.Linear( | |
| num_nodes_pooling_layer * 2, num_nodes_resnet1 | |
| ) | |
| self.resnet1_bn = torch.nn.BatchNorm1d( | |
| num_nodes_resnet1, eps=1e-3, momentum=batchnorm_momentum | |
| ) | |
| self.resnet2_dense = torch.nn.Linear(num_nodes_resnet1, num_nodes_last_layer) | |
| self.resnet2_bn = torch.nn.BatchNorm1d( | |
| num_nodes_last_layer, eps=1e-3, momentum=batchnorm_momentum | |
| ) | |
| def output_size(self) -> int: | |
| if self.embedding_node.startswith("resnet1"): | |
| return self.num_nodes_resnet1 | |
| elif self.embedding_node.startswith("resnet2"): | |
| return self.num_nodes_last_layer | |
| return self.num_nodes_pooling_layer | |
| def forward( | |
| self, | |
| xs_pad: torch.Tensor, | |
| ilens: torch.Tensor, | |
| prev_states: torch.Tensor = None, | |
| ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: | |
| endpoints = OrderedDict() | |
| res_out, ilens = super().forward(xs_pad, ilens) | |
| endpoints["resnet0_bn"] = res_out | |
| if self.pooling_type == "frame_gsp": | |
| features = statistic_pooling(res_out, ilens, (3,)) | |
| else: | |
| features, ilens = windowed_statistic_pooling( | |
| res_out, ilens, (2, 3), self.pool_size, self.stride | |
| ) | |
| features = features.transpose(1, 2) | |
| endpoints["pooling"] = features | |
| features = self.resnet1_dense(features) | |
| endpoints["resnet1_dense"] = features | |
| features = F.relu(features) | |
| endpoints["resnet1_relu"] = features | |
| features = self.resnet1_bn(features.transpose(1, 2)).transpose(1, 2) | |
| endpoints["resnet1_bn"] = features | |
| features = self.resnet2_dense(features) | |
| endpoints["resnet2_dense"] = features | |
| features = F.relu(features) | |
| endpoints["resnet2_relu"] = features | |
| features = self.resnet2_bn(features.transpose(1, 2)).transpose(1, 2) | |
| endpoints["resnet2_bn"] = features | |
| return endpoints[self.embedding_node], ilens, None | |
| def gen_tf2torch_map_dict(self): | |
| tensor_name_prefix_torch = self.tf2torch_tensor_name_prefix_torch | |
| tensor_name_prefix_tf = self.tf2torch_tensor_name_prefix_tf | |
| train_steps = 300000 | |
| map_dict_local = { | |
| # torch: conv1d.weight in "out_channel in_channel kernel_size" | |
| # tf : conv1d.weight in "kernel_size in_channel out_channel" | |
| # torch: linear.weight in "out_channel in_channel" | |
| # tf : dense.weight in "in_channel out_channel" | |
| "{}.pre_conv.weight".format(tensor_name_prefix_torch): { | |
| "name": "{}/pre_conv/kernel".format(tensor_name_prefix_tf), | |
| "squeeze": None, | |
| "transpose": (3, 2, 0, 1), | |
| }, | |
| "{}.pre_conv_bn.bias".format(tensor_name_prefix_torch): { | |
| "name": "{}/pre_conv_bn/beta".format(tensor_name_prefix_tf), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.pre_conv_bn.weight".format(tensor_name_prefix_torch): { | |
| "name": "{}/pre_conv_bn/gamma".format(tensor_name_prefix_tf), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.pre_conv_bn.running_mean".format(tensor_name_prefix_torch): { | |
| "name": "{}/pre_conv_bn/moving_mean".format(tensor_name_prefix_tf), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.pre_conv_bn.running_var".format(tensor_name_prefix_torch): { | |
| "name": "{}/pre_conv_bn/moving_variance".format(tensor_name_prefix_tf), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.pre_conv_bn.num_batches_tracked".format( | |
| tensor_name_prefix_torch | |
| ): train_steps, | |
| } | |
| for layer_idx in range(3): | |
| map_dict_local.update( | |
| { | |
| "{}.resnet{}_dense.weight".format( | |
| tensor_name_prefix_torch, layer_idx | |
| ): { | |
| "name": "{}/resnet{}_dense/kernel".format( | |
| tensor_name_prefix_tf, layer_idx | |
| ), | |
| "squeeze": None, | |
| "transpose": (3, 2, 0, 1) if layer_idx == 0 else (1, 0), | |
| }, | |
| "{}.resnet{}_dense.bias".format( | |
| tensor_name_prefix_torch, layer_idx | |
| ): { | |
| "name": "{}/resnet{}_dense/bias".format( | |
| tensor_name_prefix_tf, layer_idx | |
| ), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.resnet{}_bn.weight".format( | |
| tensor_name_prefix_torch, layer_idx | |
| ): { | |
| "name": "{}/resnet{}_bn/gamma".format( | |
| tensor_name_prefix_tf, layer_idx | |
| ), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.resnet{}_bn.bias".format(tensor_name_prefix_torch, layer_idx): { | |
| "name": "{}/resnet{}_bn/beta".format( | |
| tensor_name_prefix_tf, layer_idx | |
| ), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.resnet{}_bn.running_mean".format( | |
| tensor_name_prefix_torch, layer_idx | |
| ): { | |
| "name": "{}/resnet{}_bn/moving_mean".format( | |
| tensor_name_prefix_tf, layer_idx | |
| ), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.resnet{}_bn.running_var".format( | |
| tensor_name_prefix_torch, layer_idx | |
| ): { | |
| "name": "{}/resnet{}_bn/moving_variance".format( | |
| tensor_name_prefix_tf, layer_idx | |
| ), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.resnet{}_bn.num_batches_tracked".format( | |
| tensor_name_prefix_torch, layer_idx | |
| ): train_steps, | |
| } | |
| ) | |
| for block_idx in range(len(self.layers_in_block)): | |
| for layer_idx in range(self.layers_in_block[block_idx]): | |
| for i in ["1", "2", "_sc"]: | |
| map_dict_local.update( | |
| { | |
| "{}.block_{}.layer_{}.conv{}.weight".format( | |
| tensor_name_prefix_torch, block_idx, layer_idx, i | |
| ): { | |
| "name": "{}/block_{}/layer_{}/conv{}/kernel".format( | |
| tensor_name_prefix_tf, block_idx, layer_idx, i | |
| ), | |
| "squeeze": None, | |
| "transpose": (3, 2, 0, 1), | |
| }, | |
| "{}.block_{}.layer_{}.bn{}.weight".format( | |
| tensor_name_prefix_torch, block_idx, layer_idx, i | |
| ): { | |
| "name": "{}/block_{}/layer_{}/bn{}/gamma".format( | |
| tensor_name_prefix_tf, block_idx, layer_idx, i | |
| ), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.block_{}.layer_{}.bn{}.bias".format( | |
| tensor_name_prefix_torch, block_idx, layer_idx, i | |
| ): { | |
| "name": "{}/block_{}/layer_{}/bn{}/beta".format( | |
| tensor_name_prefix_tf, block_idx, layer_idx, i | |
| ), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.block_{}.layer_{}.bn{}.running_mean".format( | |
| tensor_name_prefix_torch, block_idx, layer_idx, i | |
| ): { | |
| "name": "{}/block_{}/layer_{}/bn{}/moving_mean".format( | |
| tensor_name_prefix_tf, block_idx, layer_idx, i | |
| ), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.block_{}.layer_{}.bn{}.running_var".format( | |
| tensor_name_prefix_torch, block_idx, layer_idx, i | |
| ): { | |
| "name": "{}/block_{}/layer_{}/bn{}/moving_variance".format( | |
| tensor_name_prefix_tf, block_idx, layer_idx, i | |
| ), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.block_{}.layer_{}.bn{}.num_batches_tracked".format( | |
| tensor_name_prefix_torch, block_idx, layer_idx, i | |
| ): train_steps, | |
| } | |
| ) | |
| return map_dict_local | |
| def convert_tf2torch( | |
| self, | |
| var_dict_tf, | |
| var_dict_torch, | |
| ): | |
| map_dict = self.gen_tf2torch_map_dict() | |
| var_dict_torch_update = dict() | |
| for name in sorted(var_dict_torch.keys(), reverse=False): | |
| if name.startswith(self.tf2torch_tensor_name_prefix_torch): | |
| if name in map_dict: | |
| if "num_batches_tracked" not in name: | |
| name_tf = map_dict[name]["name"] | |
| data_tf = var_dict_tf[name_tf] | |
| if map_dict[name]["squeeze"] is not None: | |
| data_tf = np.squeeze( | |
| data_tf, axis=map_dict[name]["squeeze"] | |
| ) | |
| if map_dict[name]["transpose"] is not None: | |
| data_tf = np.transpose(data_tf, map_dict[name]["transpose"]) | |
| data_tf = ( | |
| torch.from_numpy(data_tf).type(torch.float32).to("cpu") | |
| ) | |
| assert ( | |
| var_dict_torch[name].size() == data_tf.size() | |
| ), "{}, {}, {} != {}".format( | |
| name, name_tf, var_dict_torch[name].size(), data_tf.size() | |
| ) | |
| var_dict_torch_update[name] = data_tf | |
| logging.info( | |
| "torch tensor: {}, {}, loading from tf tensor: {}, {}".format( | |
| name, | |
| data_tf.size(), | |
| name_tf, | |
| var_dict_tf[name_tf].shape, | |
| ) | |
| ) | |
| else: | |
| var_dict_torch_update[name] = ( | |
| torch.Tensor(map_dict[name]).type(torch.int64).to("cpu") | |
| ) | |
| logging.info( | |
| "torch tensor: {}, manually assigning to: {}".format( | |
| name, map_dict[name] | |
| ) | |
| ) | |
| else: | |
| logging.warning("{} is missed from tf checkpoint".format(name)) | |
| return var_dict_torch_update | |
| class ResNet34SpL2RegDiar(ResNet34_SP_L2Reg): | |
| def __init__( | |
| self, | |
| input_size, | |
| embedding_node="resnet1_dense", | |
| use_head_conv=True, | |
| batchnorm_momentum=0.5, | |
| use_head_maxpool=False, | |
| num_nodes_pooling_layer=256, | |
| layers_in_block=(3, 4, 6, 3), | |
| filters_in_block=(32, 64, 128, 256), | |
| num_nodes_resnet1=256, | |
| num_nodes_last_layer=256, | |
| pooling_type="window_shift", | |
| pool_size=20, | |
| stride=1, | |
| tf2torch_tensor_name_prefix_torch="encoder", | |
| tf2torch_tensor_name_prefix_tf="seq2seq/speech_encoder", | |
| ): | |
| """ | |
| Author: Speech Lab, Alibaba Group, China | |
| TOLD: A Novel Two-Stage Overlap-Aware Framework for Speaker Diarization | |
| https://arxiv.org/abs/2303.05397 | |
| """ | |
| super(ResNet34SpL2RegDiar, self).__init__( | |
| input_size, | |
| use_head_conv=use_head_conv, | |
| batchnorm_momentum=batchnorm_momentum, | |
| use_head_maxpool=use_head_maxpool, | |
| num_nodes_pooling_layer=num_nodes_pooling_layer, | |
| layers_in_block=layers_in_block, | |
| filters_in_block=filters_in_block, | |
| ) | |
| self.embedding_node = embedding_node | |
| self.num_nodes_resnet1 = num_nodes_resnet1 | |
| self.num_nodes_last_layer = num_nodes_last_layer | |
| self.pooling_type = pooling_type | |
| self.pool_size = pool_size | |
| self.stride = stride | |
| self.tf2torch_tensor_name_prefix_torch = tf2torch_tensor_name_prefix_torch | |
| self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tf | |
| self.resnet1_dense = torch.nn.Linear( | |
| num_nodes_pooling_layer * 2, num_nodes_resnet1 | |
| ) | |
| self.resnet1_bn = torch.nn.BatchNorm1d( | |
| num_nodes_resnet1, eps=1e-3, momentum=batchnorm_momentum | |
| ) | |
| self.resnet2_dense = torch.nn.Linear(num_nodes_resnet1, num_nodes_last_layer) | |
| self.resnet2_bn = torch.nn.BatchNorm1d( | |
| num_nodes_last_layer, eps=1e-3, momentum=batchnorm_momentum | |
| ) | |
| def output_size(self) -> int: | |
| if self.embedding_node.startswith("resnet1"): | |
| return self.num_nodes_resnet1 | |
| elif self.embedding_node.startswith("resnet2"): | |
| return self.num_nodes_last_layer | |
| return self.num_nodes_pooling_layer | |
| def forward( | |
| self, | |
| xs_pad: torch.Tensor, | |
| ilens: torch.Tensor, | |
| prev_states: torch.Tensor = None, | |
| ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: | |
| endpoints = OrderedDict() | |
| res_out, ilens = super().forward(xs_pad, ilens) | |
| endpoints["resnet0_bn"] = res_out | |
| if self.pooling_type == "frame_gsp": | |
| features = statistic_pooling(res_out, ilens, (2,)) | |
| else: | |
| features, ilens = windowed_statistic_pooling( | |
| res_out, ilens, (2,), self.pool_size, self.stride | |
| ) | |
| features = features.transpose(1, 2) | |
| endpoints["pooling"] = features | |
| features = self.resnet1_dense(features) | |
| endpoints["resnet1_dense"] = features | |
| features = F.relu(features) | |
| endpoints["resnet1_relu"] = features | |
| features = self.resnet1_bn(features.transpose(1, 2)).transpose(1, 2) | |
| endpoints["resnet1_bn"] = features | |
| features = self.resnet2_dense(features) | |
| endpoints["resnet2_dense"] = features | |
| features = F.relu(features) | |
| endpoints["resnet2_relu"] = features | |
| features = self.resnet2_bn(features.transpose(1, 2)).transpose(1, 2) | |
| endpoints["resnet2_bn"] = features | |
| return endpoints[self.embedding_node], ilens, None | |
| def gen_tf2torch_map_dict(self): | |
| tensor_name_prefix_torch = self.tf2torch_tensor_name_prefix_torch | |
| tensor_name_prefix_tf = self.tf2torch_tensor_name_prefix_tf | |
| train_steps = 720000 | |
| map_dict_local = { | |
| # torch: conv1d.weight in "out_channel in_channel kernel_size" | |
| # tf : conv1d.weight in "kernel_size in_channel out_channel" | |
| # torch: linear.weight in "out_channel in_channel" | |
| # tf : dense.weight in "in_channel out_channel" | |
| "{}.pre_conv.weight".format(tensor_name_prefix_torch): { | |
| "name": "{}/pre_conv/kernel".format(tensor_name_prefix_tf), | |
| "squeeze": None, | |
| "transpose": (3, 2, 0, 1), | |
| }, | |
| "{}.pre_conv_bn.bias".format(tensor_name_prefix_torch): { | |
| "name": "{}/pre_conv_bn/beta".format(tensor_name_prefix_tf), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.pre_conv_bn.weight".format(tensor_name_prefix_torch): { | |
| "name": "{}/pre_conv_bn/gamma".format(tensor_name_prefix_tf), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.pre_conv_bn.running_mean".format(tensor_name_prefix_torch): { | |
| "name": "{}/pre_conv_bn/moving_mean".format(tensor_name_prefix_tf), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.pre_conv_bn.running_var".format(tensor_name_prefix_torch): { | |
| "name": "{}/pre_conv_bn/moving_variance".format(tensor_name_prefix_tf), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.pre_conv_bn.num_batches_tracked".format( | |
| tensor_name_prefix_torch | |
| ): train_steps, | |
| } | |
| for layer_idx in range(3): | |
| map_dict_local.update( | |
| { | |
| "{}.resnet{}_dense.weight".format( | |
| tensor_name_prefix_torch, layer_idx | |
| ): { | |
| "name": "{}/resnet{}_dense/kernel".format( | |
| tensor_name_prefix_tf, layer_idx | |
| ), | |
| "squeeze": None, | |
| "transpose": (2, 1, 0) if layer_idx == 0 else (1, 0), | |
| }, | |
| "{}.resnet{}_dense.bias".format( | |
| tensor_name_prefix_torch, layer_idx | |
| ): { | |
| "name": "{}/resnet{}_dense/bias".format( | |
| tensor_name_prefix_tf, layer_idx | |
| ), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.resnet{}_bn.weight".format( | |
| tensor_name_prefix_torch, layer_idx | |
| ): { | |
| "name": "{}/resnet{}_bn/gamma".format( | |
| tensor_name_prefix_tf, layer_idx | |
| ), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.resnet{}_bn.bias".format(tensor_name_prefix_torch, layer_idx): { | |
| "name": "{}/resnet{}_bn/beta".format( | |
| tensor_name_prefix_tf, layer_idx | |
| ), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.resnet{}_bn.running_mean".format( | |
| tensor_name_prefix_torch, layer_idx | |
| ): { | |
| "name": "{}/resnet{}_bn/moving_mean".format( | |
| tensor_name_prefix_tf, layer_idx | |
| ), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.resnet{}_bn.running_var".format( | |
| tensor_name_prefix_torch, layer_idx | |
| ): { | |
| "name": "{}/resnet{}_bn/moving_variance".format( | |
| tensor_name_prefix_tf, layer_idx | |
| ), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.resnet{}_bn.num_batches_tracked".format( | |
| tensor_name_prefix_torch, layer_idx | |
| ): train_steps, | |
| } | |
| ) | |
| for block_idx in range(len(self.layers_in_block)): | |
| for layer_idx in range(self.layers_in_block[block_idx]): | |
| for i in ["1", "2", "_sc"]: | |
| map_dict_local.update( | |
| { | |
| "{}.block_{}.layer_{}.conv{}.weight".format( | |
| tensor_name_prefix_torch, block_idx, layer_idx, i | |
| ): { | |
| "name": "{}/block_{}/layer_{}/conv{}/kernel".format( | |
| tensor_name_prefix_tf, block_idx, layer_idx, i | |
| ), | |
| "squeeze": None, | |
| "transpose": (3, 2, 0, 1), | |
| }, | |
| "{}.block_{}.layer_{}.bn{}.weight".format( | |
| tensor_name_prefix_torch, block_idx, layer_idx, i | |
| ): { | |
| "name": "{}/block_{}/layer_{}/bn{}/gamma".format( | |
| tensor_name_prefix_tf, block_idx, layer_idx, i | |
| ), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.block_{}.layer_{}.bn{}.bias".format( | |
| tensor_name_prefix_torch, block_idx, layer_idx, i | |
| ): { | |
| "name": "{}/block_{}/layer_{}/bn{}/beta".format( | |
| tensor_name_prefix_tf, block_idx, layer_idx, i | |
| ), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.block_{}.layer_{}.bn{}.running_mean".format( | |
| tensor_name_prefix_torch, block_idx, layer_idx, i | |
| ): { | |
| "name": "{}/block_{}/layer_{}/bn{}/moving_mean".format( | |
| tensor_name_prefix_tf, block_idx, layer_idx, i | |
| ), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.block_{}.layer_{}.bn{}.running_var".format( | |
| tensor_name_prefix_torch, block_idx, layer_idx, i | |
| ): { | |
| "name": "{}/block_{}/layer_{}/bn{}/moving_variance".format( | |
| tensor_name_prefix_tf, block_idx, layer_idx, i | |
| ), | |
| "squeeze": None, | |
| "transpose": None, | |
| }, | |
| "{}.block_{}.layer_{}.bn{}.num_batches_tracked".format( | |
| tensor_name_prefix_torch, block_idx, layer_idx, i | |
| ): train_steps, | |
| } | |
| ) | |
| return map_dict_local | |
| def convert_tf2torch( | |
| self, | |
| var_dict_tf, | |
| var_dict_torch, | |
| ): | |
| map_dict = self.gen_tf2torch_map_dict() | |
| var_dict_torch_update = dict() | |
| for name in sorted(var_dict_torch.keys(), reverse=False): | |
| if name.startswith(self.tf2torch_tensor_name_prefix_torch): | |
| if name in map_dict: | |
| if "num_batches_tracked" not in name: | |
| name_tf = map_dict[name]["name"] | |
| data_tf = var_dict_tf[name_tf] | |
| if map_dict[name]["squeeze"] is not None: | |
| data_tf = np.squeeze( | |
| data_tf, axis=map_dict[name]["squeeze"] | |
| ) | |
| if map_dict[name]["transpose"] is not None: | |
| data_tf = np.transpose(data_tf, map_dict[name]["transpose"]) | |
| data_tf = ( | |
| torch.from_numpy(data_tf).type(torch.float32).to("cpu") | |
| ) | |
| assert ( | |
| var_dict_torch[name].size() == data_tf.size() | |
| ), "{}, {}, {} != {}".format( | |
| name, name_tf, var_dict_torch[name].size(), data_tf.size() | |
| ) | |
| var_dict_torch_update[name] = data_tf | |
| logging.info( | |
| "torch tensor: {}, {}, loading from tf tensor: {}, {}".format( | |
| name, | |
| data_tf.size(), | |
| name_tf, | |
| var_dict_tf[name_tf].shape, | |
| ) | |
| ) | |
| else: | |
| var_dict_torch_update[name] = ( | |
| torch.from_numpy(np.array(map_dict[name])) | |
| .type(torch.int64) | |
| .to("cpu") | |
| ) | |
| logging.info( | |
| "torch tensor: {}, manually assigning to: {}".format( | |
| name, map_dict[name] | |
| ) | |
| ) | |
| else: | |
| logging.warning("{} is missed from tf checkpoint".format(name)) | |
| return var_dict_torch_update | |