forked from ming024/FastSpeech2
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathModels.py
135 lines (100 loc) · 4.72 KB
/
Models.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
import torch
import torch.nn as nn
import numpy as np
import transformer.Constants as Constants
from transformer.Layers import FFTBlock
from text.symbols import symbols
import hparams as hp
def get_sinusoid_encoding_table(n_position, d_hid, padding_idx=None):
''' Sinusoid position encoding table '''
def cal_angle(position, hid_idx):
return position / np.power(10000, 2 * (hid_idx // 2) / d_hid)
def get_posi_angle_vec(position):
return [cal_angle(position, hid_j) for hid_j in range(d_hid)]
sinusoid_table = np.array([get_posi_angle_vec(pos_i)
for pos_i in range(n_position)])
sinusoid_table[:, 0::2] = np.sin(sinusoid_table[:, 0::2]) # dim 2i
sinusoid_table[:, 1::2] = np.cos(sinusoid_table[:, 1::2]) # dim 2i+1
if padding_idx is not None:
# zero vector for padding dimension
sinusoid_table[padding_idx] = 0.
return torch.FloatTensor(sinusoid_table)
class Encoder(nn.Module):
''' Encoder '''
def __init__(self,
n_src_vocab=len(symbols)+1,
len_max_seq=hp.max_seq_len,
d_word_vec=hp.encoder_hidden,
n_layers=hp.encoder_layer,
n_head=hp.encoder_head,
d_k=hp.encoder_hidden // hp.encoder_head,
d_v=hp.encoder_hidden // hp.encoder_head,
d_model=hp.encoder_hidden,
d_inner=hp.fft_conv1d_filter_size,
dropout=hp.encoder_dropout):
super(Encoder, self).__init__()
n_position = len_max_seq + 1
self.src_word_emb = nn.Embedding(
n_src_vocab, d_word_vec, padding_idx=Constants.PAD)
self.position_enc = nn.Parameter(
get_sinusoid_encoding_table(n_position, d_word_vec).unsqueeze(0), requires_grad=False)
self.layer_stack = nn.ModuleList([FFTBlock(
d_model, d_inner, n_head, d_k, d_v, dropout=dropout) for _ in range(n_layers)])
def forward(self, src_seq, mask, return_attns=False):
enc_slf_attn_list = []
batch_size, max_len = src_seq.shape[0], src_seq.shape[1]
# -- Prepare masks
slf_attn_mask = mask.unsqueeze(1).expand(-1, max_len, -1)
# -- Forward
if not self.training and src_seq.shape[1] > hp.max_seq_len:
enc_output = self.src_word_emb(src_seq) + get_sinusoid_encoding_table(src_seq.shape[1], hp.encoder_hidden)[
:src_seq.shape[1], :].unsqueeze(0).expand(batch_size, -1, -1).to(src_seq.device)
else:
enc_output = self.src_word_emb(
src_seq) + self.position_enc[:, :max_len, :].expand(batch_size, -1, -1)
for enc_layer in self.layer_stack:
enc_output, enc_slf_attn = enc_layer(
enc_output,
mask=mask,
slf_attn_mask=slf_attn_mask)
if return_attns:
enc_slf_attn_list += [enc_slf_attn]
return enc_output
class Decoder(nn.Module):
""" Decoder """
def __init__(self,
len_max_seq=hp.max_seq_len,
d_word_vec=hp.encoder_hidden,
n_layers=hp.decoder_layer,
n_head=hp.decoder_head,
d_k=hp.decoder_hidden // hp.decoder_head,
d_v=hp.decoder_hidden // hp.decoder_head,
d_model=hp.decoder_hidden,
d_inner=hp.fft_conv1d_filter_size,
dropout=hp.decoder_dropout):
super(Decoder, self).__init__()
n_position = len_max_seq + 1
self.position_enc = nn.Parameter(
get_sinusoid_encoding_table(n_position, d_word_vec).unsqueeze(0), requires_grad=False)
self.layer_stack = nn.ModuleList([FFTBlock(
d_model, d_inner, n_head, d_k, d_v, dropout=dropout) for _ in range(n_layers)])
def forward(self, enc_seq, mask, return_attns=False):
dec_slf_attn_list = []
batch_size, max_len = enc_seq.shape[0], enc_seq.shape[1]
# -- Prepare masks
slf_attn_mask = mask.unsqueeze(1).expand(-1, max_len, -1)
# -- Forward
if not self.training and enc_seq.shape[1] > hp.max_seq_len:
dec_output = enc_seq + get_sinusoid_encoding_table(enc_seq.shape[1], hp.decoder_hidden)[
:enc_seq.shape[1], :].unsqueeze(0).expand(batch_size, -1, -1).to(enc_seq.device)
else:
dec_output = enc_seq + \
self.position_enc[:, :max_len, :].expand(batch_size, -1, -1)
for dec_layer in self.layer_stack:
dec_output, dec_slf_attn = dec_layer(
dec_output,
mask=mask,
slf_attn_mask=slf_attn_mask)
if return_attns:
dec_slf_attn_list += [dec_slf_attn]
return dec_output