modify demo page

crashmoon · Nov 18, 2020 · 41baf3b · 41baf3b
1 parent 35efa49
commit 41baf3b
Show file tree

Hide file tree

Showing 57 changed files with 333 additions and 7 deletions.
diff --git a/demo/LJ001-0001.wav b/demo/LJ001-0001.wav
diff --git a/demo/LJ001-0002.wav b/demo/LJ001-0002.wav
diff --git a/demo/LJ001-0003.wav b/demo/LJ001-0003.wav
diff --git a/demo/LJ001-0004.wav b/demo/LJ001-0004.wav
diff --git a/demo/LJ001-0005.wav b/demo/LJ001-0005.wav
diff --git a/demo/LJ001-0006.wav b/demo/LJ001-0006.wav
diff --git a/demo/LJ001-0007.wav b/demo/LJ001-0007.wav
diff --git a/demo/LJ001-0008.wav b/demo/LJ001-0008.wav
diff --git a/demo/LJ001-0009.wav b/demo/LJ001-0009.wav
diff --git a/demo/LJ001-0010.wav b/demo/LJ001-0010.wav
diff --git a/demo/ground-truth_10_waveglow.wav b/demo/ground-truth_10_waveglow.wav
diff --git a/demo/ground-truth_1_waveglow.wav b/demo/ground-truth_1_waveglow.wav
diff --git a/demo/ground-truth_2_waveglow.wav b/demo/ground-truth_2_waveglow.wav
diff --git a/demo/ground-truth_3_waveglow.wav b/demo/ground-truth_3_waveglow.wav
diff --git a/demo/ground-truth_4_waveglow.wav b/demo/ground-truth_4_waveglow.wav
diff --git a/demo/ground-truth_5_waveglow.wav b/demo/ground-truth_5_waveglow.wav
diff --git a/demo/ground-truth_6_waveglow.wav b/demo/ground-truth_6_waveglow.wav
diff --git a/demo/ground-truth_7_waveglow.wav b/demo/ground-truth_7_waveglow.wav
diff --git a/demo/ground-truth_8_waveglow.wav b/demo/ground-truth_8_waveglow.wav
diff --git a/demo/ground-truth_9_waveglow.wav b/demo/ground-truth_9_waveglow.wav
diff --git a/demo/long_sentence.wav b/demo/long_sentence.wav
diff --git a/demo/step_300000_1.png b/demo/step_300000_1.png
diff --git a/demo/step_300000_10.png b/demo/step_300000_10.png
diff --git a/demo/step_300000_10_melgan.wav b/demo/step_300000_10_melgan.wav
diff --git a/demo/step_300000_10_waveglow.wav b/demo/step_300000_10_waveglow.wav
diff --git a/demo/step_300000_1_melgan.wav b/demo/step_300000_1_melgan.wav
diff --git a/demo/step_300000_1_waveglow.wav b/demo/step_300000_1_waveglow.wav
diff --git a/demo/step_300000_2.png b/demo/step_300000_2.png
diff --git a/demo/step_300000_2_melgan.wav b/demo/step_300000_2_melgan.wav
diff --git a/demo/step_300000_2_waveglow.wav b/demo/step_300000_2_waveglow.wav
diff --git a/demo/step_300000_3.png b/demo/step_300000_3.png
diff --git a/demo/step_300000_3_melgan.wav b/demo/step_300000_3_melgan.wav
diff --git a/demo/step_300000_3_waveglow.wav b/demo/step_300000_3_waveglow.wav
diff --git a/demo/step_300000_4.png b/demo/step_300000_4.png
diff --git a/demo/step_300000_4_melgan.wav b/demo/step_300000_4_melgan.wav
diff --git a/demo/step_300000_4_waveglow.wav b/demo/step_300000_4_waveglow.wav
diff --git a/demo/step_300000_5.png b/demo/step_300000_5.png
diff --git a/demo/step_300000_5_melgan.wav b/demo/step_300000_5_melgan.wav
diff --git a/demo/step_300000_5_waveglow.wav b/demo/step_300000_5_waveglow.wav
diff --git a/demo/step_300000_6.png b/demo/step_300000_6.png
diff --git a/demo/step_300000_6_melgan.wav b/demo/step_300000_6_melgan.wav
diff --git a/demo/step_300000_6_waveglow.wav b/demo/step_300000_6_waveglow.wav
diff --git a/demo/step_300000_7.png b/demo/step_300000_7.png
diff --git a/demo/step_300000_7_melgan.wav b/demo/step_300000_7_melgan.wav
diff --git a/demo/step_300000_7_waveglow.wav b/demo/step_300000_7_waveglow.wav
diff --git a/demo/step_300000_8.png b/demo/step_300000_8.png
diff --git a/demo/step_300000_8_melgan.wav b/demo/step_300000_8_melgan.wav
diff --git a/demo/step_300000_8_waveglow.wav b/demo/step_300000_8_waveglow.wav
diff --git a/demo/step_300000_9.png b/demo/step_300000_9.png
diff --git a/demo/step_300000_9_melgan.wav b/demo/step_300000_9_melgan.wav
diff --git a/demo/step_300000_9_waveglow.wav b/demo/step_300000_9_waveglow.wav
diff --git a/evaluate.py b/evaluate.py
@@ -155,7 +155,6 @@ def evaluate(model, step, vocoder=None):
         vocoder = utils.get_melgan()
     elif hp.vocoder == 'waveglow':
         vocoder = utils.get_waveglow()
-    vocoder.to(device)
 
     # Init directories
     if not os.path.exists(hp.log_path):

diff --git a/index.html b/index.html
@@ -0,0 +1,328 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <meta charset="utf-8" />
+  <meta name="viewport" content="width=device-width,initial-scale=1" />
+  <title>FastSpeech 2 Audio Samples</title>
+  <style>
+    body {
+      margin: 0;
+      padding: 10px 30px;
+      background: #fff;
+      color: #111;
+      font-size: 17px;
+      font-family: sans-serif;
+      font-weight: 400;
+      line-height: 1.8;
+      overflow-x: hidden;
+      -webkit-font-smoothing: antialiased;
+    }
+
+    img {
+      width: 30%;
+    }
+  </style>
+</head>
+
+<body>
+  Audio samples and spectrograms generated by <a href="https://github.com/ming024/FastSpeech2">ming024's FastSpeech2
+    implementation</a>.
+
+  <hr>
+  <table>
+    <thead>
+      <tr>
+        <th style="text-align: center">Very Long Sentence Generation</th>
+      </tr>
+    </thead>
+    <tbody>
+      <td style="text-align: center"><audio controls="controls">
+          <source src="./demo/long_sentence.wav" autoplay /></audio></td>
+      </tr>
+    </tbody>
+  </table>
+  <em>Advanced text to speech models such as FastSpeech can synthesize speech significantly faster than previous
+    autoregressive models with comparable quality. The training of FastSpeech model relies on an autoregressive teacher
+    model for duration prediction and knowledge distillation, which can ease the one-to-many mapping problem in TTS.
+    However, FastSpeech has several disadvantages: 1) the teacher student distillation pipeline is complicated, 2) the
+    duration extracted from the teacher model is not accurate enough, and the target mel-spectrograms distilled from
+    teacher model suffer from information loss due to data simplification, both of which limit the voice quality.</em>
+  <hr>
+
+  <ul>
+    <li>These samples are generated by a model trained for 300k steps
+      <a href="https://drive.google.com/file/d/1jXNDPMt1ybTN97_MztoTFyrPIthoQuSO/view?usp=sharing">checkpoint</a>.</li>
+    <li>These sentences are the first 10 sentences in the held-out validation set (LJ001, LJ002 and LJ003).</li>
+    <li>These samples are selected without any human inspection.</li>
+    <li>All synthesized samples are converted from mel-spectrograms to audio signals by <a
+        href="https://github.com/NVIDIA/waveglow">NVIDIA's pretrained Waveglow</a> and <a
+        href="https://github.com/seungwonpark/melgan">seungwonpark's pretrained MelGAN</a>.</li>
+  </ul>
+
+  <hr>
+
+  <table>
+    <thead>
+      <tr>
+        <th style="text-align: center">Ground-Truth</th>
+        <th style="text-align: center">Ground-Truth(WaveGlow)</th>
+        <th style="text-align: center">Synthesized(WaveGlow)</th>
+        <th style="text-align: center">Synthesized(MelGAN)</th>
+      </tr>
+    </thead>
+    <tbody>
+      <tr>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/LJ001-0001.wav" autoplay /></audio></td>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/ground-truth_1_waveglow.wav" autoplay /></audio></td>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/step_300000_1_waveglow.wav" autoplay /></audio></td>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/step_300000_1_melgan.wav" autoplay /></audio></td>
+      </tr>
+    </tbody>
+  </table>
+  <em>Printing, in the only sense with which we are at present concerned, differs from most if not from all the arts and
+    crafts represented in the Exhibition</em>
+
+  <hr>
+
+  <table>
+    <thead>
+      <tr>
+        <th style="text-align: center">Ground-Truth</th>
+        <th style="text-align: center">Ground-Truth(WaveGlow)</th>
+        <th style="text-align: center">Synthesized(WaveGlow)</th>
+        <th style="text-align: center">Synthesized(MelGAN)</th>
+      </tr>
+    </thead>
+    <tbody>
+      <tr>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/LJ001-0002.wav" autoplay /></audio></td>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/ground-truth_2_waveglow.wav" autoplay /></audio></td>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/step_300000_2_waveglow.wav" autoplay /></audio></td>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/step_300000_2_melgan.wav" autoplay /></audio></td>
+      </tr>
+    </tbody>
+  </table>
+  <em>in being comparatively modern.</em>
+
+  <hr>
+
+  <table>
+    <thead>
+      <tr>
+        <th style="text-align: center">Ground-Truth</th>
+        <th style="text-align: center">Ground-Truth(WaveGlow)</th>
+        <th style="text-align: center">Synthesized(WaveGlow)</th>
+        <th style="text-align: center">Synthesized(MelGAN)</th>
+      </tr>
+    </thead>
+    <tbody>
+      <tr>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/LJ001-0003.wav" autoplay /></audio></td>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/ground-truth_3_waveglow.wav" autoplay /></audio></td>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/step_300000_3_waveglow.wav" autoplay /></audio></td>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/step_300000_3_melgan.wav" autoplay /></audio></td>
+      </tr>
+    </tbody>
+  </table>
+  <em>For although the Chinese took impressions from wood blocks engraved in relief for centuries before the woodcutters
+    of the Netherlands, by a similar process</em>
+
+  <hr>
+
+  <table>
+    <thead>
+      <tr>
+        <th style="text-align: center">Ground-Truth</th>
+        <th style="text-align: center">Ground-Truth(WaveGlow)</th>
+        <th style="text-align: center">Synthesized(WaveGlow)</th>
+        <th style="text-align: center">Synthesized(MelGAN)</th>
+      </tr>
+    </thead>
+    <tbody>
+      <tr>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/LJ001-0004.wav" autoplay /></audio></td>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/ground-truth_4_waveglow.wav" autoplay /></audio></td>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/step_300000_4_waveglow.wav" autoplay /></audio></td>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/step_300000_4_melgan.wav" autoplay /></audio></td>
+      </tr>
+    </tbody>
+  </table>
+  <em>produced the block books, which were the immediate predecessors of the true printed book,</em>
+
+  <hr>
+
+  <table>
+    <thead>
+      <tr>
+        <th style="text-align: center">Ground-Truth</th>
+        <th style="text-align: center">Ground-Truth(WaveGlow)</th>
+        <th style="text-align: center">Synthesized(WaveGlow)</th>
+        <th style="text-align: center">Synthesized(MelGAN)</th>
+      </tr>
+    </thead>
+    <tbody>
+      <tr>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/LJ001-0005.wav" autoplay /></audio></td>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/ground-truth_5_waveglow.wav" autoplay /></audio></td>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/step_300000_5_waveglow.wav" autoplay /></audio></td>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/step_300000_5_melgan.wav" autoplay /></audio></td>
+      </tr>
+    </tbody>
+  </table>
+  <em>the invention of movable metal letters in the middle of the fifteenth century may justly be considered as the
+    invention of the art of printing.</em>
+
+  <hr>
+
+  <table>
+    <thead>
+      <tr>
+        <th style="text-align: center">Ground-Truth</th>
+        <th style="text-align: center">Ground-Truth(WaveGlow)</th>
+        <th style="text-align: center">Synthesized(WaveGlow)</th>
+        <th style="text-align: center">Synthesized(MelGAN)</th>
+      </tr>
+    </thead>
+    <tbody>
+      <tr>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/LJ001-0006.wav" autoplay /></audio></td>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/ground-truth_6_waveglow.wav" autoplay /></audio></td>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/step_300000_6_waveglow.wav" autoplay /></audio></td>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/step_300000_6_melgan.wav" autoplay /></audio></td>
+      </tr>
+    </tbody>
+  </table>
+  <em>And it is worth mention in passing that, as an example of fine typography,</em>
+
+  <hr>
+
+  <table>
+    <thead>
+      <tr>
+        <th style="text-align: center">Ground-Truth</th>
+        <th style="text-align: center">Ground-Truth(WaveGlow)</th>
+        <th style="text-align: center">Synthesized(WaveGlow)</th>
+        <th style="text-align: center">Synthesized(MelGAN)</th>
+      </tr>
+    </thead>
+    <tbody>
+      <tr>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/LJ001-0007.wav" autoplay /></audio></td>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/ground-truth_7_waveglow.wav" autoplay /></audio></td>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/step_300000_7_waveglow.wav" autoplay /></audio></td>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/step_300000_7_melgan.wav" autoplay /></audio></td>
+      </tr>
+    </tbody>
+  </table>
+  <em>the earliest book printed with movable types, the Gutenberg, or "forty-two line Bible" of about 1455,</em>
+
+  <hr>
+
+  <table>
+    <thead>
+      <tr>
+        <th style="text-align: center">Ground-Truth</th>
+        <th style="text-align: center">Ground-Truth(WaveGlow)</th>
+        <th style="text-align: center">Synthesized(WaveGlow)</th>
+        <th style="text-align: center">Synthesized(MelGAN)</th>
+      </tr>
+    </thead>
+    <tbody>
+      <tr>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/LJ001-0008.wav" autoplay /></audio></td>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/ground-truth_8_waveglow.wav" autoplay /></audio></td>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/step_300000_8_waveglow.wav" autoplay /></audio></td>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/step_300000_8_melgan.wav" autoplay /></audio></td>
+      </tr>
+    </tbody>
+  </table>
+  <em>has never been surpassed.</em>
+
+  <hr>
+
+  <table>
+    <thead>
+      <tr>
+        <th style="text-align: center">Ground-Truth</th>
+        <th style="text-align: center">Ground-Truth(WaveGlow)</th>
+        <th style="text-align: center">Synthesized(WaveGlow)</th>
+        <th style="text-align: center">Synthesized(MelGAN)</th>
+      </tr>
+    </thead>
+    <tbody>
+      <tr>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/LJ001-0009.wav" autoplay /></audio></td>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/ground-truth_9_waveglow.wav" autoplay /></audio></td>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/step_300000_9_waveglow.wav" autoplay /></audio></td>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/step_300000_9_melgan.wav" autoplay /></audio></td>
+      </tr>
+    </tbody>
+  </table>
+  <em>Printing, then, for our purpose, may be considered as the art of making books by means of movable types.</em>
+
+  <hr>
+
+  <table>
+    <thead>
+      <tr>
+        <th style="text-align: center">Ground-Truth</th>
+        <th style="text-align: center">Ground-Truth(WaveGlow)</th>
+        <th style="text-align: center">Synthesized(WaveGlow)</th>
+        <th style="text-align: center">Synthesized(MelGAN)</th>
+      </tr>
+    </thead>
+    <tbody>
+      <tr>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/LJ001-0010.wav" autoplay /></audio></td>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/ground-truth_10_waveglow.wav" autoplay /></audio></td>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/step_300000_10_waveglow.wav" autoplay /></audio></td>
+        <td style="text-align: center"><audio controls="controls">
+            <source src="./demo/step_300000_10_melgan.wav" autoplay /></audio></td>
+      </tr>
+    </tbody>
+  </table>
+  <em>Now, as all books not primarily intended as picture-books consist principally of types composed to form
+    letterpress,</em>
+</body>
+
+</html>
diff --git a/synth/LJSpeech/step_300000.png b/synth/LJSpeech/step_300000.png
diff --git a/synthesize.py b/synthesize.py
@@ -89,10 +89,8 @@ def synthesize(model, waveglow, melgan, text, sentence, prefix=''):
     melgan = waveglow = None
     if hp.vocoder == 'melgan':
         melgan = utils.get_melgan()
-        melgan.to(device)
     elif hp.vocoder == 'waveglow':
         waveglow = utils.get_waveglow()
-        waveglow.to(device)
 
     for sentence in sentences:
         text = preprocess(sentence)

diff --git a/train.py b/train.py
@@ -56,10 +56,8 @@ def main(args):
     # Load vocoder
     if hp.vocoder == 'melgan':
         melgan = utils.get_melgan()
-        melgan.to(device)
     elif hp.vocoder == 'waveglow':
         waveglow = utils.get_waveglow()
-        waveglow.to(device)
 
     # Init logger
     log_path = hp.log_path

diff --git a/utils.py b/utils.py
@@ -37,7 +37,7 @@ def get_alignment(tier):
             end_idx = len(phones)
         else:
             phones.append(p)
-        durations.append(int(e*hp.sampling_rate/hp.hop_length)-int(s*hp.sampling_rate/hp.hop_length))
+        durations.append(int(np.round(e*hp.sampling_rate/hp.hop_length)-np.round(s*hp.sampling_rate/hp.hop_length)))
 
     # Trimming tailing silences
     phones = phones[:end_idx]
@@ -92,7 +92,7 @@ def add_axis(fig, old_ax, offset=0):
         ax2.set_ylabel('Energy', color='darkviolet')
         ax2.yaxis.set_label_position('right')
         ax2.tick_params(labelsize='x-small', colors='darkviolet', bottom=False, labelbottom=False, left=False, labelleft=False, right=True, labelright=True)
-
+        
     plt.savefig(filename, dpi=200)
     plt.close()
 
@@ -113,6 +113,7 @@ def get_waveglow():
     for m in waveglow.modules():
         if 'Conv' in str(type(m)):
             setattr(m, 'padding_mode', 'zeros')
+    waveglow.to(device)
 
     return waveglow
 
@@ -132,6 +133,8 @@ def melgan_infer(mel, melgan, path):
 def get_melgan():
     melgan = torch.hub.load('seungwonpark/melgan', 'melgan')
     melgan.eval()
+    melgan.to(device)
+
     return melgan
 
 def pad_1D(inputs, PAD=0):