Run examples on TaskCluster

Fixes mozilla#2353

examples/ffmpeg_vad_streaming/test.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -xe
+
+THIS=$(dirname "$0")
+
+pushd ${THIS}
+  source ../tests.sh
+
+  npm install $(get_npm_package_url)
+  npm install
+
+  node ./index.js --audio $HOME/DeepSpeech/audio/2830-3980-0043.wav \
+                  --lm $HOME/DeepSpeech/models/lm.binary \
+                  --trie $HOME/DeepSpeech/models/trie \
+                  --model $HOME/DeepSpeech/models/output_graph.pbmm \
+                  --alphabet $HOME/DeepSpeech/models/alphabet.txt
+
+  node ./index.js --audio $HOME/DeepSpeech/audio/4507-16021-0012.wav \
+                  --lm $HOME/DeepSpeech/models/lm.binary \
+                  --trie $HOME/DeepSpeech/models/trie \
+                  --model $HOME/DeepSpeech/models/output_graph.pbmm \
+                  --alphabet $HOME/DeepSpeech/models/alphabet.txt
+
+  node ./index.js --audio $HOME/DeepSpeech/audio/8455-210777-0068.wav \
+                  --lm $HOME/DeepSpeech/models/lm.binary \
+                  --trie $HOME/DeepSpeech/models/trie \
+                  --model $HOME/DeepSpeech/models/output_graph.pbmm \
+                  --alphabet $HOME/DeepSpeech/models/alphabet.txt
+popd

examples/mic_vad_streaming/mic_vad_streaming.py

@@ -20,8 +20,10 @@ class Audio(object):
     CHANNELS = 1
     BLOCKS_PER_SECOND = 50
 
-    def __init__(self, callback=None, device=None, input_rate=RATE_PROCESS):
+    def __init__(self, callback=None, device=None, input_rate=RATE_PROCESS, file=None):
         def proxy_callback(in_data, frame_count, time_info, status):
+            if self.chunk is not None:
+                in_data = self.wf.readframes(self.chunk)
             callback(in_data)
             return (None, pyaudio.paContinue)
         if callback is None: callback = lambda in_data: self.buffer_queue.put(in_data)
@@ -42,9 +44,13 @@ def proxy_callback(in_data, frame_count, time_info, status):
             'stream_callback': proxy_callback,
         }
 
+        self.chunk = None
         # if not default device
         if self.device:
             kwargs['input_device_index'] = self.device
+        elif file is not None:
+            self.chunk = 320
+            self.wf = wave.open(file, 'rb')
 
         self.stream = self.pa.open(**kwargs)
         self.stream.start_stream()
@@ -96,8 +102,8 @@ def write_wav(self, filename, data):
 class VADAudio(Audio):
     """Filter & segment audio with voice activity detection."""
 
-    def __init__(self, aggressiveness=3, device=None, input_rate=None):
-        super().__init__(device=device, input_rate=input_rate)
+    def __init__(self, aggressiveness=3, device=None, input_rate=None, file=None):
+        super().__init__(device=device, input_rate=input_rate, file=file)
         self.vad = webrtcvad.Vad(aggressiveness)
 
     def frame_generator(self):
@@ -121,6 +127,9 @@ def vad_collector(self, padding_ms=300, ratio=0.75, frames=None):
         triggered = False
 
         for frame in frames:
+            if len(frame) < 640:
+                return
+
             is_speech = self.vad.is_speech(frame, self.sample_rate)
 
             if not triggered:
@@ -162,7 +171,8 @@ def main(ARGS):
     # Start audio with VAD
     vad_audio = VADAudio(aggressiveness=ARGS.vad_aggressiveness,
                          device=ARGS.device,
-                         input_rate=ARGS.rate)
+                         input_rate=ARGS.rate,
+                         file=ARGS.file)
     print("Listening (ctrl-C to exit)...")
     frames = vad_audio.vad_collector()
 
@@ -204,6 +214,8 @@ def main(ARGS):
         help="Disable spinner")
     parser.add_argument('-w', '--savewav',
         help="Save .wav files of utterences to given directory")
+    parser.add_argument('-f', '--file',
+        help="Read from .wav file instead of microphone")
 
     parser.add_argument('-m', '--model', required=True,
                         help="Path to the model (protocol buffer binary file, or entire directory containing all standard-named files for model)")
@@ -214,7 +226,7 @@ def main(ARGS):
     parser.add_argument('-t', '--trie', default='trie',
                         help="Path to the language model trie file created with native_client/generate_trie. Default: trie")
     parser.add_argument('-d', '--device', type=int, default=None,
-                        help="Device input index (Int) as listed by pyaudio.PyAudio.get_device_info_by_index(). If not provided, falls back to PyAudio.get_default_device()")
+                        help="Device input index (Int) as listed by pyaudio.PyAudio.get_device_info_by_index(). If not provided, falls back to PyAudio.get_default_device().")
     parser.add_argument('-r', '--rate', type=int, default=DEFAULT_SAMPLE_RATE,
                         help=f"Input device sample rate. Default: {DEFAULT_SAMPLE_RATE}. Your device may require 44100.")
     parser.add_argument('-nf', '--n_features', type=int, default=N_FEATURES,

examples/mic_vad_streaming/test.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -xe
+
+THIS=$(dirname "$0")
+
+pushd ${THIS}
+  source ../tests.sh
+
+  pip install --user $(get_python_wheel_url "$1")
+  pip install --user -r requirements.txt
+
+  pulseaudio &
+
+  python mic_vad_streaming.py \
+	  --model $HOME/DeepSpeech/models/output_graph.pbmm \
+	  --alphabet $HOME/DeepSpeech/models/alphabet.txt \
+	  --lm $HOME/DeepSpeech/models/lm.binary \
+	  --trie $HOME/DeepSpeech/models/trie \
+	  --file $HOME/DeepSpeech/audio/2830-3980-0043.wav
+popd

examples/nodejs_wav/test.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -xe
+
+THIS=$(dirname "$0")
+
+pushd ${THIS}
+  source ../tests.sh
+
+  npm install $(get_npm_package_url)
+  npm install
+
+  ln -s $HOME/DeepSpeech/models models
+
+  node index.js $HOME/DeepSpeech/audio/2830-3980-0043.wav
+  node index.js $HOME/DeepSpeech/audio/8455-210777-0068.wav
+  node index.js $HOME/DeepSpeech/audio/4507-16021-0012.wav
+popd

examples/tests.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -xe
+
+THIS=$(dirname "$0")
+
+source ../../taskcluster/tc-tests-utils.sh
+
+DEP_TASK_ID=$(curl -s https://queue.taskcluster.net/v1/task/${TASK_ID} | python -c 'import json; import sys; print(" ".join(json.loads(sys.stdin.read())["dependencies"]));')
+
+get_python_wheel_url()
+{
+  local this_python_version=$1
+
+  extract_python_versions "${this_python_version}" "pyver" "pyver_pkg" "py_unicode_type" "pyconf" "pyalias"
+
+  echo "$(get_python_pkg_url "${pyver_pkg}" "${py_unicode_type}" "deepspeech" https://queue.taskcluster.net/v1/task/${DEP_TASK_ID}/artifacts/public)"
+}
+
+get_npm_package_url()
+{
+  echo "https://queue.taskcluster.net/v1/task/${DEP_TASK_ID}/artifacts/public/deepspeech-${DS_VERSION}.tgz"
+}

examples/vad_transcriber/audioTranscript_cmd.py

@@ -22,9 +22,9 @@ def main(args):
     parser.add_argument('--stream', required=False, action='store_true',
                         help='To use deepspeech streaming interface')
     args = parser.parse_args()
-    if args.stream is True and len(sys.argv[1:]) == 3:
+    if args.stream is True:
              print("Opening mic for streaming")
-    elif args.audio is not None and len(sys.argv[1:]) == 6:
+    elif args.audio is not None:
             logging.debug("Transcribing audio file @ %s" % args.audio)
     else:
         parser.print_help()

examples/vad_transcriber/test.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -xe
+
+THIS=$(dirname "$0")
+
+pushd ${THIS}
+  source ../tests.sh
+
+  pip install --user $(get_python_wheel_url "$1")
+  pip install --user -r requirements.txt
+
+  python audioTranscript_cmd.py \
+	  --audio $HOME/DeepSpeech/audio/2830-3980-0043.wav \
+	  --aggressive 0 \
+	  --model $HOME/DeepSpeech/models/
+
+  python audioTranscript_cmd.py \
+	  --audio $HOME/DeepSpeech/audio/2830-3980-0043.wav \
+	  --aggressive 0 \
+	  --model $HOME/DeepSpeech/models/ \
+	  --stream
+popd

taskcluster/examples-base.tyml

@@ -0,0 +1,61 @@
+$if: '(event.event != "push") && (event.event != "tag")'
+then:
+  taskId: ${taskcluster.taskId}
+  provisionerId: ${taskcluster.docker.provisionerId}
+  workerType: ${taskcluster.docker.workerType}
+  taskGroupId: ${taskcluster.taskGroupId}
+  schedulerId: ${taskcluster.schedulerId}
+  dependencies:
+    $map: { $eval: build.dependencies }
+    each(b):
+      $eval: as_slugid(b)
+  created: { $fromNow: '0 sec' }
+  deadline: { $fromNow: '1 day' }
+  expires: { $fromNow: '7 days' }
+
+  extra:
+    github:
+      { $eval: taskcluster.github_events.pull_request }
+
+  routes:
+    - "notify.irc-channel.${notifications.irc}.on-exception"
+    - "notify.irc-channel.${notifications.irc}.on-failed"
+
+  scopes: [
+    "queue:route:notify.irc-channel.*"
+  ]
+
+  payload:
+    maxRunTime: { $eval: to_int(build.maxRunTime) }
+    image: ${build.docker_image}
+
+    env:
+      DEEPSPEECH_MODEL: "https://github.com/lissyx/DeepSpeech/releases/download/tc-0.6.0/models.tar.gz"
+      DEEPSPEECH_AUDIO: "https://github.com/mozilla/DeepSpeech/releases/download/v0.4.1/audio-0.4.1.tar.gz"
+      PIP_DEFAULT_TIMEOUT: "60"
+
+    command:
+      - "/bin/bash"
+      - "--login"
+      - "-cxe"
+      - $let:
+          extraSystemSetup: { $eval: strip(str(build.system_setup)) }
+        in: >
+          apt-get -qq update && apt-get -qq -y upgrade && apt-get -qq -y install git sox sudo && ${extraSystemSetup} &&
+          adduser --system --home ${system.homedir.linux} ${system.username} &&
+          cd ${system.homedir.linux} &&
+          echo -e "#!/bin/bash\nset -xe\n env && id && mkdir ~/DeepSpeech/ && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && wget -O - $DEEPSPEECH_MODEL | tar -C ~/DeepSpeech/ -xzvf - && wget -O - $DEEPSPEECH_AUDIO | tar -C ~/DeepSpeech/ -xzvf - " > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
+          sudo -H -u ${system.username} /bin/bash /tmp/clone.sh &&
+          sudo -H -u ${system.username} --preserve-env /bin/bash ${build.args.tests_cmdline}
+
+    artifacts:
+      "public":
+        type: "directory"
+        path: "/tmp/artifacts/"
+        expires: { $fromNow: '7 days' }
+
+  metadata:
+    name: ${build.metadata.name}
+    description: ${build.metadata.description}
+    owner: ${event.head.user.email}
+    source: ${event.head.repo.url}

taskcluster/examples-ffmpeg_vad_streaming-node10.yml

@@ -0,0 +1,13 @@
+build:
+  template_file: examples-base.tyml
+  docker_image: "node:10"
+  dependencies:
+    - "node-package-cpu"
+  system_setup:
+    >
+      apt-get -qq -y install ffmpeg
+  args:
+    tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/examples/ffmpeg_vad_streaming/test.sh"
+  metadata:
+    name: "DeepSpeech examples: ffmpeg VAD Streaming NodeJS v10.x"
+    description: "DeepSpeech examples: ffmpeg VAD Streaming NodeJS v10.x"

taskcluster/examples-ffmpeg_vad_streaming-node8.yml

@@ -0,0 +1,13 @@
+build:
+  template_file: examples-base.tyml
+  docker_image: "node:8"
+  dependencies:
+    - "node-package-cpu"
+  system_setup:
+    >
+      apt-get -qq -y install ffmpeg
+  args:
+    tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/examples/ffmpeg_vad_streaming/test.sh"
+  metadata:
+    name: "DeepSpeech examples: ffmpeg VAD Streaming NodeJS v8.x"
+    description: "DeepSpeech examples: ffmpeg VAD Streaming NodeJS v8.x"

taskcluster/examples-mic_vad_streaming-py36.yml

@@ -0,0 +1,13 @@
+build:
+  template_file: examples-base.tyml
+  docker_image: "python:3.6"
+  dependencies:
+    - "linux-amd64-cpu-opt"
+  system_setup:
+    >
+      apt-get -qq -y install portaudio19-dev pulseaudio
+  args:
+    tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/examples/mic_vad_streaming/test.sh 3.6.0:m"
+  metadata:
+    name: "DeepSpeech examples: mic VAD streaming Py3.6"
+    description: "DeepSpeech examples: mic VAD streaming Python 3.6"

taskcluster/examples-mic_vad_streaming-py37.yml

@@ -0,0 +1,13 @@
+build:
+  template_file: examples-base.tyml
+  docker_image: "python:3.7"
+  dependencies:
+    - "linux-amd64-cpu-opt"
+  system_setup:
+    >
+      apt-get -qq -y install portaudio19-dev pulseaudio
+  args:
+    tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/examples/mic_vad_streaming/test.sh 3.7.0:m"
+  metadata:
+    name: "DeepSpeech examples: mic VAD streaming Py3.7"
+    description: "DeepSpeech examples: mic VAD streaming Python 3.7"

taskcluster/examples-nodejs_wav-node10.yml

@@ -0,0 +1,10 @@
+build:
+  template_file: examples-base.tyml
+  docker_image: "node:10"
+  dependencies:
+    - "node-package-cpu"
+  args:
+    tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/examples/nodejs_wav/test.sh"
+  metadata:
+    name: "DeepSpeech examples: NodeJS WAV NodeJS v10.x"
+    description: "DeepSpeech examples: NodeJS WAV NodeJS v10.x"

taskcluster/examples-nodejs_wav-node8.yml

@@ -0,0 +1,10 @@
+build:
+  template_file: examples-base.tyml
+  docker_image: "node:8"
+  dependencies:
+    - "node-package-cpu"
+  args:
+    tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/examples/nodejs_wav/test.sh"
+  metadata:
+    name: "DeepSpeech examples: NodeJS WAV NodeJS v8.x"
+    description: "DeepSpeech examples: NodeJS WAV NodeJS v8.x"

taskcluster/examples-vad_transcriber-py35.yml

@@ -0,0 +1,10 @@
+build:
+  template_file: examples-base.tyml
+  docker_image: "python:3.5"
+  dependencies:
+    - "linux-amd64-cpu-opt"
+  args:
+    tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/examples/vad_transcriber/test.sh 3.5.0:m"
+  metadata:
+    name: "DeepSpeech examples: VAD transcriber Py3.5"
+    description: "DeepSpeech examples: VAD transcriberaming Python 3.5"

taskcluster/examples-vad_transcriber-py36.yml

@@ -0,0 +1,10 @@
+build:
+  template_file: examples-base.tyml
+  docker_image: "python:3.6"
+  dependencies:
+    - "linux-amd64-cpu-opt"
+  args:
+    tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/examples/vad_transcriber/test.sh 3.6.0:m"
+  metadata:
+    name: "DeepSpeech examples: VAD transcriber Py3.6"
+    description: "DeepSpeech examples: VAD transcriberaming Python 3.6"

taskcluster/examples-vad_transcriber-py37.yml

@@ -0,0 +1,10 @@
+build:
+  template_file: examples-base.tyml
+  docker_image: "python:3.7"
+  dependencies:
+    - "linux-amd64-cpu-opt"
+  args:
+    tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/examples/vad_transcriber/test.sh 3.7.0:m"
+  metadata:
+    name: "DeepSpeech examples: VAD transcriber Py3.7"
+    description: "DeepSpeech examples: VAD transcriberaming Python 3.7"