-Tried suspending and recording to save resources, worked on Desktop but recorder stops posting data on Android.

-Added speech prompts
-Split up next question transitions via events and callbacks, screen will now congratulate user and repeat the answer before continuing

Author: GamerSg

DeepSpeechC.cpp

@@ -84,14 +84,23 @@ DeepSpeech::~DeepSpeech()
 
 void DeepSpeech::startRecording()
 {
+    qInfo()<<"Recording";
     std::lock_guard<std::mutex> guard(lock);
+    if(recorder->state() == QAudio::State::SuspendedState)
+    {
+        recorder->resume();
+    }
+    qInfo()<<"Recorder state now is "<<recorder->state();
     int ret = DS_CreateStream(model, &state);
     qInfo()<<"Setting up Model Stream : "<<ret;
 }
 
 void DeepSpeech::stopRecording()
 {
+    qInfo()<<"Stopping Recording";
     std::lock_guard<std::mutex> guard(lock);
+    //recorder->suspend();
+    qInfo()<<"Recorder state now is "<<recorder->state();
     qInfo()<<"Clearing Hotwords and Freeing stream"<<this->thread();
     DS_ClearHotWords(model);
     if(state)
@@ -107,24 +116,24 @@ void DeepSpeech::setHotword(QString hotword)
    DS_AddHotWord(model, word.c_str(), 5.0f );
 }
 
-void DeepSpeech::clearHotwords()
-{
- //   DS_ClearHotWords(model);
-}
-
-
 void DeepSpeech::transcribeAudio()
 {
     try {
+        //qInfo()<<"Buffer size "<<recorder->bufferSize();
         std::lock_guard<std::mutex> guard(lock);
+        qInfo()<<"Bytes ready"<<recorder->bytesReady();
+        QIODevice* device = static_cast<QIODevice*>(sender());
         if(state)
         {
-            QIODevice* device = static_cast<QIODevice*>(sender());
             QByteArray data = device->readAll();
             qInfo()<<"Received audio bytes "<<data.size();
+            if(data.size() > 50000)
+            {//Optimisation for phones or slow hardware, skip these bytes to prevent snowballing buffer size
+                return;
+            }
             DS_FeedAudioContent(state, reinterpret_cast<const short*>(data.data()), data.size()/2);
-            //  qInfo()<<"Transcribed "<<data.size()<<" bytes...";
-            qInfo()<<"Transcribing on thread "<<this->thread();
+        //  qInfo()<<"Transcribed "<<data.size()<<" bytes...";
+   //       qInfo()<<"Transcribing on thread "<<this->thread();
             samplesCollected += data.size()/2;
 
             if(samplesCollected > 16000)
@@ -140,6 +149,7 @@ void DeepSpeech::transcribeAudio()
                 samplesCollected = 0;
             }
         }
+
     }  catch (std::exception& e) {
         qInfo()<<e.what();
     }

DeepSpeechC.h

@@ -20,7 +20,6 @@ public slots:
     void init();
     void transcribeAudio();
     void setHotword(QString hotword);
-    void clearHotwords();
     void stopRecording();
     void startRecording();
 signals:
@@ -31,8 +30,11 @@ public slots:
     StreamingState* state = nullptr;
     int samplesCollected = 0;
     void doTranscription(QIODevice* device);
+    void createRecorder();
 
     std::mutex lock;
+    bool recording = false;
+    int idealSample = 0;
 };
 
 

Game.cpp

@@ -1,6 +1,8 @@
 #include <QFileInfo>
 #include <stdlib.h>     /* srand, rand */
 #include <time.h>       /* time */
+#include <QCoreApplication>
+
 
 #include "Game.h"
 #include "DeepSpeechC.h"
@@ -13,13 +15,15 @@ Game::Game(QObject *parent) : QObject(parent)
    player = new QMediaPlayer(this);
    loadMusic("audio/theme.mp3");    //To cache opening music for faster playback
 
+   speech = new QTextToSpeech(this);
+
    qInfo()<<"Game created on thread "<<this->thread();
    ds.moveToThread(&deepThread);
    deepThread.start();
    ds.init();
    //Subscribe to all transcribed text
    connect(&ds,&DeepSpeech::transcript, this, &Game::userSpeech);
-
+   connect(speech,&QTextToSpeech::stateChanged, this, &Game::speechDone);
    //Create questions
    qsns.push_back( {"cat", "qrc:/img/cat.png"});
    qsns.push_back( {"bus", "qrc:/img/bus.png"});
@@ -46,9 +50,9 @@ Game* Game::getGame(QQmlEngine *engine, QJSEngine *scriptEngine)
 void Game::listenFor(QString word)
 {
     qInfo()<<"Game listenFor "<<word<<" on "<<this->thread();
-    listeningFor = word;
-    ds.stopRecording();    //Clear stream and recoup memory
+    listeningFor = word;  
     ds.startRecording();
+    ds.setHotword("quit");
     ds.setHotword(word);
 }
 
@@ -66,17 +70,49 @@ void Game::loadMusic(QString file)
     player->setMedia(QUrl("qrc:/"+file));
 }
 
+void Game::speechDone(QTextToSpeech::State state)
+{
+    if(state == QTextToSpeech::State::Ready)
+    {
+        if(gameState == GAME)
+        {
+            showNextQsn();
+        }
+        else if(gameState == QUIT)
+        {
+            QCoreApplication::quit();
+        }
+    }
+}
+
 void Game::openingMenu()
 {
     qInfo()<<"Opening Menu";
     loadMusic("audio/theme.mp3");
     player->play();
+    listenFor("start");
+}
 
+void Game::speak(QString text)
+{
+    speech->say(text);
 }
 
-void Game::nextQsn()
+void Game::nextQsn(bool sayit)
 {
+    if(sayit)
+    {
+        ds.stopRecording();    //Clear stream and recoup memory
+        speak("Correct, this is a " + listeningFor);
+    }
+    else
+    {
+        showNextQsn();
+    }
+}
 
+void Game::showNextQsn()
+{
     int qsnNum =rand() % qsns.size();
     listenFor(qsns[qsnNum].name);
     emit showQsn(qsns[qsnNum].name, qsns[qsnNum].img);
@@ -89,20 +125,36 @@ void Game::userSpeech(QString text)
     {//Word detected
         if(gameState == INTRO)
         {
-
             gameState = GAME;
             emit gameStart();
             nextQsn();
         }
         else
+        {           
+            qInfo()<<"Correct, moving on to next question";
+            nextQsn(true);
+        }
+    }
+    else if(text.contains("quit"))
+    {
+        ds.stopRecording();
+        if(gameState == GAME)
         {
-            nextQsn();
+            speak("This was a "+listeningFor+". Thank you for playing What's That, try again next time! Bye bye");
+            gameState = QUIT;
         }
+        else
+        {
+            QCoreApplication::quit();
+        }
+
+
     }
 }
 
 void Game::destroy()
 {
     if(backend)
         delete backend;
+    //No need to delete QT classes, QT will automatically delete them
 }

Game.h

@@ -6,13 +6,15 @@
 #include <QMediaPlayer>
 #include <QThread>
 #include <string>
+#include <QTextToSpeech>
 
 #include "DeepSpeechC.h"
 
 enum GameState
 {
     INTRO,
-    GAME
+    GAME,
+    QUIT
 };
 
 struct Question
@@ -33,6 +35,8 @@ public slots:
     void openingMenu();
     void userSpeech(QString text);
     void listenFor(QString word);
+    void speak(QString text);
+    void speechDone(QTextToSpeech::State state);
 
 signals:    
     void gameStart();
@@ -42,7 +46,8 @@ public slots:
     static Game* backend;
     QMediaPlayer* player=nullptr;
     void loadMusic(QString file);
-    void nextQsn();
+    void nextQsn(bool sayit=false);
+    void showNextQsn();
 
     DeepSpeech ds;
     QThread deepThread;  //Run DeepSpeech in diff thread to prevent UI Lag
@@ -51,6 +56,8 @@ public slots:
 
     std::vector<Question> qsns;
 
+    QTextToSpeech* speech;
+
 };
 
 #endif // GAME_H

Intro.qml

@@ -33,7 +33,7 @@ Page {
         ]
 
         Component.onCompleted: {
-            Game.listenFor("start");
+            //Game.listenFor("start");
         }
 
         ColumnLayout {
@@ -113,10 +113,20 @@ Page {
             endFrame: 230
             enabled: true
             startFrame: 0
+            property bool playedIntro: false
+            property bool playedPrompt: false
             onCurrentFrameChanged: {
-                if (currentFrame == 100)
+                var frame = Math.floor(currentFrame);
+                //console.log(frame);
+                if (frame == 100 && playedIntro==false)
                 {
                     Game.openingMenu();
+                    playedIntro = true;
+                }
+                else if(frame == 199 && playedPrompt==false)
+                {
+                    Game.speak("Say start to begin!");
+                    playedPrompt = true;
                 }
             }
 

WhatsThat.pro

@@ -1,4 +1,4 @@
-QT += quick multimedia
+QT += quick multimedia texttospeech
 
 CONFIG += c++11