Skip to content

Commit

Permalink
Update voice debug for new API (home-assistant#15913)
Browse files Browse the repository at this point in the history 
* Update voice debug for new API

* Update imports

* Remove wrong key

* Some HTML formatting
  • Loading branch information
@balloob
balloob authored Mar 23, 2023
1 parent 74cfcca commit d9dbb69
Show file tree
Hide file tree
Showing 3 changed files with 169 additions and 47 deletions.
17 changes: 17 additions & 0 deletions src/data/stt.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
export interface SpeechMetadata {
language: string;
format: "wav" | "ogg";
codec: "pcm" | "opus";
bit_rate: 8 | 16 | 24 | 32;
sample_rate:
| 8000
| 11000
| 16000
| 18900
| 22000
| 32000
| 37800
| 44100
| 48000;
channel: 1 | 2;
}
69 changes: 42 additions & 27 deletions src/data/voice_assistant.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import { HomeAssistant } from "../types";
import { ConversationResult } from "./conversation";
import type { HomeAssistant } from "../types";
import type { ConversationResult } from "./conversation";
import type { ResolvedMediaSource } from "./media_source";
import type { SpeechMetadata } from "./stt";

interface PipelineEventBase {
timestamp: string;
Expand All @@ -12,8 +14,8 @@ interface PipelineRunStartEvent extends PipelineEventBase {
language: string;
};
}
interface PipelineRunFinishEvent extends PipelineEventBase {
type: "run-finish";
interface PipelineRunEndEvent extends PipelineEventBase {
type: "run-end";
data: Record<string, never>;
}

Expand All @@ -27,11 +29,16 @@ interface PipelineErrorEvent extends PipelineEventBase {

interface PipelineSTTStartEvent extends PipelineEventBase {
type: "stt-start";
data: Record<string, never>;
data: {
engine: string;
metadata: SpeechMetadata;
};
}
interface PipelineSTTFinishEvent extends PipelineEventBase {
type: "stt-finish";
data: Record<string, never>;
interface PipelineSTTEndEvent extends PipelineEventBase {
type: "stt-end";
data: {
text: string;
};
}

interface PipelineIntentStartEvent extends PipelineEventBase {
Expand All @@ -41,36 +48,44 @@ interface PipelineIntentStartEvent extends PipelineEventBase {
intent_input: string;
};
}
interface PipelineIntentFinishEvent extends PipelineEventBase {
type: "intent-finish";
interface PipelineIntentEndEvent extends PipelineEventBase {
type: "intent-end";
data: {
intent_output: ConversationResult;
};
}

interface PipelineTTSStartEvent extends PipelineEventBase {
type: "tts-start";
data: Record<string, never>;
data: {
engine: string;
tts_input: string;
};
}
interface PipelineTTSFinishEvent extends PipelineEventBase {
type: "tts-finish";
data: Record<string, never>;
interface PipelineTTSEndEvent extends PipelineEventBase {
type: "tts-end";
data: {
tts_output: ResolvedMediaSource;
};
}

type PipelineRunEvent =
| PipelineRunStartEvent
| PipelineRunFinishEvent
| PipelineRunEndEvent
| PipelineErrorEvent
| PipelineSTTStartEvent
| PipelineSTTFinishEvent
| PipelineSTTEndEvent
| PipelineIntentStartEvent
| PipelineIntentFinishEvent
| PipelineIntentEndEvent
| PipelineTTSStartEvent
| PipelineTTSFinishEvent;
| PipelineTTSEndEvent;

interface PipelineRunOptions {
start_stage: "stt" | "intent" | "tts";
end_stage: "stt" | "intent" | "tts";
language?: string;
pipeline?: string;
intent_input?: string;
input?: { text: string };
conversation_id?: string | null;
}

Expand All @@ -80,16 +95,16 @@ export interface PipelineRun {
stage: "ready" | "stt" | "intent" | "tts" | "done" | "error";
run: PipelineRunStartEvent["data"];
error?: PipelineErrorEvent["data"];
stt?: PipelineSTTStartEvent["data"] & Partial<PipelineSTTFinishEvent["data"]>;
stt?: PipelineSTTStartEvent["data"] & Partial<PipelineSTTEndEvent["data"]>;
intent?: PipelineIntentStartEvent["data"] &
Partial<PipelineIntentFinishEvent["data"]>;
tts?: PipelineTTSStartEvent["data"] & Partial<PipelineTTSFinishEvent["data"]>;
Partial<PipelineIntentEndEvent["data"]>;
tts?: PipelineTTSStartEvent["data"] & Partial<PipelineTTSEndEvent["data"]>;
}

export const runPipelineFromText = (
hass: HomeAssistant,
callback: (event: PipelineRun) => void,
options: PipelineRunOptions = {}
options: PipelineRunOptions
) => {
let run: PipelineRun | undefined;

Expand Down Expand Up @@ -121,17 +136,17 @@ export const runPipelineFromText = (

if (updateEvent.type === "stt-start") {
run = { ...run, stage: "stt", stt: updateEvent.data };
} else if (updateEvent.type === "stt-finish") {
} else if (updateEvent.type === "stt-end") {
run = { ...run, stt: { ...run.stt!, ...updateEvent.data } };
} else if (updateEvent.type === "intent-start") {
run = { ...run, stage: "intent", intent: updateEvent.data };
} else if (updateEvent.type === "intent-finish") {
} else if (updateEvent.type === "intent-end") {
run = { ...run, intent: { ...run.intent!, ...updateEvent.data } };
} else if (updateEvent.type === "tts-start") {
run = { ...run, stage: "tts", tts: updateEvent.data };
} else if (updateEvent.type === "tts-finish") {
} else if (updateEvent.type === "tts-end") {
run = { ...run, tts: { ...run.tts!, ...updateEvent.data } };
} else if (updateEvent.type === "run-finish") {
} else if (updateEvent.type === "run-end") {
run = { ...run, stage: "done" };
unsubProm.then((unsub) => unsub());
} else if (updateEvent.type === "error") {
Expand Down
130 changes: 110 additions & 20 deletions ...ls/config/integrations/integration-panels/voice_assistant/assist/assist-pipeline-debug.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import {
import "../../../../../../layouts/hass-subpage";
import { SubscribeMixin } from "../../../../../../mixins/subscribe-mixin";
import { haStyle } from "../../../../../../resources/styles";
import { HomeAssistant } from "../../../../../../types";
import type { HomeAssistant } from "../../../../../../types";
import { formatNumber } from "../../../../../../common/number/format_number";

const RUN_DATA = {
Expand All @@ -25,11 +25,33 @@ const ERROR_DATA = {
message: "Message",
};

const STT_DATA = {
engine: "Engine",
};

const INTENT_DATA = {
engine: "Engine",
intent_input: "Input",
};

const TTS_DATA = {
engine: "Engine",
tts_input: "Input",
};

const STAGES: Record<PipelineRun["stage"], number> = {
ready: 0,
stt: 1,
intent: 2,
tts: 3,
done: 4,
error: 5,
};

const hasStage = (run: PipelineRun, stage: PipelineRun["stage"]) =>
STAGES[run.init_options.start_stage] <= STAGES[stage] &&
STAGES[stage] <= STAGES[run.init_options.end_stage];

const renderProgress = (
hass: HomeAssistant,
pipelineRun: PipelineRun,
Expand All @@ -39,7 +61,7 @@ const renderProgress = (
(ev) => ev.type === `${stage}-start`
);
const finishEvent = pipelineRun.events.find(
(ev) => ev.type === `${stage}-finish`
(ev) => ev.type === `${stage}-end`
);

if (!startEvent) {
Expand Down Expand Up @@ -142,25 +164,91 @@ export class AssistPipelineDebug extends SubscribeMixin(LitElement) {
: ""}
</div>
</ha-card>
<ha-card>
<div class="card-content">
<div class="row heading">
<span>Natural Language Processing</span>
${renderProgress(this.hass, this._pipelineRun, "intent")}
</div>
${this._pipelineRun.intent
? html`
<div class="card-content">
${renderData(this._pipelineRun.intent, INTENT_DATA)}
${dataMinusKeysRender(
this._pipelineRun.intent,
INTENT_DATA
${hasStage(this._pipelineRun, "stt")
? html`
<ha-card>
<div class="card-content">
<div class="row heading">
<span>Speech-to-Text</span>
${renderProgress(
this.hass,
this._pipelineRun,
"stt"
)}
</div>
`
: ""}
</div>
</ha-card>
${this._pipelineRun.stt
? html`
<div class="card-content">
${renderData(this._pipelineRun.stt, STT_DATA)}
${dataMinusKeysRender(
this._pipelineRun.stt,
STT_DATA
)}
</div>
`
: ""}
</div>
</ha-card>
`
: ""}
${hasStage(this._pipelineRun, "intent")
? html`
<ha-card>
<div class="card-content">
<div class="row heading">
<span>Natural Language Processing</span>
${renderProgress(
this.hass,
this._pipelineRun,
"intent"
)}
</div>
${this._pipelineRun.intent
? html`
<div class="card-content">
${renderData(
this._pipelineRun.intent,
INTENT_DATA
)}
${dataMinusKeysRender(
this._pipelineRun.intent,
INTENT_DATA
)}
</div>
`
: ""}
</div>
</ha-card>
`
: ""}
${hasStage(this._pipelineRun, "tts")
? html`
<ha-card>
<div class="card-content">
<div class="row heading">
<span>Text-to-Speech</span>
${renderProgress(
this.hass,
this._pipelineRun,
"tts"
)}
</div>
${this._pipelineRun.tts
? html`
<div class="card-content">
${renderData(this._pipelineRun.tts, TTS_DATA)}
${dataMinusKeysRender(
this._pipelineRun.tts,
TTS_DATA
)}
</div>
`
: ""}
</div>
</ha-card>
`
: ""}
<ha-card>
<ha-expansion-panel>
<span slot="header">Raw</span>
Expand All @@ -182,7 +270,9 @@ export class AssistPipelineDebug extends SubscribeMixin(LitElement) {
this._pipelineRun = run;
},
{
intent_input: this._newRunInput.value,
start_stage: "intent",
end_stage: "intent",
input: { text: this._newRunInput.value },
}
);
}
Expand Down

0 comments on commit d9dbb69

Please sign in to comment.