Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Start listening again after vad #59

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Start listening again after vad #59

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3314173
Simultaneous listening and speaking
Shulyaka Dec 20, 2023
37dd866
stop old task after wake
Shulyaka Dec 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 52 additions & 14 deletions bin/pipeline_run.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,25 +104,63 @@ async def main() -> None:

rhasspy = Rhasspy.load(args.config)

loop = asyncio.get_running_loop()

old_task = None

while True:
pipeline_result = await run_pipeline(
rhasspy,
args.pipeline,
samples_per_chunk=args.samples_per_chunk,
asr_chunks_to_buffer=args.asr_chunks_to_buffer,
wake_detection=wake_detection,
asr_wav_in=asr_wav_in,
asr_transcript=asr_transcript,
intent_result=intent_result,
handle_result=handle_result,
tts_wav_in=tts_wav_in,
stop_after=args.stop_after,
wake_future = loop.create_future()
vad_future = loop.create_future()
new_task = asyncio.create_task(
run_pipeline(
rhasspy,
args.pipeline,
samples_per_chunk=args.samples_per_chunk,
asr_chunks_to_buffer=args.asr_chunks_to_buffer,
wake_detection=wake_detection,
wake_future=wake_future,
vad_future=vad_future,
asr_wav_in=asr_wav_in,
asr_transcript=asr_transcript,
intent_result=intent_result,
handle_result=handle_result,
tts_wav_in=tts_wav_in,
stop_after=args.stop_after,
)
)

json.dump(pipeline_result.to_dict(), sys.stdout, ensure_ascii=False)
print("")
pending = {new_task, wake_future, vad_future}
if old_task is not None and not old_task.done():
pending.add(old_task)

while len(pending):
done, pending = await asyncio.wait(
pending, return_when=asyncio.FIRST_COMPLETED
)

if old_task in done:
if not old_task.cancelled():
pipeline_result = old_task.result()
json.dump(pipeline_result.to_dict(), sys.stdout, ensure_ascii=False)
print("")
old_task = None

if wake_future in done and old_task in pending:
old_task.cancel()

if vad_future in done and wake_future.done():
old_task = new_task
break

if new_task in done:
break

if not args.loop:
if not new_task.done():
await asyncio.wait({new_task})
pipeline_result = new_task.result()
json.dump(pipeline_result.to_dict(), sys.stdout, ensure_ascii=False)
print("")
break


Expand Down
24 changes: 20 additions & 4 deletions rhasspy3/pipeline.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
"""Full voice loop (pipeline)."""
import io
import logging
from asyncio import Future
from collections import deque
from dataclasses import dataclass, fields
from enum import Enum
Expand Down Expand Up @@ -62,10 +63,12 @@ async def run(
mic_program: Optional[Union[str, PipelineProgramConfig]] = None,
wake_program: Optional[Union[str, PipelineProgramConfig]] = None,
wake_detection: Optional[Detection] = None,
wake_future: Optional[Future] = None,
asr_program: Optional[Union[str, PipelineProgramConfig]] = None,
asr_wav_in: Optional[IO[bytes]] = None,
asr_transcript: Optional[Transcript] = None,
vad_program: Optional[Union[str, PipelineProgramConfig]] = None,
vad_future: Optional[Future] = None,
intent_result: Optional[Union[Intent, NotRecognized]] = None,
intent_program: Optional[Union[str, PipelineProgramConfig]] = None,
handle_result: Optional[Union[Handled, NotHandled]] = None,
Expand Down Expand Up @@ -125,21 +128,22 @@ async def run(
assert asr_program is not None, "No asr program"
assert vad_program is not None, "No vad program"
await _mic_asr(
rhasspy, mic_program, asr_program, vad_program, pipeline_result
rhasspy, mic_program, asr_program, vad_program, pipeline_result, vad_future
)
elif stop_after == StopAfterDomain.WAKE:
# Audio input, wake word detection, segmentation, speech to text
assert wake_program is not None, "No vad program"
assert wake_program is not None, "No wake program"
await _mic_wake(
rhasspy,
mic_program,
wake_program,
pipeline_result,
wake_detection=wake_detection,
wake_future=wake_future,
)
return pipeline_result
else:
assert wake_program is not None, "No vad program"
assert wake_program is not None, "No wake program"
assert asr_program is not None, "No asr program"
assert vad_program is not None, "No vad program"
await _mic_wake_asr(
Expand All @@ -152,13 +156,17 @@ async def run(
asr_chunks_to_buffer=asr_chunks_to_buffer,
wake_detection=wake_detection,
wake_after=wake_after,
wake_future=wake_future,
vad_future=vad_future,
)

if asr_after is not None:
await run_command(rhasspy, asr_after)

asr_transcript = pipeline_result.asr_transcript
pipeline_result.asr_transcript = asr_transcript
wake_detection = pipeline_result.wake_detection
pipeline_result.asr_transcript = asr_transcript
pipeline_result.wake_detection = wake_detection

if (stop_after == StopAfterDomain.ASR) or (
(intent_program is None) and (handle_program is None)
Expand Down Expand Up @@ -220,6 +228,7 @@ async def _mic_wake(
wake_program: Union[str, PipelineProgramConfig],
pipeline_result: PipelineResult,
wake_detection: Optional[Detection] = None,
wake_future: Optional[Future] = None,
):
"""Just wake word detection."""
async with (await create_process(rhasspy, MIC_DOMAIN, mic_program)) as mic_proc:
Expand All @@ -232,6 +241,7 @@ async def _mic_wake(
)

if wake_detection is not None:
wake_future.set_result(wake_detection)
pipeline_result.wake_detection = wake_detection
else:
_LOGGER.debug("run: no wake word detected")
Expand All @@ -244,6 +254,7 @@ async def _mic_asr(
vad_program: Union[str, PipelineProgramConfig],
pipeline_result: PipelineResult,
asr_chunks_to_buffer: int = 0,
vad_future: Optional[Future] = None,
):
"""Just asr transcription (+ silence detection)."""
async with (await create_process(rhasspy, MIC_DOMAIN, mic_program)) as mic_proc, (
Expand All @@ -259,6 +270,7 @@ async def _mic_asr(
mic_proc.stdout,
asr_proc.stdin,
)
vad_future.set_result(True)
while True:
asr_event = await async_read_event(asr_proc.stdout)
if asr_event is None:
Expand All @@ -279,6 +291,8 @@ async def _mic_wake_asr(
asr_chunks_to_buffer: int = 0,
wake_detection: Optional[Detection] = None,
wake_after: Optional[CommandConfig] = None,
wake_future: Optional[Future] = None,
vad_future: Optional[Future] = None,
):
"""Wake word detect + asr transcription (+ silence detection)."""
chunk_buffer: Optional[Deque[Event]] = (
Expand All @@ -298,6 +312,7 @@ async def _mic_wake_asr(
)

if wake_detection is not None:
wake_future.set_result(wake_detection)
if wake_after is not None:
await run_command(rhasspy, wake_after)

Expand All @@ -309,6 +324,7 @@ async def _mic_wake_asr(
asr_proc.stdin,
chunk_buffer,
)
vad_future.set_result(True)
while True:
asr_event = await async_read_event(asr_proc.stdout)
if asr_event is None:
Expand Down