diff --git a/mods/autopilot/src/machine/machine.ts b/mods/autopilot/src/machine/machine.ts index 0cda9bd..8e3a33d 100644 --- a/mods/autopilot/src/machine/machine.ts +++ b/mods/autopilot/src/machine/machine.ts @@ -34,10 +34,18 @@ const machine = machineSetup.createMachine({ src: "doGreetUser", description: "Answer the call and play the first message", input: ({ context }) => ({ context }), - onDone: { - target: "idle", - description: "Transition to idle after the initial greeting." - }, + onDone: [ + { + target: "waitingForSpeechTimeout", + guard: "hasSpeechResult", + description: + "Process the speech captured while the greeting was playing." + }, + { + target: "idle", + description: "Transition to idle after the initial greeting." + } + ], onError: { target: "idle" } @@ -45,7 +53,13 @@ const machine = machineSetup.createMachine({ on: { SPEECH_START: { target: "listeningToUser", - description: "The user barged in during the greeting." + description: "The user barged in during the greeting.", + guard: ({ context }) => context.allowUserBargeIn + }, + SPEECH_RESULT: { + description: + "Keep the speech the user produced while the greeting was playing, without cutting it short.", + actions: [{ type: "appendSpeech" }] } } }, @@ -56,6 +70,13 @@ const machine = machineSetup.createMachine({ SPEECH_START: { target: "listeningToUser", description: "Event from VAD system." + }, + SPEECH_RESULT: { + target: "waitingForSpeechTimeout", + description: + "Speech that ends right as playback does, so no SPEECH_START is seen here.", + actions: [{ type: "appendSpeech" }], + reenter: true } }, after: { @@ -83,10 +104,18 @@ const machine = machineSetup.createMachine({ src: "doAnnounceIdleTimeout", description: "Play the idle message", input: ({ context }) => ({ context }), - onDone: { - target: "idle", - description: "Re-arm the idle clock once the idle message finishes." - }, + onDone: [ + { + target: "waitingForSpeechTimeout", + guard: "hasSpeechResult", + description: + "Process the speech captured while the idle message was playing." + }, + { + target: "idle", + description: "Re-arm the idle clock once the idle message finishes." + } + ], onError: { target: "idle" } @@ -94,7 +123,13 @@ const machine = machineSetup.createMachine({ on: { SPEECH_START: { target: "listeningToUser", - description: "The user spoke while the idle message was playing." + description: "The user spoke while the idle message was playing.", + guard: ({ context }) => context.allowUserBargeIn + }, + SPEECH_RESULT: { + description: + "Keep the speech the user produced while the idle message was playing, without cutting it short.", + actions: [{ type: "appendSpeech" }] } } }, diff --git a/mods/autopilot/test/bargeIn.test.ts b/mods/autopilot/test/bargeIn.test.ts new file mode 100644 index 0000000..3a2a1fe --- /dev/null +++ b/mods/autopilot/test/bargeIn.test.ts @@ -0,0 +1,281 @@ +/* eslint-disable no-invalid-this */ +/* + * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com) + * http://github.com/fonoster/fonoster + * + * This file is part of Fonoster + * + * Licensed under the MIT License (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://opensource.org/licenses/MIT + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import * as chai from "chai"; +import { expect } from "chai"; +import chaiAsPromised from "chai-as-promised"; +import { createSandbox } from "sinon"; +import sinonChai from "sinon-chai"; +import { createActor, SimulatedClock } from "xstate"; +import { LanguageModel } from "../src/models"; +import { Voice } from "../src/voice"; + +chai.use(chaiAsPromised); +chai.use(sinonChai); +const sandbox = createSandbox(); + +const FIRST_MESSAGE = "Welcome message"; +const GOODBYE_MESSAGE = "Goodbye message"; +const IDLE_MESSAGE = "Idle message"; +const ASSISTANT_RESPONSE = "Assistant response"; +const IDLE_TIMEOUT = 4500; +const MAX_SPEECH_WAIT_TIMEOUT = 500; + +const flush = () => new Promise((resolve) => setImmediate(resolve)); + +// Same deferred playback harness as the idle timeout suite: voice.say only +// resolves when the test says so, which keeps the assistant "mid-sentence" +// for as long as the test needs. +const createDeferredSay = () => { + const pending: Array<() => void> = []; + const say = sandbox + .stub() + .callsFake(() => new Promise((resolve) => pending.push(resolve))); + const finishPlayback = async () => { + const resolve = pending.shift(); + if (!resolve) throw new Error("no playback in progress"); + resolve(); + await flush(); + }; + return { say, finishPlayback }; +}; + +const getActorInput = (allowUserBargeIn: boolean) => { + const deferred = createDeferredSay(); + + return { + deferred, + languageModel: { + invoke: sandbox.stub().resolves({ + type: "say", + content: ASSISTANT_RESPONSE + }) + } as unknown as LanguageModel, + voice: { + mediaSessionRef: "mediaSessionRef", + answer: sandbox.stub().resolves(), + sgather: sandbox.stub().resolves({ + stop: sandbox.stub().resolves(), + onData: sandbox.stub() + }), + stream: sandbox.stub().resolves({ + stop: sandbox.stub().resolves(), + onData: sandbox.stub() + }), + playDtmf: sandbox.stub().resolves(), + stopSpeech: sandbox.stub().resolves(), + say: deferred.say, + hangup: sandbox.stub().resolves() + } as unknown as Voice, + conversationSettings: { + firstMessage: FIRST_MESSAGE, + goodbyeMessage: GOODBYE_MESSAGE, + systemPrompt: "System template", + systemErrorMessage: "System error message", + maxSpeechWaitTimeout: MAX_SPEECH_WAIT_TIMEOUT, + maxSessionDuration: 30 * 60 * 1000, + allowUserBargeIn, + idleOptions: { + message: IDLE_MESSAGE, + timeout: IDLE_TIMEOUT, + maxTimeoutCount: 3 + }, + vad: { + activationThreshold: 0.3, + deactivationThreshold: 0.2, + debounceFrames: 3 + } + } + }; +}; + +const startActor = async (input: ReturnType) => { + const { machine } = await import("../src/machine"); + const clock = new SimulatedClock(); + const actor = createActor(machine, { + clock, + input: { + voice: input.voice, + languageModel: input.languageModel, + conversationSettings: input.conversationSettings + } + }); + actor.start(); + await flush(); + const advance = async (ms: number) => { + clock.increment(ms); + await flush(); + }; + return { actor, advance }; +}; + +// Move an actor to "announcingIdleTimeout" with the idle message playing. +const reachIdleAnnouncement = async ( + input: ReturnType, + advance: (ms: number) => Promise +) => { + await input.deferred.finishPlayback(); + await advance(IDLE_TIMEOUT); +}; + +describe("@autopilot/machine barge-in", function () { + afterEach(() => sandbox.restore()); + + it("does not cut the greeting short when barge-in is disabled", async function () { + // Arrange + const input = getActorInput(false); + const { actor } = await startActor(input); + expect(actor.getSnapshot().value).to.equal("greeting"); + + // Act + actor.send({ type: "SPEECH_START" }); + await flush(); + + // Assert + expect(actor.getSnapshot().value).to.equal("greeting"); + expect(input.voice.stopSpeech).to.not.have.been.called; + + await input.deferred.finishPlayback(); + expect(actor.getSnapshot().value).to.equal("idle"); + + actor.stop(); + }); + + it("does not cut the idle message short when barge-in is disabled", async function () { + // Arrange + const input = getActorInput(false); + const { actor, advance } = await startActor(input); + await reachIdleAnnouncement(input, advance); + expect(actor.getSnapshot().value).to.equal("announcingIdleTimeout"); + + // Act + actor.send({ type: "SPEECH_START" }); + await flush(); + + // Assert + expect(actor.getSnapshot().value).to.equal("announcingIdleTimeout"); + expect(input.voice.stopSpeech).to.not.have.been.called; + + await input.deferred.finishPlayback(); + expect(actor.getSnapshot().value).to.equal("idle"); + + actor.stop(); + }); + + it("still processes speech produced during the greeting once it finishes", async function () { + // Arrange + const input = getActorInput(false); + const { actor, advance } = await startActor(input); + + // Act: the caller answers over the greeting + actor.send({ type: "SPEECH_START" }); + actor.send({ type: "SPEECH_RESULT", speech: "Hello", responseTime: 100 }); + await flush(); + expect(actor.getSnapshot().value).to.equal("greeting"); + + await input.deferred.finishPlayback(); + + // Assert + expect(actor.getSnapshot().value).to.equal("waitingForSpeechTimeout"); + expect(actor.getSnapshot().context.speechBuffer).to.equal("Hello"); + + await advance(MAX_SPEECH_WAIT_TIMEOUT); + expect(actor.getSnapshot().value).to.equal("processingUserRequest"); + expect(input.languageModel.invoke).to.have.been.calledOnce; + expect(input.deferred.say).to.have.been.calledWith(ASSISTANT_RESPONSE); + + actor.stop(); + }); + + it("still processes speech produced during the idle message once it finishes", async function () { + // Arrange + const input = getActorInput(false); + const { actor, advance } = await startActor(input); + await reachIdleAnnouncement(input, advance); + expect(actor.getSnapshot().value).to.equal("announcingIdleTimeout"); + + // Act + actor.send({ type: "SPEECH_START" }); + actor.send({ + type: "SPEECH_RESULT", + speech: "Yes, I am here", + responseTime: 100 + }); + await flush(); + expect(actor.getSnapshot().value).to.equal("announcingIdleTimeout"); + + await input.deferred.finishPlayback(); + + // Assert + expect(actor.getSnapshot().value).to.equal("waitingForSpeechTimeout"); + expect(actor.getSnapshot().context.speechBuffer).to.equal("Yes, I am here"); + + await advance(MAX_SPEECH_WAIT_TIMEOUT); + expect(actor.getSnapshot().value).to.equal("processingUserRequest"); + expect(input.languageModel.invoke).to.have.been.calledOnce; + + actor.stop(); + }); + + it("processes a speech result that lands just after playback ended", async function () { + // Arrange: the VAD start event was consumed while the greeting played, so + // the machine is already in "idle" when the transcript arrives. + const input = getActorInput(false); + const { actor, advance } = await startActor(input); + await input.deferred.finishPlayback(); + expect(actor.getSnapshot().value).to.equal("idle"); + + // Act + actor.send({ type: "SPEECH_RESULT", speech: "Hello", responseTime: 100 }); + await flush(); + + // Assert + expect(actor.getSnapshot().value).to.equal("waitingForSpeechTimeout"); + expect(actor.getSnapshot().context.speechBuffer).to.equal("Hello"); + + await advance(MAX_SPEECH_WAIT_TIMEOUT); + expect(actor.getSnapshot().value).to.equal("processingUserRequest"); + expect(input.languageModel.invoke).to.have.been.calledOnce; + + actor.stop(); + }); + + it("keeps cutting the greeting and the idle message short when barge-in is enabled", async function () { + // Arrange + const input = getActorInput(true); + const { actor, advance } = await startActor(input); + + // Act and assert: the greeting is interrupted + actor.send({ type: "SPEECH_START" }); + await flush(); + expect(actor.getSnapshot().value).to.equal("listeningToUser"); + expect(input.voice.stopSpeech).to.have.been.calledOnce; + + // Act and assert: so is the idle message + await advance(IDLE_TIMEOUT); + expect(actor.getSnapshot().value).to.equal("announcingIdleTimeout"); + actor.send({ type: "SPEECH_START" }); + await flush(); + expect(actor.getSnapshot().value).to.equal("listeningToUser"); + expect(input.voice.stopSpeech).to.have.been.calledTwice; + expect(input.deferred.say).to.have.been.calledWith(IDLE_MESSAGE); + + actor.stop(); + }); +}); diff --git a/mods/autopilot/test/machine.test.ts b/mods/autopilot/test/machine.test.ts index 9b54c6e..f13b42c 100644 --- a/mods/autopilot/test/machine.test.ts +++ b/mods/autopilot/test/machine.test.ts @@ -307,8 +307,10 @@ describe("@autopilot/machine", function () { const actor = createActor(machine, { input }); - // Act: get to processingUserRequest + // Act: get to processingUserRequest. With barge-in off the greeting cannot + // be cut short, so wait for it to finish before speaking. actor.start(); + await waitFor(4600); actor.send({ type: "SPEECH_START" }); await waitFor(50); actor.send({ type: "SPEECH_RESULT", speech: "Hello", responseTime: 1000 }); @@ -341,8 +343,10 @@ describe("@autopilot/machine", function () { const actor = createActor(machine, { input }); - // Act: get to processingUserRequest + // Act: get to processingUserRequest. With barge-in off the greeting cannot + // be cut short, so wait for it to finish before speaking. actor.start(); + await waitFor(4600); actor.send({ type: "SPEECH_START" }); await waitFor(50); actor.send({ type: "SPEECH_RESULT", speech: "Hello?", responseTime: 1000 });