fix(autopilot): honor allowUserBargeIn during the greeting and the idle message

The greeting and announcingIdleTimeout states accepted SPEECH_START
unconditionally, so the caller cut the first message and the idle prompt
short even with allowUserBargeIn set to false, unlike processingUserRequest
and waitingForSpeechTimeout which already guard that event.

Guarding the event alone would drop the utterance: idle only listens for
SPEECH_START, so speech produced while the assistant was talking would
never be processed. Both playing states now buffer SPEECH_RESULT without
stopping playback and hand it to waitingForSpeechTimeout when the message
finishes, and idle accepts a SPEECH_RESULT that lands right after playback
ended.

Closes #891

mods/autopilot/src/machine/machine.ts

@@ -34,10 +34,18 @@ const machine = machineSetup.createMachine({
         src: "doGreetUser",
         description: "Answer the call and play the first message",
         input: ({ context }) => ({ context }),
-        onDone: {
+        onDone: [
+          {
+            target: "waitingForSpeechTimeout",
+            guard: "hasSpeechResult",
+            description:
+              "Process the speech captured while the greeting was playing."
+          },
+          {
             target: "idle",
             description: "Transition to idle after the initial greeting."
-        },
+          }
+        ],
         onError: {
           target: "idle"
         }
@@ -45,7 +53,13 @@ const machine = machineSetup.createMachine({
       on: {
         SPEECH_START: {
           target: "listeningToUser",
-          description: "The user barged in during the greeting."
+          description: "The user barged in during the greeting.",
+          guard: ({ context }) => context.allowUserBargeIn
+        },
+        SPEECH_RESULT: {
+          description:
+            "Keep the speech the user produced while the greeting was playing, without cutting it short.",
+          actions: [{ type: "appendSpeech" }]
         }
       }
     },
@@ -56,6 +70,13 @@ const machine = machineSetup.createMachine({
         SPEECH_START: {
           target: "listeningToUser",
           description: "Event from VAD system."
+        },
+        SPEECH_RESULT: {
+          target: "waitingForSpeechTimeout",
+          description:
+            "Speech that ends right as playback does, so no SPEECH_START is seen here.",
+          actions: [{ type: "appendSpeech" }],
+          reenter: true
         }
       },
       after: {
@@ -83,10 +104,18 @@ const machine = machineSetup.createMachine({
         src: "doAnnounceIdleTimeout",
         description: "Play the idle message",
         input: ({ context }) => ({ context }),
-        onDone: {
+        onDone: [
+          {
+            target: "waitingForSpeechTimeout",
+            guard: "hasSpeechResult",
+            description:
+              "Process the speech captured while the idle message was playing."
+          },
+          {
             target: "idle",
             description: "Re-arm the idle clock once the idle message finishes."
-        },
+          }
+        ],
         onError: {
           target: "idle"
         }
@@ -94,7 +123,13 @@ const machine = machineSetup.createMachine({
       on: {
         SPEECH_START: {
           target: "listeningToUser",
-          description: "The user spoke while the idle message was playing."
+          description: "The user spoke while the idle message was playing.",
+          guard: ({ context }) => context.allowUserBargeIn
+        },
+        SPEECH_RESULT: {
+          description:
+            "Keep the speech the user produced while the idle message was playing, without cutting it short.",
+          actions: [{ type: "appendSpeech" }]
         }
       }
     },

mods/autopilot/test/bargeIn.test.ts

@@ -0,0 +1,281 @@
+/* eslint-disable no-invalid-this */
+/*
+ * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
+ * http://github.com/fonoster/fonoster
+ *
+ * This file is part of Fonoster
+ *
+ * Licensed under the MIT License (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    https://opensource.org/licenses/MIT
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import * as chai from "chai";
+import { expect } from "chai";
+import chaiAsPromised from "chai-as-promised";
+import { createSandbox } from "sinon";
+import sinonChai from "sinon-chai";
+import { createActor, SimulatedClock } from "xstate";
+import { LanguageModel } from "../src/models";
+import { Voice } from "../src/voice";
+
+chai.use(chaiAsPromised);
+chai.use(sinonChai);
+const sandbox = createSandbox();
+
+const FIRST_MESSAGE = "Welcome message";
+const GOODBYE_MESSAGE = "Goodbye message";
+const IDLE_MESSAGE = "Idle message";
+const ASSISTANT_RESPONSE = "Assistant response";
+const IDLE_TIMEOUT = 4500;
+const MAX_SPEECH_WAIT_TIMEOUT = 500;
+
+const flush = () => new Promise((resolve) => setImmediate(resolve));
+
+// Same deferred playback harness as the idle timeout suite: voice.say only
+// resolves when the test says so, which keeps the assistant "mid-sentence"
+// for as long as the test needs.
+const createDeferredSay = () => {
+  const pending: Array<() => void> = [];
+  const say = sandbox
+    .stub()
+    .callsFake(() => new Promise<void>((resolve) => pending.push(resolve)));
+  const finishPlayback = async () => {
+    const resolve = pending.shift();
+    if (!resolve) throw new Error("no playback in progress");
+    resolve();
+    await flush();
+  };
+  return { say, finishPlayback };
+};
+
+const getActorInput = (allowUserBargeIn: boolean) => {
+  const deferred = createDeferredSay();
+
+  return {
+    deferred,
+    languageModel: {
+      invoke: sandbox.stub().resolves({
+        type: "say",
+        content: ASSISTANT_RESPONSE
+      })
+    } as unknown as LanguageModel,
+    voice: {
+      mediaSessionRef: "mediaSessionRef",
+      answer: sandbox.stub().resolves(),
+      sgather: sandbox.stub().resolves({
+        stop: sandbox.stub().resolves(),
+        onData: sandbox.stub()
+      }),
+      stream: sandbox.stub().resolves({
+        stop: sandbox.stub().resolves(),
+        onData: sandbox.stub()
+      }),
+      playDtmf: sandbox.stub().resolves(),
+      stopSpeech: sandbox.stub().resolves(),
+      say: deferred.say,
+      hangup: sandbox.stub().resolves()
+    } as unknown as Voice,
+    conversationSettings: {
+      firstMessage: FIRST_MESSAGE,
+      goodbyeMessage: GOODBYE_MESSAGE,
+      systemPrompt: "System template",
+      systemErrorMessage: "System error message",
+      maxSpeechWaitTimeout: MAX_SPEECH_WAIT_TIMEOUT,
+      maxSessionDuration: 30 * 60 * 1000,
+      allowUserBargeIn,
+      idleOptions: {
+        message: IDLE_MESSAGE,
+        timeout: IDLE_TIMEOUT,
+        maxTimeoutCount: 3
+      },
+      vad: {
+        activationThreshold: 0.3,
+        deactivationThreshold: 0.2,
+        debounceFrames: 3
+      }
+    }
+  };
+};
+
+const startActor = async (input: ReturnType<typeof getActorInput>) => {
+  const { machine } = await import("../src/machine");
+  const clock = new SimulatedClock();
+  const actor = createActor(machine, {
+    clock,
+    input: {
+      voice: input.voice,
+      languageModel: input.languageModel,
+      conversationSettings: input.conversationSettings
+    }
+  });
+  actor.start();
+  await flush();
+  const advance = async (ms: number) => {
+    clock.increment(ms);
+    await flush();
+  };
+  return { actor, advance };
+};
+
+// Move an actor to "announcingIdleTimeout" with the idle message playing.
+const reachIdleAnnouncement = async (
+  input: ReturnType<typeof getActorInput>,
+  advance: (ms: number) => Promise<void>
+) => {
+  await input.deferred.finishPlayback();
+  await advance(IDLE_TIMEOUT);
+};
+
+describe("@autopilot/machine barge-in", function () {
+  afterEach(() => sandbox.restore());
+
+  it("does not cut the greeting short when barge-in is disabled", async function () {
+    // Arrange
+    const input = getActorInput(false);
+    const { actor } = await startActor(input);
+    expect(actor.getSnapshot().value).to.equal("greeting");
+
+    // Act
+    actor.send({ type: "SPEECH_START" });
+    await flush();
+
+    // Assert
+    expect(actor.getSnapshot().value).to.equal("greeting");
+    expect(input.voice.stopSpeech).to.not.have.been.called;
+
+    await input.deferred.finishPlayback();
+    expect(actor.getSnapshot().value).to.equal("idle");
+
+    actor.stop();
+  });
+
+  it("does not cut the idle message short when barge-in is disabled", async function () {
+    // Arrange
+    const input = getActorInput(false);
+    const { actor, advance } = await startActor(input);
+    await reachIdleAnnouncement(input, advance);
+    expect(actor.getSnapshot().value).to.equal("announcingIdleTimeout");
+
+    // Act
+    actor.send({ type: "SPEECH_START" });
+    await flush();
+
+    // Assert
+    expect(actor.getSnapshot().value).to.equal("announcingIdleTimeout");
+    expect(input.voice.stopSpeech).to.not.have.been.called;
+
+    await input.deferred.finishPlayback();
+    expect(actor.getSnapshot().value).to.equal("idle");
+
+    actor.stop();
+  });
+
+  it("still processes speech produced during the greeting once it finishes", async function () {
+    // Arrange
+    const input = getActorInput(false);
+    const { actor, advance } = await startActor(input);
+
+    // Act: the caller answers over the greeting
+    actor.send({ type: "SPEECH_START" });
+    actor.send({ type: "SPEECH_RESULT", speech: "Hello", responseTime: 100 });
+    await flush();
+    expect(actor.getSnapshot().value).to.equal("greeting");
+
+    await input.deferred.finishPlayback();
+
+    // Assert
+    expect(actor.getSnapshot().value).to.equal("waitingForSpeechTimeout");
+    expect(actor.getSnapshot().context.speechBuffer).to.equal("Hello");
+
+    await advance(MAX_SPEECH_WAIT_TIMEOUT);
+    expect(actor.getSnapshot().value).to.equal("processingUserRequest");
+    expect(input.languageModel.invoke).to.have.been.calledOnce;
+    expect(input.deferred.say).to.have.been.calledWith(ASSISTANT_RESPONSE);
+
+    actor.stop();
+  });
+
+  it("still processes speech produced during the idle message once it finishes", async function () {
+    // Arrange
+    const input = getActorInput(false);
+    const { actor, advance } = await startActor(input);
+    await reachIdleAnnouncement(input, advance);
+    expect(actor.getSnapshot().value).to.equal("announcingIdleTimeout");
+
+    // Act
+    actor.send({ type: "SPEECH_START" });
+    actor.send({
+      type: "SPEECH_RESULT",
+      speech: "Yes, I am here",
+      responseTime: 100
+    });
+    await flush();
+    expect(actor.getSnapshot().value).to.equal("announcingIdleTimeout");
+
+    await input.deferred.finishPlayback();
+
+    // Assert
+    expect(actor.getSnapshot().value).to.equal("waitingForSpeechTimeout");
+    expect(actor.getSnapshot().context.speechBuffer).to.equal("Yes, I am here");
+
+    await advance(MAX_SPEECH_WAIT_TIMEOUT);
+    expect(actor.getSnapshot().value).to.equal("processingUserRequest");
+    expect(input.languageModel.invoke).to.have.been.calledOnce;
+
+    actor.stop();
+  });
+
+  it("processes a speech result that lands just after playback ended", async function () {
+    // Arrange: the VAD start event was consumed while the greeting played, so
+    // the machine is already in "idle" when the transcript arrives.
+    const input = getActorInput(false);
+    const { actor, advance } = await startActor(input);
+    await input.deferred.finishPlayback();
+    expect(actor.getSnapshot().value).to.equal("idle");
+
+    // Act
+    actor.send({ type: "SPEECH_RESULT", speech: "Hello", responseTime: 100 });
+    await flush();
+
+    // Assert
+    expect(actor.getSnapshot().value).to.equal("waitingForSpeechTimeout");
+    expect(actor.getSnapshot().context.speechBuffer).to.equal("Hello");
+
+    await advance(MAX_SPEECH_WAIT_TIMEOUT);
+    expect(actor.getSnapshot().value).to.equal("processingUserRequest");
+    expect(input.languageModel.invoke).to.have.been.calledOnce;
+
+    actor.stop();
+  });
+
+  it("keeps cutting the greeting and the idle message short when barge-in is enabled", async function () {
+    // Arrange
+    const input = getActorInput(true);
+    const { actor, advance } = await startActor(input);
+
+    // Act and assert: the greeting is interrupted
+    actor.send({ type: "SPEECH_START" });
+    await flush();
+    expect(actor.getSnapshot().value).to.equal("listeningToUser");
+    expect(input.voice.stopSpeech).to.have.been.calledOnce;
+
+    // Act and assert: so is the idle message
+    await advance(IDLE_TIMEOUT);
+    expect(actor.getSnapshot().value).to.equal("announcingIdleTimeout");
+    actor.send({ type: "SPEECH_START" });
+    await flush();
+    expect(actor.getSnapshot().value).to.equal("listeningToUser");
+    expect(input.voice.stopSpeech).to.have.been.calledTwice;
+    expect(input.deferred.say).to.have.been.calledWith(IDLE_MESSAGE);
+
+    actor.stop();
+  });
+});

mods/autopilot/test/machine.test.ts

@@ -307,8 +307,10 @@ describe("@autopilot/machine", function () {
 
     const actor = createActor(machine, { input });
 
-    // Act: get to processingUserRequest
+    // Act: get to processingUserRequest. With barge-in off the greeting cannot
+    // be cut short, so wait for it to finish before speaking.
     actor.start();
+    await waitFor(4600);
     actor.send({ type: "SPEECH_START" });
     await waitFor(50);
     actor.send({ type: "SPEECH_RESULT", speech: "Hello", responseTime: 1000 });
@@ -341,8 +343,10 @@ describe("@autopilot/machine", function () {
 
     const actor = createActor(machine, { input });
 
-    // Act: get to processingUserRequest
+    // Act: get to processingUserRequest. With barge-in off the greeting cannot
+    // be cut short, so wait for it to finish before speaking.
     actor.start();
+    await waitFor(4600);
     actor.send({ type: "SPEECH_START" });
     await waitFor(50);
     actor.send({ type: "SPEECH_RESULT", speech: "Hello?", responseTime: 1000 });