Upstream fonoster/fonoster state of the touched files before b6f9124ad6

mods/autopilot/src/machine/machine.ts

@@ -0,0 +1,214 @@
+/**
+ * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
+ * http://github.com/fonoster/fonoster
+ *
+ * This file is part of Fonoster
+ *
+ * Licensed under the MIT License (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    https://opensource.org/licenses/MIT
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { and } from "xstate";
+import { assign } from "xstate";
+import { context } from "./context";
+import { machineSetup } from "./setup";
+
+const machine = machineSetup.createMachine({
+  context,
+  id: "fnAI",
+  initial: "greeting",
+  states: {
+    greeting: {
+      // The greeting is invoked (not fired-and-forgotten) so the machine only
+      // moves to "idle", and therefore only arms the idle clock, once the
+      // first message has been fully played.
+      invoke: {
+        src: "doGreetUser",
+        description: "Answer the call and play the first message",
+        input: ({ context }) => ({ context }),
+        onDone: {
+          target: "idle",
+          description: "Transition to idle after the initial greeting."
+        },
+        onError: {
+          target: "idle"
+        }
+      },
+      on: {
+        SPEECH_START: {
+          target: "listeningToUser",
+          description: "The user barged in during the greeting."
+        }
+      }
+    },
+
+    idle: {
+      entry: [{ type: "cleanSpeech" }],
+      on: {
+        SPEECH_START: {
+          target: "listeningToUser",
+          description: "Event from VAD system."
+        }
+      },
+      after: {
+        IDLE_TIMEOUT: [
+          {
+            target: "hangup",
+            actions: { type: "goodbye" },
+            guard: and(["idleTimeoutCountExceedsMax"])
+          },
+          {
+            target: "announcingIdleTimeout"
+          }
+        ]
+      }
+    },
+
+    announcingIdleTimeout: {
+      // A dedicated state, with no IDLE_TIMEOUT of its own, so the idle clock
+      // is stopped while the idle message plays. It is re-armed only when the
+      // message finishes and the machine goes back to "idle". This prevents
+      // back-to-back idle announcements (and a premature hangup) when the
+      // idle timeout is short relative to the message duration.
+      entry: [{ type: "increaseIdleTimeoutCount" }],
+      invoke: {
+        src: "doAnnounceIdleTimeout",
+        description: "Play the idle message",
+        input: ({ context }) => ({ context }),
+        onDone: {
+          target: "idle",
+          description: "Re-arm the idle clock once the idle message finishes."
+        },
+        onError: {
+          target: "idle"
+        }
+      },
+      on: {
+        SPEECH_START: {
+          target: "listeningToUser",
+          description: "The user spoke while the idle message was playing."
+        }
+      }
+    },
+
+    listeningToUser: {
+      entry: [
+        { type: "interruptPlayback" },
+        { type: "resetIdleTimeoutCount" },
+        assign({ hasLateSpeech: false })
+      ],
+      on: {
+        SPEECH_RESULT: {
+          target: "waitingForSpeechTimeout",
+          actions: {
+            type: "appendSpeech"
+          },
+          description: "Append final speech and process the request.",
+          reenter: true
+        }
+      },
+      after: {
+        IDLE_TIMEOUT: [
+          {
+            target: "hangup",
+            actions: { type: "goodbye" },
+            guard: and(["idleTimeoutCountExceedsMax"])
+          },
+          {
+            target: "announcingIdleTimeout"
+          }
+        ]
+      }
+    },
+
+    waitingForSpeechTimeout: {
+      on: {
+        SPEECH_START: {
+          target: "listeningToUser",
+          description: "User started speaking again.",
+          guard: ({ context }) => context.allowUserBargeIn
+        },
+        SPEECH_RESULT: {
+          target: "waitingForSpeechTimeout",
+          description: "Capture late speech, but only once.",
+          guard: ({ context }) => !context.hasLateSpeech,
+          actions: [{ type: "appendSpeech" }, assign({ hasLateSpeech: true })]
+        }
+      },
+      after: {
+        MAX_SPEECH_WAIT_TIMEOUT: {
+          target: "processingUserRequest",
+          description:
+            "This will give the person time to breathe and speak again.",
+          reenter: true
+        }
+      }
+    },
+
+    hangup: {
+      type: "final"
+    },
+
+    processingUserRequest: {
+      on: {
+        SPEECH_START: {
+          target: "listeningToUser",
+          description: "Event from VAD or similar system.",
+          guard: ({ context }) => context.allowUserBargeIn,
+          // We assume that the user wants to steer the conversation
+          // back to the agent so we clean the speech buffer
+          actions: [{ type: "cleanSpeech" }]
+        },
+        SPEECH_RESULT: {
+          target: "processingUserRequest",
+          description:
+            "Capture only a single late speech across the entire request processing.",
+          guard: ({ context }) =>
+            context.allowUserBargeIn && !context.hasLateSpeech,
+          actions: [
+            { type: "interruptPlayback" },
+            { type: "appendSpeech" },
+            assign(({ self }) => {
+              return {
+                previousState: self.getSnapshot().value,
+                hasLateSpeech: true
+              };
+            })
+          ],
+          reenter: true
+        }
+      },
+      invoke: {
+        src: "doProcessUserRequest",
+        description: "Process the user request",
+        input: ({ context }) => ({ context }),
+        onDone: {
+          target: "listeningToUser",
+          reenter: true,
+          actions: [
+            { type: "cleanSpeech" },
+            assign({
+              isFirstTurn: false
+            })
+          ]
+        }
+      }
+    }
+  },
+  after: {
+    MAX_SESSION_DURATION: {
+      target: ".hangup",
+      actions: { type: "goodbye" }
+    }
+  }
+});
+
+export { machine };

mods/autopilot/test/machine.test.ts

@@ -0,0 +1,371 @@
+/* eslint-disable no-invalid-this */
+/*
+ * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
+ * http://github.com/fonoster/fonoster
+ *
+ * This file is part of Fonoster
+ *
+ * Licensed under the MIT License (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    https://opensource.org/licenses/MIT
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import * as chai from "chai";
+import { expect } from "chai";
+import chaiAsPromised from "chai-as-promised";
+import { createSandbox } from "sinon";
+import sinonChai from "sinon-chai";
+import { createActor } from "xstate";
+import { LanguageModel } from "../src/models";
+import { Voice } from "../src/voice";
+
+chai.use(chaiAsPromised);
+chai.use(sinonChai);
+const sandbox = createSandbox();
+
+const waitFor = async (time: number) => new Promise((r) => setTimeout(r, time));
+
+const FIRST_MESSAGE = "Welcome message";
+const GOODBYE_MESSAGE = "Goodbye message";
+const SYSTEM_ERROR_MESSAGE = "System error message";
+const IDLE_MESSAGE = "Idle message";
+const ASSISTANT_RESPONSE = "Assistant response";
+
+const getActorInput = () => ({
+  languageModel: {
+    invoke: sandbox.stub().resolves({
+      type: "say",
+      content: ASSISTANT_RESPONSE
+    })
+  } as unknown as LanguageModel,
+  voice: {
+    sessionRef: "sessionRef",
+    answer: sandbox.stub().resolves(),
+    sgather: sandbox.stub().resolves({
+      stop: sandbox.stub().resolves(),
+      onData: sandbox.stub()
+    }),
+    stream: sandbox.stub().resolves({
+      stop: sandbox.stub().resolves(),
+      onData: sandbox.stub()
+    }),
+    playDtmf: sandbox.stub().resolves().callsFake(() => {
+      // Add a delay to emulate the dtmf tone
+      return new Promise<void>((resolve) => {
+        setTimeout(() => resolve(), 500);
+      });
+    }),
+    stopSpeech: sandbox.stub().resolves(),
+    // Add a delay to emulate the voice response
+    say: sandbox.stub().resolves().callsFake((content: string) => {
+      return new Promise<void>((resolve) => setTimeout(() => resolve(), 4000));
+    }),
+    hangup: sandbox.stub().resolves()
+  } as unknown as Voice,
+  conversationSettings: {
+    firstMessage: FIRST_MESSAGE,
+    goodbyeMessage: GOODBYE_MESSAGE,
+    systemPrompt: "System template",
+    systemErrorMessage: SYSTEM_ERROR_MESSAGE,
+    maxSpeechWaitTimeout: 500,
+    maxSessionDuration: 30 * 60 * 1000,
+    initialDtmf: "1",
+    allowUserBargeIn: true,
+    idleOptions: {
+      message: IDLE_MESSAGE,
+      timeout: 30000,
+      maxTimeoutCount: 3
+    },
+    transferOptions: {
+      phoneNumber: "+1234567890",
+      message: "Transferring call"
+    },
+    vad: {
+      activationThreshold: 0.3,
+      deactivationThreshold: 0.2,
+      debounceFrames: 3
+    }
+  }
+});
+
+describe("@autopilot/machine", function () {
+  this.slow(30000);
+
+  afterEach(() => sandbox.restore());
+  
+  it("should say the first message then set the state to 'idle'", async function () {
+    // Arrange
+    const { machine } = await import("../src/machine");
+
+    const input = getActorInput();
+    const actor = createActor(machine, {
+      input
+    });
+
+    // Act
+    actor.start();
+
+    // Wait just enough time to answer, play the dtmf (500ms) and say the
+    // first message (4000ms). The machine stays in "greeting" until then.
+    await waitFor(4600);
+
+    // Assert
+    const { context, value: state } = actor.getSnapshot();
+    expect(state).to.equal("idle");
+    expect(input.voice.answer).to.have.been.calledOnce;
+    expect(input.voice.say).to.have.been.calledOnceWith(FIRST_MESSAGE);
+    expect(input.voice.hangup).to.not.have.been.called;
+    expect(context.firstMessage).to.equal(FIRST_MESSAGE);
+    expect(context.goodbyeMessage).to.equal(GOODBYE_MESSAGE);
+    expect(context.systemErrorMessage).to.equal(SYSTEM_ERROR_MESSAGE);
+    expect(context.idleMessage).to.equal(IDLE_MESSAGE);
+    expect(context.idleTimeout).to.equal(30000);
+    expect(context.maxSpeechWaitTimeout).to.equal(500);
+    expect(context.transferMessage).to.equal("Transferring call");
+    expect(context.transferPhoneNumber).to.equal("+1234567890");
+    expect(context.maxIdleTimeoutCount).to.equal(3);
+    expect(context.idleTimeoutCount).to.equal(0);
+    expect(context.speechBuffer).to.equal("");
+    expect(context.allowUserBargeIn).to.equal(true);
+
+    // Cleanup
+    actor.stop();
+  }).timeout(20000);
+
+  it("should set the state to 'idle' and then 'listeningToUser'", async function () {
+    // Arrange
+    const { machine } = await import("../src/machine");
+
+    const input = getActorInput();
+    const actor = createActor(machine, {
+      input
+    });
+
+    // Act
+    actor.start();
+    actor.send({ type: "SPEECH_START" });
+
+    // Assert
+    const { context, value: state } = actor.getSnapshot();
+    expect(state).to.equal("listeningToUser");
+    expect(context.speechBuffer).to.equal("");
+    expect(context.idleTimeoutCount).to.equal(0);
+    expect(input.voice.stopSpeech).to.have.been.calledOnce;
+
+    // Cleanup
+    actor.stop();
+  }).timeout(20000);
+
+  it("does not crash when stopSpeech rejects because the call already hung up", async function () {
+    // Arrange: this reproduces the scenario where the caller hangs up right
+    // as the assistant is speaking. interruptPlayback (a fire-and-forget
+    // entry action) calls voice.stopSpeech(), whose underlying verb rejects
+    // because the session ended before it got a response. Since XState does
+    // not await/track an action's returned promise, an uncaught rejection
+    // here previously crashed the whole process.
+    const { machine } = await import("../src/machine");
+
+    const input = getActorInput();
+    input.voice.stopSpeech = sandbox
+      .stub()
+      .rejects(new Error("voice session ended before the StopSay verb completed"));
+
+    const unhandledRejections: unknown[] = [];
+    const onUnhandledRejection = (reason: unknown) => unhandledRejections.push(reason);
+    process.on("unhandledRejection", onUnhandledRejection);
+
+    const actor = createActor(machine, { input });
+
+    try {
+      // Act
+      actor.start();
+      actor.send({ type: "SPEECH_START" });
+
+      // Give the rejected stopSpeech() promise a chance to settle.
+      await waitFor(50);
+
+      // Assert
+      const { value: state } = actor.getSnapshot();
+      expect(state).to.equal("listeningToUser");
+      expect(input.voice.stopSpeech).to.have.been.calledOnce;
+      expect(unhandledRejections).to.have.lengthOf(0);
+    } finally {
+      process.removeListener("unhandledRejection", onUnhandledRejection);
+      actor.stop();
+    }
+  }).timeout(20000);
+
+  it("should append the speech to the buffer and set the state to 'listeningToUser'", async function () {
+    // Arrange
+    const { machine } = await import("../src/machine");
+
+    const input = getActorInput();
+    const actor = createActor(machine, {
+      input
+    });
+
+    // Act
+    actor.start();
+
+    actor.send({ type: "SPEECH_START" });
+    actor.send({ type: "SPEECH_RESULT", speech: "Well, I personally think that the best way to learn is by doing", responseTime: 1000 });
+    actor.send({ type: "SPEECH_START" });
+    actor.send({ type: "SPEECH_RESULT", speech: "and yeah", responseTime: 1000 });
+
+    await waitFor(50);
+
+    // Goes to "waitingForSpeechTimeout"
+    actor.send({ type: "SPEECH_RESULT", speech: "thats all", responseTime: 1000 });
+
+    const { context, value: state } = actor.getSnapshot();
+    expect(state).to.equal("waitingForSpeechTimeout");
+    expect(context.speechBuffer).to.equal("Well, I personally think that the best way to learn is by doing and yeah thats all");
+    expect(context.idleTimeoutCount).to.equal(0);
+
+    // Cleanup
+    actor.stop();
+  }).timeout(20000);
+
+  it("from 'listeningToUser' call SPEECH_RESULT process the request and goes back to 'listeningToUser'", async function () {
+    // Arrange
+    const { machine } = await import("../src/machine");
+
+    const input = getActorInput();
+    const actor = createActor(machine, {
+      input
+    });
+
+    // Act
+    actor.start();
+    actor.send({ type: "SPEECH_START" });
+    actor.send({ type: "SPEECH_RESULT", speech: "Hello", responseTime: 1000 });
+
+    // Wait just enough to get a MAX_SPEECH_WAIT_TIMEOUT and process the request
+    await waitFor(5500);
+
+    // Assert
+    const { context, value: state } = actor.getSnapshot();
+    expect(state).to.equal("listeningToUser");
+    expect(context.speechBuffer).to.equal("");
+    expect(input.voice.say).to.have.been.calledWith(ASSISTANT_RESPONSE);
+
+    // Cleanup
+    actor.stop();
+  }).timeout(20000);
+
+  it("goes from 'processingUserRequest' back to 'processingUserRequest' when SPEECH_RESULT is received", async function () {
+    // Arrange
+    const { machine } = await import("../src/machine");
+
+    const input = getActorInput();
+    const actor = createActor(machine, {
+      input
+    });
+
+    // Act
+    actor.start();
+    actor.send({ type: "SPEECH_START" });
+
+    await waitFor(50);
+    let { value: state0 } = actor.getSnapshot();
+    expect(state0).to.equal("listeningToUser");
+
+    actor.send({ type: "SPEECH_RESULT", speech: "Hello?", responseTime: 1000 });
+
+    // Wait just enough to get a MAX_SPEECH_WAIT_TIMEOUT
+    await waitFor(600);
+    let { value: state1 } = actor.getSnapshot();
+    expect(state1).to.equal("processingUserRequest");
+
+    actor.send({ type: "SPEECH_RESULT", speech: "Hello!", responseTime: 1000 });
+
+    // Just enough to be in the middle of processing the request
+    await waitFor(2000);
+
+    // Assert
+    const { context, value: state } = actor.getSnapshot();
+    expect(state).to.equal("processingUserRequest");
+    expect(context.speechBuffer).to.equal("Hello? Hello!");
+    expect(input.voice.say).to.have.been.calledWith(ASSISTANT_RESPONSE);
+
+    // Cleanup
+    actor.stop();
+  }).timeout(20000);
+
+  it("with barge-in off, SPEECH_START during processingUserRequest is ignored and AI completes via onDone", async function () {
+    // Arrange: barge-in disabled
+    const { machine } = await import("../src/machine");
+    const input = getActorInput();
+    input.conversationSettings.allowUserBargeIn = false;
+
+    const actor = createActor(machine, { input });
+
+    // Act: get to processingUserRequest
+    actor.start();
+    actor.send({ type: "SPEECH_START" });
+    await waitFor(50);
+    actor.send({ type: "SPEECH_RESULT", speech: "Hello", responseTime: 1000 });
+    await waitFor(600);
+    expect(actor.getSnapshot().value).to.equal("processingUserRequest");
+
+    // Send SPEECH_START while AI is "speaking" (barge-in would interrupt if enabled)
+    actor.send({ type: "SPEECH_START" });
+
+    // Assert: still in processingUserRequest (guard blocked transition)
+    expect(actor.getSnapshot().value).to.equal("processingUserRequest");
+
+    // Wait for AI to finish; should transition via onDone only
+    await waitFor(5500);
+
+    const { context, value: state } = actor.getSnapshot();
+    expect(state).to.equal("listeningToUser");
+    expect(context.speechBuffer).to.equal("");
+    expect(input.voice.say).to.have.been.calledWith(ASSISTANT_RESPONSE);
+    expect(input.languageModel.invoke).to.have.been.calledOnce;
+
+    actor.stop();
+  }).timeout(20000);
+
+  it("with barge-in off, SPEECH_RESULT during processingUserRequest is ignored and AI completes via onDone", async function () {
+    // Arrange: barge-in disabled
+    const { machine } = await import("../src/machine");
+    const input = getActorInput();
+    input.conversationSettings.allowUserBargeIn = false;
+
+    const actor = createActor(machine, { input });
+
+    // Act: get to processingUserRequest
+    actor.start();
+    actor.send({ type: "SPEECH_START" });
+    await waitFor(50);
+    actor.send({ type: "SPEECH_RESULT", speech: "Hello?", responseTime: 1000 });
+    await waitFor(600);
+    expect(actor.getSnapshot().value).to.equal("processingUserRequest");
+
+    // Send late SPEECH_RESULT (would interrupt and reenter if barge-in were enabled)
+    actor.send({ type: "SPEECH_RESULT", speech: "Hello!", responseTime: 1000 });
+    await waitFor(2000);
+
+    // Assert: still in processingUserRequest, buffer unchanged, no reenter (invoke only once)
+    let snapshot = actor.getSnapshot();
+    expect(snapshot.value).to.equal("processingUserRequest");
+    expect(snapshot.context.speechBuffer).to.equal("Hello?");
+    expect(input.languageModel.invoke).to.have.been.calledOnce;
+
+    // Wait for AI to finish via onDone
+    await waitFor(5500);
+    snapshot = actor.getSnapshot();
+    expect(snapshot.value).to.equal("listeningToUser");
+    expect(input.voice.say).to.have.been.calledWith(ASSISTANT_RESPONSE);
+    expect(input.languageModel.invoke).to.have.been.calledOnce;
+
+    actor.stop();
+  }).timeout(20000);
+});