From 20009aed53d8864c9204d43a17895168a777d2cc Mon Sep 17 00:00:00 2001 From: Ilan Bigio Date: Mon, 16 Dec 2024 13:06:08 -0800 Subject: Initial commit --- LICENSE | 21 + README.md | 110 + webapp/.env.example | 4 + webapp/.gitignore | 37 + webapp/app/api/twilio/numbers/route.ts | 31 + webapp/app/api/twilio/route.ts | 6 + webapp/app/api/twilio/webhook-local/route.ts | 3 + webapp/app/favicon.ico | Bin 0 -> 65534 bytes webapp/app/globals.css | 69 + webapp/app/layout.tsx | 23 + webapp/app/page.tsx | 5 + webapp/components.json | 17 + webapp/components/backend-tag.tsx | 7 + webapp/components/call-interface.tsx | 95 + webapp/components/checklist-and-config.tsx | 352 +++ webapp/components/function-calls-panel.tsx | 118 + webapp/components/phone-number-checklist.tsx | 67 + webapp/components/session-configuration-panel.tsx | 291 ++ webapp/components/tool-configuration-dialog.tsx | 104 + webapp/components/top-bar.tsx | 37 + webapp/components/transcript.tsx | 104 + webapp/components/types.ts | 31 + webapp/components/ui/alert.tsx | 59 + webapp/components/ui/badge.tsx | 36 + webapp/components/ui/button.tsx | 56 + webapp/components/ui/card.tsx | 79 + webapp/components/ui/checkbox.tsx | 30 + webapp/components/ui/dialog.tsx | 122 + webapp/components/ui/input.tsx | 25 + webapp/components/ui/label.tsx | 26 + webapp/components/ui/scroll-area.tsx | 48 + webapp/components/ui/select.tsx | 160 + webapp/components/ui/textarea.tsx | 24 + webapp/lib/handle-realtime-event.ts | 231 ++ webapp/lib/tool-templates.ts | 65 + webapp/lib/twilio.ts | 13 + webapp/lib/use-backend-tools.ts | 32 + webapp/lib/utils.ts | 6 + webapp/next.config.mjs | 4 + webapp/package-lock.json | 3240 +++++++++++++++++++++ webapp/package.json | 37 + webapp/postcss.config.mjs | 8 + webapp/public/next.svg | 1 + webapp/public/vercel.svg | 1 + webapp/tailwind.config.ts | 80 + webapp/tsconfig.json | 26 + websocket-server/.env.example | 4 + websocket-server/.gitignore | 37 + websocket-server/package-lock.json | 1519 ++++++++++ websocket-server/package.json | 32 + websocket-server/src/functionHandlers.ts | 33 + websocket-server/src/server.ts | 77 + websocket-server/src/sessionManager.ts | 286 ++ websocket-server/src/twiml.xml | 8 + websocket-server/src/types.ts | 31 + websocket-server/tsconfig.json | 18 + 56 files changed, 7986 insertions(+) create mode 100644 LICENSE create mode 100644 README.md create mode 100644 webapp/.env.example create mode 100644 webapp/.gitignore create mode 100644 webapp/app/api/twilio/numbers/route.ts create mode 100644 webapp/app/api/twilio/route.ts create mode 100644 webapp/app/api/twilio/webhook-local/route.ts create mode 100644 webapp/app/favicon.ico create mode 100644 webapp/app/globals.css create mode 100644 webapp/app/layout.tsx create mode 100644 webapp/app/page.tsx create mode 100644 webapp/components.json create mode 100644 webapp/components/backend-tag.tsx create mode 100644 webapp/components/call-interface.tsx create mode 100644 webapp/components/checklist-and-config.tsx create mode 100644 webapp/components/function-calls-panel.tsx create mode 100644 webapp/components/phone-number-checklist.tsx create mode 100644 webapp/components/session-configuration-panel.tsx create mode 100644 webapp/components/tool-configuration-dialog.tsx create mode 100644 webapp/components/top-bar.tsx create mode 100644 webapp/components/transcript.tsx create mode 100644 webapp/components/types.ts create mode 100644 webapp/components/ui/alert.tsx create mode 100644 webapp/components/ui/badge.tsx create mode 100644 webapp/components/ui/button.tsx create mode 100644 webapp/components/ui/card.tsx create mode 100644 webapp/components/ui/checkbox.tsx create mode 100644 webapp/components/ui/dialog.tsx create mode 100644 webapp/components/ui/input.tsx create mode 100644 webapp/components/ui/label.tsx create mode 100644 webapp/components/ui/scroll-area.tsx create mode 100644 webapp/components/ui/select.tsx create mode 100644 webapp/components/ui/textarea.tsx create mode 100644 webapp/lib/handle-realtime-event.ts create mode 100644 webapp/lib/tool-templates.ts create mode 100644 webapp/lib/twilio.ts create mode 100644 webapp/lib/use-backend-tools.ts create mode 100644 webapp/lib/utils.ts create mode 100644 webapp/next.config.mjs create mode 100644 webapp/package-lock.json create mode 100644 webapp/package.json create mode 100644 webapp/postcss.config.mjs create mode 100644 webapp/public/next.svg create mode 100644 webapp/public/vercel.svg create mode 100644 webapp/tailwind.config.ts create mode 100644 webapp/tsconfig.json create mode 100644 websocket-server/.env.example create mode 100644 websocket-server/.gitignore create mode 100644 websocket-server/package-lock.json create mode 100644 websocket-server/package.json create mode 100644 websocket-server/src/functionHandlers.ts create mode 100644 websocket-server/src/server.ts create mode 100644 websocket-server/src/sessionManager.ts create mode 100644 websocket-server/src/twiml.xml create mode 100644 websocket-server/src/types.ts create mode 100644 websocket-server/tsconfig.json diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..1efeca2 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 OpenAI + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..3acc522 --- /dev/null +++ b/README.md @@ -0,0 +1,110 @@ +# OpenAI Realtime API with Twilio Quickstart + +Combine OpenAI's Realtime API and Twilio's phone calling capability to build an AI calling assistant. + +Screenshot 2024-12-17 at 1 46 18 AM + +## Quick Setup + +Open three terminal windows: + +| Terminal | Purpose | Quick Reference (see below for more) | +| -------- | ----------------------------- | ------------------------------------ | +| 1 | To run the `webapp` | `npm run dev` | +| 2 | To run the `websocket-server` | `npm run dev` | +| 3 | To run `ngrok` | `ngrok http 8081` | + +Make sure all vars in `webapp/.env` and `websocket-server/.env` are set correctly. See [full setup](#full-setup) section for more. + +## Overview + +This repo implements a phone calling assistant with the Realtime API and Twilio, and had two main parts: the `webapp`, and the `websocket-server`. + +1. `webapp`: NextJS app to serve as a frontend for call configuration and transcripts +2. `websocket-server`: Express backend that handles connection from Twilio, connects it to the Realtime API, and forwards messages to the frontend + +Screenshot 2024-12-17 at 12 54 12 PM + +Twilio uses TwiML (a form of XML) to specify how to handle a phone call. When a call comes in we tell Twilio to start a bi-directional stream to our backend, where we forward messages between the call and the Realtime API. (`{{WS_URL}}` is replaced with our websocket endpoint.) + +```xml + + + + + Connected + + + + Disconnected + +``` + +We use `ngrok` to make our server reachable by Twilio. + +### Life of a phone call + +Setup + +1. We run ngrok to make our server reachable by Twilio +1. We set the Twilio webhook to our ngrok address +1. Frontend connects to the backend (`wss://[your_backend]/logs`), ready for a call + +Call + +1. Call is placed to Twilio-managed number +1. Twilio queries the webhook (`http://[your_backend]/twiml`) for TwiML instructions +1. Twilio opens a bi-directional stream to the backend (`wss://[your_backend]/call`) +1. The backend connects to the Realtime API, and starts forwarding messages: + - between Twilio and the Realtime API + - between the frontend and the Realtime API + +### Function Calling + +This demo mocks out function calls so you can provide sample responses. In reality you could handle the function call, execute some code, and then supply the response back to the model. + +## Full Setup + +1. Make sure your [auth & env](#detailed-auth--env) is configured correctly. + +2. Run webapp. + +```shell +cd webapp +npm install +npm run dev +``` + +3. Run websocket server. + +```shell +cd websocket-server +npm install +npm run dev +``` + +## Detailed Auth & Env + +### OpenAI & Twilio + +Set your credentials in `webapp/.env` and `websocket-server` - see `webapp/.env.example` and `websocket-server.env.example` for reference. + +### Ngrok + +Twilio needs to be able to reach your websocket server. If you're running it locally, your ports are inaccessible by default. [ngrok](https://ngrok.com/) can make them temporarily accessible. + +We have set the `websocket-server` to run on port `8081` by default, so that is the port we will be forwarding. + +```shell +ngrok http 8081 +``` + +Make note of the `Forwarding` URL. (e.g. `https://54c5-35-170-32-42.ngrok-free.app`) + +### Websocket URL + +Your server should now be accessible at the `Forwarding` URL when run, so set the `PUBLIC_URL` in `websocket-server/.env`. See `websocket-server/.env.example` for reference. + +# Additional Notes + +This repo isn't polished, and the security practices leave some to be desired. Please only use this as reference, and make sure to audit your app with security and engineering before deploying! diff --git a/webapp/.env.example b/webapp/.env.example new file mode 100644 index 0000000..7007f18 --- /dev/null +++ b/webapp/.env.example @@ -0,0 +1,4 @@ +# rename this to .env + +TWILIO_ACCOUNT_SID="" +TWILIO_AUTH_TOKEN="" \ No newline at end of file diff --git a/webapp/.gitignore b/webapp/.gitignore new file mode 100644 index 0000000..00bba9b --- /dev/null +++ b/webapp/.gitignore @@ -0,0 +1,37 @@ +# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. + +# dependencies +/node_modules +/.pnp +.pnp.js +.yarn/install-state.gz + +# testing +/coverage + +# next.js +/.next/ +/out/ + +# production +/build + +# misc +.DS_Store +*.pem + +# debug +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# local env files +.env*.local +.env + +# vercel +.vercel + +# typescript +*.tsbuildinfo +next-env.d.ts diff --git a/webapp/app/api/twilio/numbers/route.ts b/webapp/app/api/twilio/numbers/route.ts new file mode 100644 index 0000000..53c13c8 --- /dev/null +++ b/webapp/app/api/twilio/numbers/route.ts @@ -0,0 +1,31 @@ +import twilioClient from "@/lib/twilio"; + +export async function GET() { + if (!twilioClient) { + return Response.json( + { error: "Twilio client not initialized" }, + { status: 500 } + ); + } + + const incomingPhoneNumbers = await twilioClient.incomingPhoneNumbers.list({ + limit: 20, + }); + return Response.json(incomingPhoneNumbers); +} + +export async function POST(req: Request) { + if (!twilioClient) { + return Response.json( + { error: "Twilio client not initialized" }, + { status: 500 } + ); + } + + const { phoneNumberSid, voiceUrl } = await req.json(); + const incomingPhoneNumber = await twilioClient + .incomingPhoneNumbers(phoneNumberSid) + .update({ voiceUrl }); + + return Response.json(incomingPhoneNumber); +} diff --git a/webapp/app/api/twilio/route.ts b/webapp/app/api/twilio/route.ts new file mode 100644 index 0000000..6313295 --- /dev/null +++ b/webapp/app/api/twilio/route.ts @@ -0,0 +1,6 @@ +export async function GET() { + const credentialsSet = Boolean( + process.env.TWILIO_ACCOUNT_SID && process.env.TWILIO_AUTH_TOKEN + ); + return Response.json({ credentialsSet }); +} diff --git a/webapp/app/api/twilio/webhook-local/route.ts b/webapp/app/api/twilio/webhook-local/route.ts new file mode 100644 index 0000000..2dc5420 --- /dev/null +++ b/webapp/app/api/twilio/webhook-local/route.ts @@ -0,0 +1,3 @@ +export async function GET() { + return Response.json({ webhookUrl: process.env.TWILIO_WEBHOOK_URL }); +} diff --git a/webapp/app/favicon.ico b/webapp/app/favicon.ico new file mode 100644 index 0000000..f3e821d Binary files /dev/null and b/webapp/app/favicon.ico differ diff --git a/webapp/app/globals.css b/webapp/app/globals.css new file mode 100644 index 0000000..99a7b0c --- /dev/null +++ b/webapp/app/globals.css @@ -0,0 +1,69 @@ +@tailwind base; +@tailwind components; +@tailwind utilities; + +@layer base { + :root { + --background: 0 0% 100%; + --foreground: 222.2 84% 4.9%; + --card: 0 0% 100%; + --card-foreground: 222.2 84% 4.9%; + --popover: 0 0% 100%; + --popover-foreground: 222.2 84% 4.9%; + --primary: 222.2 47.4% 11.2%; + --primary-foreground: 210 40% 98%; + --secondary: 210 40% 96.1%; + --secondary-foreground: 222.2 47.4% 11.2%; + --muted: 210 40% 96.1%; + --muted-foreground: 215.4 16.3% 46.9%; + --accent: 210 40% 96.1%; + --accent-foreground: 222.2 47.4% 11.2%; + --destructive: 0 84.2% 60.2%; + --destructive-foreground: 210 40% 98%; + --border: 214.3 31.8% 91.4%; + --input: 214.3 31.8% 91.4%; + --ring: 222.2 84% 4.9%; + --radius: 0.5rem; + --chart-1: 12 76% 61%; + --chart-2: 173 58% 39%; + --chart-3: 197 37% 24%; + --chart-4: 43 74% 66%; + --chart-5: 27 87% 67%; + } + + .dark { + --background: 222.2 84% 4.9%; + --foreground: 210 40% 98%; + --card: 222.2 84% 4.9%; + --card-foreground: 210 40% 98%; + --popover: 222.2 84% 4.9%; + --popover-foreground: 210 40% 98%; + --primary: 210 40% 98%; + --primary-foreground: 222.2 47.4% 11.2%; + --secondary: 217.2 32.6% 17.5%; + --secondary-foreground: 210 40% 98%; + --muted: 217.2 32.6% 17.5%; + --muted-foreground: 215 20.2% 65.1%; + --accent: 217.2 32.6% 17.5%; + --accent-foreground: 210 40% 98%; + --destructive: 0 62.8% 30.6%; + --destructive-foreground: 210 40% 98%; + --border: 217.2 32.6% 17.5%; + --input: 217.2 32.6% 17.5%; + --ring: 212.7 26.8% 83.9%; + --chart-1: 220 70% 50%; + --chart-2: 160 60% 45%; + --chart-3: 30 80% 55%; + --chart-4: 280 65% 60%; + --chart-5: 340 75% 55%; + } +} + +@layer base { + * { + @apply border-border; + } + body { + @apply bg-background text-foreground; + } +} \ No newline at end of file diff --git a/webapp/app/layout.tsx b/webapp/app/layout.tsx new file mode 100644 index 0000000..6907ee0 --- /dev/null +++ b/webapp/app/layout.tsx @@ -0,0 +1,23 @@ +import type { Metadata } from "next"; +import { Inter } from "next/font/google"; +import "./globals.css"; + +const inter = Inter({ subsets: ["latin"] }); + +export const metadata: Metadata = { + title: "OpenAI Realtime + Twilio", + description: + "Sample phone call assistant app for OpenAI Realtime API and Twilio", +}; + +export default function RootLayout({ + children, +}: Readonly<{ + children: React.ReactNode; +}>) { + return ( + + {children} + + ); +} diff --git a/webapp/app/page.tsx b/webapp/app/page.tsx new file mode 100644 index 0000000..07297e4 --- /dev/null +++ b/webapp/app/page.tsx @@ -0,0 +1,5 @@ +import CallInterface from "@/components/call-interface"; + +export default function Page() { + return ; +} diff --git a/webapp/components.json b/webapp/components.json new file mode 100644 index 0000000..15f2b02 --- /dev/null +++ b/webapp/components.json @@ -0,0 +1,17 @@ +{ + "$schema": "https://ui.shadcn.com/schema.json", + "style": "default", + "rsc": true, + "tsx": true, + "tailwind": { + "config": "tailwind.config.ts", + "css": "app/globals.css", + "baseColor": "slate", + "cssVariables": true, + "prefix": "" + }, + "aliases": { + "components": "@/components", + "utils": "@/lib/utils" + } +} \ No newline at end of file diff --git a/webapp/components/backend-tag.tsx b/webapp/components/backend-tag.tsx new file mode 100644 index 0000000..722a00a --- /dev/null +++ b/webapp/components/backend-tag.tsx @@ -0,0 +1,7 @@ +import React from "react"; + +export const BackendTag = () => ( + + backend + +); diff --git a/webapp/components/call-interface.tsx b/webapp/components/call-interface.tsx new file mode 100644 index 0000000..6ead90a --- /dev/null +++ b/webapp/components/call-interface.tsx @@ -0,0 +1,95 @@ +"use client"; + +import React, { useState, useEffect } from "react"; +import TopBar from "@/components/top-bar"; +import ChecklistAndConfig from "@/components/checklist-and-config"; +import SessionConfigurationPanel from "@/components/session-configuration-panel"; +import Transcript from "@/components/transcript"; +import FunctionCallsPanel from "@/components/function-calls-panel"; +import { Item } from "@/components/types"; +import handleRealtimeEvent from "@/lib/handle-realtime-event"; +import PhoneNumberChecklist from "@/components/phone-number-checklist"; + +const CallInterface = () => { + const [selectedPhoneNumber, setSelectedPhoneNumber] = useState(""); + const [allConfigsReady, setAllConfigsReady] = useState(false); + const [items, setItems] = useState([]); + const [callStatus, setCallStatus] = useState("disconnected"); + const [ws, setWs] = useState(null); + + useEffect(() => { + if (allConfigsReady && !ws) { + const newWs = new WebSocket("ws://localhost:8081/logs"); + + newWs.onopen = () => { + console.log("Connected to logs websocket"); + setCallStatus("connected"); + }; + + newWs.onmessage = (event) => { + const data = JSON.parse(event.data); + console.log("Received logs event:", data); + handleRealtimeEvent(data, setItems); + }; + + newWs.onclose = () => { + console.log("Logs websocket disconnected"); + setWs(null); + setCallStatus("disconnected"); + }; + + setWs(newWs); + } + }, [allConfigsReady, ws]); + + return ( +
+ + +
+
+ {/* Left Column */} +
+ { + if (ws && ws.readyState === WebSocket.OPEN) { + const updateEvent = { + type: "session.update", + session: { + ...config, + }, + }; + console.log("Sending update event:", updateEvent); + ws.send(JSON.stringify(updateEvent)); + } + }} + /> +
+ + {/* Middle Column: Transcript */} +
+ + +
+ + {/* Right Column: Function Calls */} +
+ +
+
+
+
+ ); +}; + +export default CallInterface; diff --git a/webapp/components/checklist-and-config.tsx b/webapp/components/checklist-and-config.tsx new file mode 100644 index 0000000..21e47d5 --- /dev/null +++ b/webapp/components/checklist-and-config.tsx @@ -0,0 +1,352 @@ +"use client"; + +import React, { useEffect, useState, useMemo } from "react"; +import { + Dialog, + DialogContent, + DialogHeader, + DialogTitle, + DialogDescription, +} from "@/components/ui/dialog"; +import { Input } from "@/components/ui/input"; +import { Button } from "@/components/ui/button"; +import { Circle, CheckCircle, Loader2 } from "lucide-react"; +import { PhoneNumber } from "@/components/types"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; + +export default function ChecklistAndConfig({ + ready, + setReady, + selectedPhoneNumber, + setSelectedPhoneNumber, +}: { + ready: boolean; + setReady: (val: boolean) => void; + selectedPhoneNumber: string; + setSelectedPhoneNumber: (val: string) => void; +}) { + const [hasCredentials, setHasCredentials] = useState(false); + const [phoneNumbers, setPhoneNumbers] = useState([]); + const [currentNumberSid, setCurrentNumberSid] = useState(""); + const [currentVoiceUrl, setCurrentVoiceUrl] = useState(""); + + const [publicUrl, setPublicUrl] = useState(""); + const [localServerUp, setLocalServerUp] = useState(false); + const [publicUrlAccessible, setPublicUrlAccessible] = useState(false); + + const [allChecksPassed, setAllChecksPassed] = useState(false); + const [webhookLoading, setWebhookLoading] = useState(false); + const [ngrokLoading, setNgrokLoading] = useState(false); + + const appendedTwimlUrl = publicUrl ? `${publicUrl}/twiml` : ""; + const isWebhookMismatch = + appendedTwimlUrl && currentVoiceUrl && appendedTwimlUrl !== currentVoiceUrl; + + useEffect(() => { + let polling = true; + + const pollChecks = async () => { + try { + // 1. Check credentials + let res = await fetch("/api/twilio"); + if (!res.ok) throw new Error("Failed credentials check"); + const credData = await res.json(); + setHasCredentials(!!credData?.credentialsSet); + + // 2. Fetch numbers + res = await fetch("/api/twilio/numbers"); + if (!res.ok) throw new Error("Failed to fetch phone numbers"); + const numbersData = await res.json(); + if (Array.isArray(numbersData) && numbersData.length > 0) { + setPhoneNumbers(numbersData); + // If currentNumberSid not set or not in the list, use first + const selected = + numbersData.find((p: PhoneNumber) => p.sid === currentNumberSid) || + numbersData[0]; + setCurrentNumberSid(selected.sid); + setCurrentVoiceUrl(selected.voiceUrl || ""); + setSelectedPhoneNumber(selected.friendlyName || ""); + } + + // 3. Check local server & public URL + let foundPublicUrl = ""; + try { + const resLocal = await fetch("http://localhost:8081/public-url"); + if (resLocal.ok) { + const pubData = await resLocal.json(); + foundPublicUrl = pubData?.publicUrl || ""; + setLocalServerUp(true); + setPublicUrl(foundPublicUrl); + } else { + throw new Error("Local server not responding"); + } + } catch { + setLocalServerUp(false); + setPublicUrl(""); + } + } catch (err) { + console.error(err); + } + }; + + pollChecks(); + const intervalId = setInterval(() => polling && pollChecks(), 1000); + return () => { + polling = false; + clearInterval(intervalId); + }; + }, [currentNumberSid, setSelectedPhoneNumber]); + + const updateWebhook = async () => { + if (!currentNumberSid || !appendedTwimlUrl) return; + try { + setWebhookLoading(true); + const res = await fetch("/api/twilio/numbers", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + phoneNumberSid: currentNumberSid, + voiceUrl: appendedTwimlUrl, + }), + }); + if (!res.ok) throw new Error("Failed to update webhook"); + setCurrentVoiceUrl(appendedTwimlUrl); + } catch (err) { + console.error(err); + } finally { + setWebhookLoading(false); + } + }; + + const checkNgrok = async () => { + if (!localServerUp || !publicUrl) return; + setNgrokLoading(true); + let success = false; + for (let i = 0; i < 5; i++) { + try { + const resTest = await fetch(publicUrl + "/public-url"); + if (resTest.ok) { + setPublicUrlAccessible(true); + success = true; + break; + } + } catch { + // retry + } + if (i < 4) { + await new Promise((r) => setTimeout(r, 3000)); + } + } + if (!success) { + setPublicUrlAccessible(false); + } + setNgrokLoading(false); + }; + + const checklist = useMemo(() => { + return [ + { + label: "Set up Twilio account", + done: hasCredentials, + description: "Then update account details in webapp/.env", + field: ( + + ), + }, + { + label: "Set up Twilio phone number", + done: phoneNumbers.length > 0, + description: "Costs around $1.15/month", + field: + phoneNumbers.length > 0 ? ( + phoneNumbers.length === 1 ? ( + + ) : ( + + ) + ) : ( + + ), + }, + { + label: "Start local WebSocket server", + done: localServerUp, + description: "cd websocket-server && npm run dev", + field: null, + }, + { + label: "Start ngrok", + done: publicUrlAccessible, + description: "Then set ngrok URL in websocket-server/.env", + field: ( +
+
+ +
+
+ +
+
+ ), + }, + { + label: "Update Twilio webhook URL", + done: !!publicUrl && !isWebhookMismatch, + description: "Can also be done manually in Twilio console", + field: ( +
+
+ +
+
+ +
+
+ ), + }, + ]; + }, [ + hasCredentials, + phoneNumbers, + currentNumberSid, + localServerUp, + publicUrl, + publicUrlAccessible, + currentVoiceUrl, + isWebhookMismatch, + appendedTwimlUrl, + webhookLoading, + ngrokLoading, + setSelectedPhoneNumber, + ]); + + useEffect(() => { + setAllChecksPassed(checklist.every((item) => item.done)); + }, [checklist]); + + useEffect(() => { + if (!ready) { + checkNgrok(); + } + }, [localServerUp, ready]); + + useEffect(() => { + if (!allChecksPassed) { + setReady(false); + } + }, [allChecksPassed, setReady]); + + const handleDone = () => setReady(true); + + return ( + + + + Setup Checklist + + This sample app requires a few steps before you get started + + + +
+ {checklist.map((item, i) => ( +
+
+
+ {item.done ? ( + + ) : ( + + )} + {item.label} +
+ {item.description && ( +

+ {item.description} +

+ )} +
+
{item.field}
+
+ ))} +
+ +
+ +
+ + + ); +} diff --git a/webapp/components/function-calls-panel.tsx b/webapp/components/function-calls-panel.tsx new file mode 100644 index 0000000..75be81f --- /dev/null +++ b/webapp/components/function-calls-panel.tsx @@ -0,0 +1,118 @@ +import React, { useState } from "react"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { ScrollArea } from "@/components/ui/scroll-area"; +import { Input } from "@/components/ui/input"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Item } from "@/components/types"; + +type FunctionCallsPanelProps = { + items: Item[]; + ws?: WebSocket | null; // pass down ws from parent +}; + +const FunctionCallsPanel: React.FC = ({ + items, + ws, +}) => { + const [responses, setResponses] = useState>({}); + + // Filter function_call items + const functionCalls = items.filter((it) => it.type === "function_call"); + + // For each function_call, check for a corresponding function_call_output + const functionCallsWithStatus = functionCalls.map((call) => { + const outputs = items.filter( + (it) => it.type === "function_call_output" && it.call_id === call.call_id + ); + const outputItem = outputs[0]; + const completed = call.status === "completed" || !!outputItem; + const response = outputItem ? outputItem.output : undefined; + return { + ...call, + completed, + response, + }; + }); + + const handleChange = (call_id: string, value: string) => { + setResponses((prev) => ({ ...prev, [call_id]: value })); + }; + + const handleSubmit = (call: Item) => { + if (!ws || ws.readyState !== WebSocket.OPEN) return; + const call_id = call.call_id || ""; + ws.send( + JSON.stringify({ + type: "conversation.item.create", + item: { + type: "function_call_output", + call_id: call_id, + output: JSON.stringify(responses[call_id] || ""), + }, + }) + ); + // Ask the model to continue after providing the tool response + ws.send(JSON.stringify({ type: "response.create" })); + }; + + return ( + + + + Function Calls + + + + +
+ {functionCallsWithStatus.map((call) => ( +
+
+

{call.name}

+ + {call.completed ? "Completed" : "Pending"} + +
+ +
+ {JSON.stringify(call.params)} +
+ + {!call.completed ? ( +
+ + handleChange(call.call_id || "", e.target.value) + } + /> + +
+ ) : ( +
+ {JSON.stringify(JSON.parse(call.response || ""))} +
+ )} +
+ ))} +
+ + + + ); +}; + +export default FunctionCallsPanel; diff --git a/webapp/components/phone-number-checklist.tsx b/webapp/components/phone-number-checklist.tsx new file mode 100644 index 0000000..5aa50db --- /dev/null +++ b/webapp/components/phone-number-checklist.tsx @@ -0,0 +1,67 @@ +// PhoneNumberChecklist.tsx +"use client"; + +import React, { useState } from "react"; +import { Card } from "@/components/ui/card"; +import { CheckCircle, Circle, Eye, EyeOff } from "lucide-react"; +import { Button } from "@/components/ui/button"; + +type PhoneNumberChecklistProps = { + selectedPhoneNumber: string; + allConfigsReady: boolean; + setAllConfigsReady: (ready: boolean) => void; +}; + +const PhoneNumberChecklist: React.FC = ({ + selectedPhoneNumber, + allConfigsReady, + setAllConfigsReady, +}) => { + const [isVisible, setIsVisible] = useState(true); + + return ( + +
+ Number +
+ + {isVisible ? selectedPhoneNumber || "None" : "••••••••••"} + + +
+
+
+
+ {allConfigsReady ? ( + + ) : ( + + )} + + {allConfigsReady ? "Setup Ready" : "Setup Not Ready"} + +
+ +
+ + ); +}; + +export default PhoneNumberChecklist; diff --git a/webapp/components/session-configuration-panel.tsx b/webapp/components/session-configuration-panel.tsx new file mode 100644 index 0000000..85c9fc1 --- /dev/null +++ b/webapp/components/session-configuration-panel.tsx @@ -0,0 +1,291 @@ +import React, { useState, useEffect } from "react"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { ScrollArea } from "@/components/ui/scroll-area"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Textarea } from "@/components/ui/textarea"; +import { Button } from "@/components/ui/button"; +import { Plus, Edit, Trash, Check, AlertCircle } from "lucide-react"; +import { toolTemplates } from "@/lib/tool-templates"; +import { ToolConfigurationDialog } from "./tool-configuration-dialog"; +import { BackendTag } from "./backend-tag"; +import { useBackendTools } from "@/lib/use-backend-tools"; + +interface SessionConfigurationPanelProps { + callStatus: string; + onSave: (config: any) => void; +} + +const SessionConfigurationPanel: React.FC = ({ + callStatus, + onSave, +}) => { + const [instructions, setInstructions] = useState( + "You are a helpful assistant in a phone call." + ); + const [voice, setVoice] = useState("ash"); + const [tools, setTools] = useState([]); + const [editingIndex, setEditingIndex] = useState(null); + const [editingSchemaStr, setEditingSchemaStr] = useState(""); + const [isJsonValid, setIsJsonValid] = useState(true); + const [openDialog, setOpenDialog] = useState(false); + const [selectedTemplate, setSelectedTemplate] = useState(""); + const [saveStatus, setSaveStatus] = useState< + "idle" | "saving" | "saved" | "error" + >("idle"); + const [hasUnsavedChanges, setHasUnsavedChanges] = useState(false); + + // Custom hook to fetch backend tools every 3 seconds + const backendTools = useBackendTools("http://localhost:8081/tools", 3000); + + // Track changes to determine if there are unsaved modifications + useEffect(() => { + setHasUnsavedChanges(true); + }, [instructions, voice, tools]); + + // Reset save status after a delay when saved + useEffect(() => { + if (saveStatus === "saved") { + const timer = setTimeout(() => { + setSaveStatus("idle"); + }, 3000); + return () => clearTimeout(timer); + } + }, [saveStatus]); + + const handleSave = async () => { + setSaveStatus("saving"); + try { + await onSave({ + instructions, + voice, + tools: tools.map((tool) => JSON.parse(tool)), + }); + setSaveStatus("saved"); + setHasUnsavedChanges(false); + } catch (error) { + setSaveStatus("error"); + } + }; + + const handleAddTool = () => { + setEditingIndex(null); + setEditingSchemaStr(""); + setSelectedTemplate(""); + setIsJsonValid(true); + setOpenDialog(true); + }; + + const handleEditTool = (index: number) => { + setEditingIndex(index); + setEditingSchemaStr(tools[index] || ""); + setSelectedTemplate(""); + setIsJsonValid(true); + setOpenDialog(true); + }; + + const handleDeleteTool = (index: number) => { + const newTools = [...tools]; + newTools.splice(index, 1); + setTools(newTools); + }; + + const handleDialogSave = () => { + try { + JSON.parse(editingSchemaStr); + } catch { + return; + } + const newTools = [...tools]; + if (editingIndex === null) { + newTools.push(editingSchemaStr); + } else { + newTools[editingIndex] = editingSchemaStr; + } + setTools(newTools); + setOpenDialog(false); + }; + + const handleTemplateChange = (val: string) => { + setSelectedTemplate(val); + + // Determine if the selected template is from local or backend + let templateObj = + toolTemplates.find((t) => t.name === val) || + backendTools.find((t: any) => t.name === val); + + if (templateObj) { + setEditingSchemaStr(JSON.stringify(templateObj, null, 2)); + setIsJsonValid(true); + } + }; + + const onSchemaChange = (value: string) => { + setEditingSchemaStr(value); + try { + JSON.parse(value); + setIsJsonValid(true); + } catch { + setIsJsonValid(false); + } + }; + + const getToolNameFromSchema = (schema: string): string => { + try { + const parsed = JSON.parse(schema); + return parsed?.name || "Untitled Tool"; + } catch { + return "Invalid JSON"; + } + }; + + const isBackendTool = (name: string): boolean => { + return backendTools.some((t: any) => t.name === name); + }; + + return ( + + +
+ + Session Configuration + +
+ {saveStatus === "error" ? ( + + + Save failed + + ) : hasUnsavedChanges ? ( + Not saved + ) : ( + + + Saved + + )} +
+
+ + + +
+
+ +