Spaces:
Running
Running
File size: 5,191 Bytes
53ea588 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 |
// SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: BSD 2-Clause License
import { useCallback, useRef, useState } from "react";
import waitForICEGatheringComplete from "../utils/waitForICEGatheringComplete";
interface Params {
url: string;
rtcConfig: ConstructorParameters<typeof RTCPeerConnection>[0];
onError: (error: Error) => void;
assistant?: string | null;
}
interface ReturnInit {
status: "init";
start: () => void;
}
interface ReturnConnecting {
status: "connecting";
stop: () => void;
}
interface ReturnConnected {
status: "connected";
stop: () => void;
micStream: MediaStream;
stream: MediaStream;
websocket: WebSocket;
}
interface ReturnError {
status: "error";
start: () => void;
error: Error;
}
type Return = ReturnInit | ReturnConnecting | ReturnConnected | ReturnError;
export default function usePipecatWebRTC(params: Params): Return {
const [status, setStatus] = useState<Return["status"]>("init");
const [error, setError] = useState<Error | null>(null);
const micStreamRef = useRef<MediaStream | null>(null);
const outputStreamRef = useRef<MediaStream | null>(null);
const pcRef = useRef<RTCPeerConnection | null>(null);
const wsRef = useRef<WebSocket | null>(null);
const stop = useCallback(() => {
if (wsRef.current) {
wsRef.current.close();
wsRef.current = null;
}
for (const stream of [micStreamRef, outputStreamRef]) {
if (stream.current) {
for (const track of stream.current.getTracks()) {
track.stop();
}
stream.current = null;
}
}
if (pcRef.current) {
for (const transceiver of pcRef.current.getTransceivers()) {
transceiver.stop();
}
pcRef.current.close();
pcRef.current = null;
}
setError(null);
setStatus("init");
}, []);
const connect = useCallback(async () => {
try {
const pc = new RTCPeerConnection(params.rtcConfig);
pcRef.current = pc;
pc.oniceconnectionstatechange = () => {
console.log("oniceconnectionstatechange", pc?.iceConnectionState);
};
pc.onconnectionstatechange = () => {
console.log("onconnectionstatechange", pc?.connectionState);
const connectionState = pc?.connectionState;
if (connectionState === "connected") {
setStatus("connected");
} else if (connectionState === "disconnected") {
stop();
} else if (connectionState === "failed") {
stop();
const err = new Error("WebRTC connection failed");
setError(err);
params.onError(err);
setStatus("error");
}
};
pc.onicecandidate = (event) => {
if (event.candidate) {
console.log("New ICE candidate:", event.candidate);
} else {
console.log("All ICE candidates have been sent.");
}
};
pc.ontrack = (e) => (outputStreamRef.current = e.streams[0]);
micStreamRef.current = await navigator.mediaDevices.getUserMedia({
audio: true,
});
const micTrack = micStreamRef.current.getAudioTracks()[0];
// SmallWebRTCTransport expects to receive both transceivers
pc.addTransceiver(micTrack, { direction: "sendrecv" });
pc.addTransceiver("video", { direction: "sendrecv" });
await pc.setLocalDescription(await pc.createOffer());
await waitForICEGatheringComplete(pc);
const offer = pc.localDescription;
const answer = await new Promise((resolve, reject) => {
const ws = new WebSocket(params.url);
wsRef.current = ws;
ws.onopen = () => {
if (offer) {
ws.send(
JSON.stringify({
sdp: offer.sdp,
type: offer.type,
...(params.assistant ? { assistant: params.assistant } : {}),
})
);
}
};
ws.onmessage = (event) => {
resolve(JSON.parse(event.data));
};
ws.onerror = (error) => {
console.error("WebSocket Error:", error);
reject(
new Error(
`WebSocket error. Is the server running and reachable at ${params.url}?`
)
);
};
});
await pc.setRemoteDescription(answer as RTCSessionDescriptionInit);
} catch (e) {
stop();
const err =
e instanceof Error
? e
: new Error("Unknown error during connection setup");
setError(err);
params.onError(err);
setStatus("error");
}
}, [stop, params.url, params.rtcConfig, params.onError]);
const start = useCallback(() => {
setStatus("connecting");
connect();
}, [connect]);
switch (status) {
case "init":
return { status, start };
case "connecting":
return { status, stop };
case "connected":
return {
status,
stop,
micStream: micStreamRef.current!,
stream: outputStreamRef.current!,
websocket: wsRef.current!,
};
case "error":
return { status, start, error: error! };
}
}
|