react example is not working

[Lionardo]

I tried to convert the convert the example to a react.js component and everything seems to be fine because I get no errors although the stream doesn't show the video or audio.

component:

 import React, { useState, useEffect, useRef, useCallback } from "react";
const DID_API = {
  key: "your_api_key_here",
  url: "https://api.d-id.com",
  service: "clips",
};

function DiDAvatar() {
  // States for all the labels and buttons
  const [peerStatus, setPeerStatus] = useState("");
  const [iceStatus, setIceStatus] = useState("");
  const [iceGatheringStatus, setIceGatheringStatus] = useState("");
  const [signalingStatus, setSignalingStatus] = useState("");
  const [streamingStatus, setStreamingStatus] = useState("");

  // Refs for the video element and the peer connection
  const videoElement: any = useRef(null);

  const [peerConnection, setpeerConnection] = useState<any | null>(null);
  // Refs for IDs and statuses
  let sessionClientAnswer;

  const [streamId, setstreamId] = useState("");
  const [newSessionId, setnewSessionId] = useState("");

  let videoIsPlaying = false;
  let lastBytesReceived;
  const maxRetryCount = 3;
  const maxDelaySec = 4;

  const presenterInputByService = {
    talks: {
      source_url: "https://d-id-public-bucket.s3.amazonaws.com/or-roman.jpg",
    },
    clips: {
      presenter_id: "rian-lZC6MmWfC1",
      driver_id: "mXra4jY38i",
    },
  };

  // Check API key
  useEffect(() => {
    return () => {
      stopAllStreams();
      closePC();
      // Any other cleanup can go here
    };
  }, []);

  useEffect(() => {
    if (peerConnection) {
      peerConnection.onicegatheringstatechange = onIceGatheringStateChange;
      peerConnection.onicecandidate = onIceCandidate;
      peerConnection.oniceconnectionstatechange = onIceConnectionStateChange;
      peerConnection.onconnectionstatechange = onConnectionStateChange;
      peerConnection.onsignalingstatechange = onSignalingStateChange;
      peerConnection.ontrack = onTrack;
      setpeerConnection(peerConnection);
      console.log("peerConnection add Eventlisteners\n\n", peerConnection);
    }
  }, [peerConnection]);

  const playIdleVideo = () => {
    videoElement.current.srcObject =
      "https://drive.google.com/file/d/1A8NqwOg36mw7XW-e7iY5gvQLLSKykzNq/view?usp=drive_link";
    videoElement.current.loop = true;
  };

  const stopAllStreams = () => {
    // Check if the peer connection exists and has streams
    if (videoElement.current.srcObject) {
      console.log("stopping video streams");
      videoElement.current.srcObject
        .getTracks()
        .forEach((track) => track.stop());
      videoElement.current.srcObject = null;
    }
  };

  const closePC = () => {
    if (peerConnection) {
      // Close the peer connection
      peerConnection.close();
      setpeerConnection(null);
      setPeerStatus("closed");
      setIceStatus("");
      setSignalingStatus("no signaling");
      setStreamingStatus("Stream ended");
      peerConnection.ontrack = null;
      console.log("stopped peer connection");
    }
  };

  // Handlers for peer connection state changes
  const onIceGatheringStateChange = () => {
    if (peerConnection) {
      const status = peerConnection.iceGatheringState;
      setIceGatheringStatus(status);
    }
  };

  const onIceCandidate = (event) => {
    if (event.candidate) {
      const { candidate, sdpMid, sdpMLineIndex } = event.candidate;
      fetch(`${DID_API.url}/${DID_API.service}/streams/${streamId}/ice`, {
        method: "POST",
        headers: {
          Authorization: `Basic ${DID_API.key}`,
          "Content-Type": "application/json",
        },
        body: JSON.stringify({
          candidate,
          sdpMid,
          sdpMLineIndex,
          session_id: newSessionId,
        }),
      });
    }
  };

  const onIceConnectionStateChange = () => {
    const status = peerConnection.iceConnectionState;
    setIceStatus(status);
    if (status === "failed" || status === "closed") {
      stopAllStreams();
      closePC();
    }
  };

  const onConnectionStateChange = () => {
    const status = peerConnection.connectionState;
    setPeerStatus(status);
  };

  const onSignalingStateChange = () => {
    const status = peerConnection.signalingState;
    setSignalingStatus(status);
  };

  const onVideoStatusChange = (videoIsPlaying, stream) => {
    let status;

    if (videoIsPlaying) {
      status = "streaming";
      const remoteStream = stream;
      runVideoElement(remoteStream);
    } else {
      status = "empty";
      playIdleVideo();
    }
    console.log("onVideoStatusChange", status);
  };
  const runVideoElement = (stream) => {
    if (!stream) return;
    videoElement.current.srcObject = stream;
    videoElement.current.src =
      "https://drive.google.com/file/d/1A8NqwOg36mw7XW-e7iY5gvQLLSKykzNq/view?usp=drive_link";
    videoElement.current.loop = false;

    // safari hotfix
    if (videoElement.current.paused) {
      videoElement.current
        .play()
        .then((_) => {})
        .catch((e) => {});
    }
  };

  const onTrack = async (event) => {
    if (!event.track) return;
    const stats = await peerConnection.getStats(event.track);
    stats.forEach((report) => {
      if (report.type === "inbound-rtp" && report.mediaType === "video") {
        const videoStatusChanged =
          videoIsPlaying !== report.bytesReceived > lastBytesReceived;

        if (videoStatusChanged) {
          videoIsPlaying = report.bytesReceived > lastBytesReceived;
          onVideoStatusChange(videoIsPlaying, event.streams[0]);
        }
        lastBytesReceived = report.bytesReceived;
      }
    });
  };
  // Other utility functions like createPeerConnection, setVideoElement, etc.
  const createPeerConnection = async (offer, iceServers) => {
    const peerConnection = new RTCPeerConnection({ iceServers });

    const remoteDescription = new RTCSessionDescription(offer);
    await peerConnection.setRemoteDescription(remoteDescription);
    console.log("set remote sdp OK");

    const sessionClientAnswer = await peerConnection.createAnswer();
    console.log("create local sdp OK");

    await peerConnection.setLocalDescription(sessionClientAnswer);
    console.log("set local sdp OK");

    setpeerConnection(peerConnection);
    return sessionClientAnswer;
  };
  const fetchWithRetries = async (url, options, retries = 1) => {
    try {
      return await fetch(url, options);
    } catch (err) {
      if (retries <= maxRetryCount) {
        const delay =
          Math.min(Math.pow(2, retries) / 4 + Math.random(), maxDelaySec) *
          1000;

        await new Promise((resolve) => setTimeout(resolve, delay));

        console.log(
          `Request failed, retrying ${retries}/${maxRetryCount}. Error ${err}`,
        );
        return fetchWithRetries(url, options, retries + 1);
      } else {
        throw new Error(`Max retries exceeded. error: ${err}`);
      }
    }
  };

  // Handler functions for buttons
  const handleConnectClick = useCallback(async () => {
    if (peerConnection && peerConnection.connectionState === "connected") {
      return;
    }

    stopAllStreams();
    closePC();

    const sessionResponse = await fetchWithRetries(
      `${DID_API.url}/${DID_API.service}/streams`,
      {
        method: "POST",
        headers: {
          Authorization: `Basic ${DID_API.key}`,
          "Content-Type": "application/json",
        },
        body: JSON.stringify(presenterInputByService[DID_API.service]),
      },
    );

    const {
      id: newStreamId,
      offer,
      ice_servers: iceServers,
      session_id: newSessionId,
    } = await sessionResponse.json();
    sessionStorage.setItem("streamId", newStreamId);
    setstreamId(newStreamId);
    setnewSessionId(newSessionId);

    try {
      sessionClientAnswer = await createPeerConnection(offer, iceServers);
    } catch (e) {
      console.log("error during streaming setup", e);
      stopAllStreams();
      closePC();
      return;
    }

    const sdpResponse = await fetch(
      `${DID_API.url}/${DID_API.service}/streams/${sessionStorage.getItem("streamId")}/sdp`,
      {
        method: "POST",
        headers: {
          Authorization: `Basic ${DID_API.key}`,
          "Content-Type": "application/json",
        },
        body: JSON.stringify({
          answer: sessionClientAnswer,
          session_id: newSessionId,
        }),
      },
    );
  }, [streamId]);

  const handleStartClick = async () => {
    // connectionState not supported in firefox
    if (
      peerConnection?.signalingState === "stable" ||
      peerConnection?.iceConnectionState === "connected"
    ) {
      const playResponse = await fetchWithRetries(
        `${DID_API.url}/${DID_API.service}/streams/${streamId}`,
        {
          method: "POST",
          headers: {
            Authorization: `Basic ${DID_API.key}`,
            "Content-Type": "application/json",
          },
          body: JSON.stringify({
            script: {
              type: "text",
              subtitles: "false",
              provider: {
                type: "microsoft",
                voice_id: "en-US-JennyNeural",
                // voice_config: {
                //   style: "string",
                //   rate: "0.5",
                //   pitch: "+2st",
                // },
                language: "English (United States)",
              },
              ssml: true,
              input: "Hello World! And welcome to getitAI.",
            },
            ...(DID_API.service === "clips" && {
              background: {
                color: "#FFFFFF",
              },
            }),
            config: {
              stitch: true,
            },
            session_id: newSessionId,
          }),
        },
      );
      console.log("playResponse", playResponse);
    }
  };

  const handleDestroyClick = async () => {
    await fetch(`${DID_API.url}/${DID_API.service}/streams/${streamId}`, {
      method: "DELETE",
      headers: {
        Authorization: `Basic ${DID_API.key}`,
        "Content-Type": "application/json",
      },
      body: JSON.stringify({ session_id: newSessionId }),
    });

    stopAllStreams();
    closePC();
  };

  // Rendering
  return (
    <div>
      <video width="400" height="400" ref={videoElement} autoPlay />
      <div>
        <li>iceGatheringStatus: {iceGatheringStatus} </li>
        <li>iceStatus: {iceStatus} </li>
        <li>PeerStatus: {peerStatus} </li>
        <li>signalingStatus: {peerConnection?.signalingState} </li>
        <li>streamingStatus: {streamingStatus} </li>
      </div>
      <button className="btn" onClick={handleConnectClick}>
        Connect
      </button>
      <button className="btn" onClick={handleStartClick}>
        Start
      </button>
      <button className="btn" onClick={handleDestroyClick}>
        Destroy
      </button>
    </div>
  );
}

export default DiDAvatar;

[AndreLCanada]

When you setstreamId(newStreamId); inside of handleConnectClick, the state does not change quickly enough for that variable to make it inside of the API url. I assigned it to a local variable in that function and was able to get the connection status.

I unfortunately ran out of credits after that and could not test further. If you find an end-to-end solution, please update this post with it 🤝

[Lionardo]

@AndreLCanada yes I fixed that. Now for some reason the onTrack function is never really applied to

peerConnection.addEventListener("track", onTrack, true);

[Lionardo]

is there an official react implementation of this demo?

[AndreLCanada]

is there an official react implementation of this demo?

Would love to see that from the Team!

[woozay]

This is a React app https://chat.d-id.com/, they should really release that code

[woozay]

@Lionardo did you manage to fix the issue with ontrack not working? It seems to work as soon as you do initialise the peer connection

[woozay]

The team released ai agents and there is a demo in React. It uses a similar concept to the streams. I adapted this code and it worked https://github.com/de-id/agents-sdk/blob/main/src/createStreamingManager.ts

[orgoro]

thanks woozay

[GunithJakka]

When you setstreamId(newStreamId); inside of handleConnectClick, the state does not change quickly enough for that variable to make it inside of the API url. I assigned it to a local variable in that function and was able to get the connection status.

I unfortunately ran out of credits after that and could not test further. If you find an end-to-end solution, please update this post with it 🤝

Hi,

I’m currently trying to test agents using talk streams in a Flutter application, similar to the demo, but I’ve run out of credits. Is there a way to test agents without credits? Do we need credits to test it, or is there an alternative method for development and testing purposes?

Any help would be greatly appreciated!