WebRTC - 3. React WebRTC

web은 svelte로 개발하고 react로 옮겨서 다시 작업을 했습니다

 

하다보니 svelte가 정말 편하구나.. 싶었던 순간이 많았던것 같습니다

 

상태관리 라이브러리는 사용하지 않았고 socket io로 소켓 통신을 처리했습니다

 

src/main/index.tsx

import React from "react";
import useWebRTCHook from "src/hooks/useWebRTCHook";
import BtnArea from "./BtnArea";
import CallPopup from "./CallPopup";
import UserListArea from "./UserListArea";
import VideoArea from "./VideoArea";
import styles from './main.module.css'

const Main = () => {
  const {
    model,
    turnOnMedia,
    turnOffMedia,
    sendOffer,
    refuse,
    sendAnswer,
    close,
    setTo,
    localRef,
    remoteRef,
  } = useWebRTCHook();

  return (
    <div className={styles.div_main}>
      <CallPopup
        model={model}
        sendAnswer={sendAnswer}
        refuse={refuse}
        close={close}
      />
      <UserListArea model={model} setTo={setTo} sendOffer={sendOffer} />
      <VideoArea model={model} localRef={localRef} remoteRef={remoteRef} />
      <BtnArea
        model={model}
        close={close}
        turnOffMedia={turnOffMedia}
        turnOnMedia={turnOnMedia}
      />
    </div>
  );
};

export default Main;

custom hooks로 로직을 관리하고, ui는 component화 시켜서 각각 역할을 하게 만들어봤습니다

 

src/hooks/useWebRTCHook.ts

import { useEffect, useRef, useState } from "react";
import { Socket, io } from "socket.io-client";
import util from "src/util/util";

const useWebRTCHook = () => {
  const socket = useRef<Socket>();
  const peer = useRef<RTCPeerConnection>();
  const from = useRef<string>();
  const localStream = useRef<MediaStream | null>(null);
  const candidateList = useRef<IceCandidateModel[]>([]);
  const toRef = useRef<string>();
  const connectionRef = useRef<boolean>();
  const localRef = useRef<HTMLVideoElement>(null);
  const remoteRef = useRef<HTMLVideoElement>(null);

  const [userList, setUserList] = useState<string[]>();
  const [isConnected, setIsConnected] = useState<boolean>(false);
  const [audioOnly, setAudioOnly] = useState<boolean>(false);
  const [to, setTo] = useState<string>();
  const [onMedia, setOnMedia] = useState<boolean>();
  const [isCalling, setIsCalling] = useState<boolean>(false);
  const [receivedCalling, setReceivedCalling] = useState<boolean>(false);

  toRef.current = to;
  connectionRef.current = isConnected;

  useEffect(() => {
    _initSocket();
    _initPeer();

    return () => {
      close();
      socket.current?.disconnect();
    };
  }, []);

  // [init] socket 이벤트들 등록
  const _initSocket = () => {
    console.log("[init] socket");

    console.log(import.meta);
    socket.current = io(import.meta.env.VITE_SERVER_IP);

    socket.current.on("connect", _socketConnected);
    socket.current.on("updateUserlist", _updateUserList);
    socket.current.on("connect_error", (e: any) => {
      console.log("connect error: ", { e });
    });
    socket.current.on("connect_timeout", (e: any) => {
      console.log("connect timeout: ", { e });
    });
    socket.current.on("offer", _receiveOffer);
    socket.current.on("answer", _receiveAnswer);
    socket.current.on("refuse", async () => {
      await close();
      _initPeer();
    });
    socket.current.on("remoteIceCandidate", _remotePeerIceCandidate);
    socket.current.on("disconnectPeer", async () => {
      await close();
      _initPeer();
    });
  };

  // [init] peer
  const _initPeer = () => {
    peer.current = new RTCPeerConnection({
      iceServers: [{ urls: "stun:stun.l.google.com:19302" }],
    });

    console.log("[init]", { peer });

    peer.current.onicecandidate = _iceCandidateEvent;
    peer.current.ontrack = _remoteStream;
    peer.current.onconnectionstatechange = _peerStateChange;
  };

  // peer state callback
  const _peerStateChange = (e: Event) => {
    console.log("[peer] connection state : ", peer.current!.connectionState, {
      e,
    });
    switch (peer.current!.connectionState) {
      case "connected":
        setIsConnected(true);
        break;
      case "disconnected":
        if (connectionRef.current) {
          close();
          _initPeer();
        }
        break;
      case "failed":
        peer.current!.restartIce();
        break;
      case "closed":
        if (connectionRef.current) {
          close();
          _initPeer();
        }
        break;
    }
  };

  // 본인 식별자 = 소켓 id
  const _socketConnected = () => {
    from.current = socket.current!.id;

    console.log("[socket] connected", { from });
  };

  // 초기에 connect되었을 때, 상대방 로그인/로그아웃 시 처리
  const _updateUserList = (data: { userList: string[] }) => {
    console.log("[userList] update", { data });

    const cUserList = data.userList.filter((e) => e !== from.current);

    setUserList(cUserList);
  };

  // [caller, callee] 미디어장치 연결
  const turnOnMedia = async () => {
    console.log("[init] media");

    try {
      localStream.current = await navigator.mediaDevices.getUserMedia({
        video: !audioOnly,
        audio: true,
      });

      console.log({ localStream });

      if (localRef.current !== null && localStream.current !== null) {
        localRef.current.srcObject = localStream.current;
      }

      localStream.current
        .getTracks()
        .forEach((track: MediaStreamTrack) =>
          peer.current!.addTrack(track, localStream.current!)
        );

      // 연결 되어있을때 tracking 다시 처리
      if (peer.current!.connectionState === "connected") {
        peer.current!.getSenders().forEach((sender) => {
          const track = localStream.current
            ?.getTracks()
            .find(
              (track: MediaStreamTrack) => track.kind === sender.track?.kind
            );

          if (track !== undefined) {
            sender.replaceTrack(track);
          }
        });
      }

      setOnMedia(true);
    } catch (e) {
      alert(e);
      return;
    }
  };

  // [caller, callee] 미디어장치 연결 해제
  const turnOffMedia = async () => {
    localStream.current?.getTracks().forEach((track) => {
      track.enabled = false;
      util.delay(1000).then(() => {
        track.stop();
      });
    });

    setOnMedia(false);
  };

  // [caller] : send offer to callee
  // peer : caller
  const sendOffer = async () => {
    if (to === undefined) {
      alert("유저를 선택해주세요.");
      return;
    }

    setIsCalling(true);

    console.log(peer.current?.connectionState);

    console.log("[offer] send", { to });

    await turnOnMedia();

    const offer = await peer.current!.createOffer({
      offerToReceiveAudio: true,
      offerToReceiveVideo: !audioOnly,
    });

    await peer.current!.setLocalDescription({
      sdp: offer.sdp,
      type: offer.type,
    });

    const offerModel: WebRTCModel = {
      to,
      from: from.current,
      audioOnly,
      offerSDP: offer.sdp,
      offerType: offer.type,
    };

    socket.current!.emit("offer", offerModel);
  };

  // [callee] : receive offer from caller
  // peer : callee
  const _receiveOffer = async (data: WebRTCModel) => {
    console.log("[offer] receive", { data });

    setTo(data.from);
    setAudioOnly(data.audioOnly!);
    setReceivedCalling(true);

    await turnOnMedia();

    await peer.current!.setRemoteDescription({
      sdp: data.offerSDP,
      type: data.offerType!,
    });
  };

  // [callee] : refuse offer
  const refuse = async () => {
    socket.current!.emit("refuse", { to });

    await close();
    _initPeer();
  };

  // [callee] : send answer to caller
  const sendAnswer = async () => {
    console.log("[answer] send", { to });

    const answer = await peer.current!.createAnswer({
      offerToReceiveAudio: true,
      offerToReceiveVideo: !audioOnly,
    });

    await peer.current!.setLocalDescription(answer);

    const answerModel: WebRTCModel = {
      to,
      answerSDP: answer.sdp,
      answerType: answer.type,
    };

    socket.current!.emit("answer", answerModel);

    setReceivedCalling(false);
  };

  // [caller] : receive answer from callee
  // peer : caller
  const _receiveAnswer = async (data: WebRTCModel) => {
    console.log("[answer] receive", { data });

    await peer.current!.setRemoteDescription({
      sdp: data.answerSDP,
      type: data.answerType!,
    });

    for (let candidate of candidateList.current) {
      if (!isConnected) {
        console.log("[connect] candidate", { candidate });
        socket.current!.emit("iceCandidate", {
          to: toRef.current,
          ...candidate,
        });

        setIsCalling(false);
        break;
      }
    }
  };

  // [caller] create candidates
  const _iceCandidateEvent = (e: RTCPeerConnectionIceEvent) => {
    if (e.candidate === null) {
      console.log("[iceCandidate] cut : null");
      return;
    }

    const { candidate, sdpMid, sdpMLineIndex } = e.candidate;

    console.log(e.candidate);

    if (
      toRef.current === undefined ||
      toRef.current === null ||
      candidate === undefined ||
      candidate === null
    ) {
      console.log("[iceCandidate] cut ", { toRef, candidate });
      return;
    }

    console.log("[iceCandidate] data :", { e, candidate });

    if (!isConnected) {
      const candidateModel: IceCandidateModel = {
        candidate,
        sdpMid,
        sdpMLineIndex,
      };

      candidateList.current = [...candidateList.current, candidateModel];

      console.log({ candidateList });
    }
  };

  // [callee] get candidates
  // peer : caller or callee
  const _remotePeerIceCandidate = async (data: IceCandidateModel) => {
    console.log("[remoteIceCandidate] data :", { data });

    try {
      const { candidate, sdpMid, sdpMLineIndex } = data;

      console.log("[remote]", { candidate, sdpMid, sdpMLineIndex });

      const iceCandidate = new RTCIceCandidate({
        candidate,
        sdpMid,
        sdpMLineIndex,
      });
      await peer.current!.addIceCandidate(iceCandidate);
    } catch (e) {
      console.log("[remoteIceCandidate] error ", { e });
    }
  };

  // [caller, callee] get remote media stream
  const _remoteStream = (e: RTCTrackEvent) => {
    console.log("[gotRemoteStream] data :", { e });

    const [stream] = e.streams;

    if (remoteRef.current !== null) {
      remoteRef.current!.srcObject = stream;
    }
  };

  // [caller, callee] close peer connection
  const close = async (isClicked = false) => {
    console.log("[close] peer", { peer });

    peer.current?.close();
    peer.current = undefined;

    setIsCalling(false);
    setReceivedCalling(false);
    setIsConnected(false);
    candidateList.current = [];

    await turnOffMedia();

    // caller가 전화 하는 도중 취소했을 때
    if (isClicked) {
      _initPeer();
      socket.current!.emit("disconnectPeer", { to });
    }
  };

  const model: RTCStateModel = {
    userList,
    to,
    receivedCalling,
    audioOnly,
    onMedia,
    isConnected,
    isCalling,
  };

  return {
    model,
    turnOnMedia,
    turnOffMedia,
    sendOffer,
    refuse,
    sendAnswer,
    close,
    setTo,
    setAudioOnly,
    localRef,
    remoteRef,
  };
};

export default useWebRTCHook;

먼저 상태 쪽을 보면, useRef를 굉장히 많이 사용을 하게 됐는데, 변경되면 안되는 값들

 

그리고 listener에 등록돼서 새로운 값을 못읽는 경우를 처리해주느라 사용을 해봤습니다

 

값을 하나로 모아서 object로 처리하는게 조금 더 깔끔할거 같다는 생각이 조금 들었는데 귀찮아서 패스..

 

코드가 조금 길은편이지만 하나하나씩 보면 생각보다 간단합니다

 

1. 소켓을 연결하고, 이전에 node에서 만들어놓은 이벤트들에 맞춰서 하나하나 넣어줍니다 = initSocket

2. peer connection을 초기화를 하고, 사용할 event들을 등록해줍니다 = initPeer

3. 소켓이 연결되면 유저리스트를 불러옵니다 = updateUserList

4. caller가 유저를 선택하고, 소켓 통신으로 offer를 보냅니다 = sendOffer

5. sendOffer를 보내기 위해서는 본인 media를 on해줍니다 = turnOnMedia

6. callee가 offer를 받고 answer를 만듭니다 = receiveOffer

7. offer수락을 위해 media를 On해줍니다 = turnOnMedia

7.1 거부시 이전에 변경된 peer, socket상태들을 초기화 합니다 = refuseAnswer

8. callee가 6에서 만든 answer를 소켓을 통해 caller에게 보냅니다 = sendAnswer

9. caller가 receive를 받고 이전에 생성된 iceCandidate를 callee에게 보냅니다 = receiveAnswer

10. callee가 iceCandidate를 받고, iceCandidate연결을 시도합니다 = remotePeerIceCandidate

 

이외에

turnOffMedia : media를 끕니다

iceCandidateEvent : peer가 연결을 시도할 때 이벤트가 발생하여 여러가지 경로를 찾아냅니다

여러개중 연결이 안되는 경로가 있기때문에 list에 담아놨다가 한번씩 시도를 합니다

remoteStream : peer간 연결이 완료되었을 때 비디오와 오디오를 streaming합니다

close : 소켓 및 피어, 필드등을 초기화 합니다

 

조금 더 보기좋게 표현 해보자면 다음과 같습니다

1. [공통] video connection

2. [caller] peer createOffer

3. [caller] peer setLocalDescription

4. [caller] send offer

5. [callee] receive offer

6. [callee] peer setRemoteDescription

7. [callee] peer createAnswer

8. [callee] peer setLocalDescription

9. [callee] send answer

10. [caller] receive answer

11. [caller] send iceCandidate

12. [callee] receive iceCandidate

13. [callee] peer addCandidate

 

핵심 로직들은 이렇게 마무리 하면 될거 같고

 

관련 컴포넌트는 대충(?) 만들었으니 필요하신분은 깃헙 링크를 참고해주세요

(outro 부분에 링크 있습니다)

 

web부분도 이렇게 마무리를 하면 될것같고, 다음은 flutter app입니다

 

Intro : https://devmemory.tistory.com/103
Node : https://devmemory.tistory.com/104
Flutter : https://devmemory.tistory.com/106
Outro : https://devmemory.tistory.com/107