From 828e3c12b96db5469a8f3ac059a7fa9d5d051c7b Mon Sep 17 00:00:00 2001 From: Jeremy Kahn Date: Sun, 6 Nov 2022 13:36:15 -0600 Subject: [PATCH] feat: [closes #20] Video support (#64) * refactor: nest ChatTranscript * feat: set up video controls * feat: show self video * feat: show peer video * feat: improve audio/video controls display * feat: flip self video * feat: improve device selection labels --- src/components/Room/PeerVideo.tsx | 52 +++++ src/components/Room/Room.tsx | 81 +++++--- src/components/Room/RoomAudioControls.tsx | 50 ++--- src/components/Room/RoomVideoControls.tsx | 121 ++++++++++++ src/components/Room/RoomVideoDisplay.tsx | 64 ++++++ src/components/Room/useRoom.ts | 21 +- src/components/Room/useRoomAudio.ts | 4 + src/components/Room/useRoomVideo.ts | 230 ++++++++++++++++++++++ src/components/Shell/Shell.tsx | 21 +- src/contexts/ShellContext.ts | 14 +- src/models/chat.ts | 6 + src/models/network.ts | 1 + src/services/PeerRoom/PeerRoom.ts | 2 + 13 files changed, 606 insertions(+), 61 deletions(-) create mode 100644 src/components/Room/PeerVideo.tsx create mode 100644 src/components/Room/RoomVideoControls.tsx create mode 100644 src/components/Room/RoomVideoDisplay.tsx create mode 100644 src/components/Room/useRoomVideo.ts diff --git a/src/components/Room/PeerVideo.tsx b/src/components/Room/PeerVideo.tsx new file mode 100644 index 0000000..f6d9231 --- /dev/null +++ b/src/components/Room/PeerVideo.tsx @@ -0,0 +1,52 @@ +import { useEffect, useRef } from 'react' +import Paper from '@mui/material/Paper' + +import { PeerNameDisplay } from 'components/PeerNameDisplay' + +interface PeerVideoProps { + isSelf?: boolean + userId: string + videoStream: MediaStream +} + +export const PeerVideo = ({ isSelf, userId, videoStream }: PeerVideoProps) => { + const videoRef = useRef(null) + + useEffect(() => { + const { current: video } = videoRef + if (!video) return + + video.autoplay = true + video.srcObject = videoStream + }, [videoRef, videoStream]) + + return ( + + + ) +} diff --git a/src/components/Room/Room.tsx b/src/components/Room/Room.tsx index 191bb0a..be3f4ec 100644 --- a/src/components/Room/Room.tsx +++ b/src/components/Room/Room.tsx @@ -13,6 +13,8 @@ import { ChatTranscript } from 'components/ChatTranscript' import { useRoom } from './useRoom' import { RoomAudioControls } from './RoomAudioControls' +import { RoomVideoControls } from './RoomVideoControls' +import { RoomVideoDisplay } from './RoomVideoDisplay' export interface RoomProps { appId?: string @@ -29,7 +31,13 @@ export function Room({ password, userId, }: RoomProps) { - const { messageLog, peerRoom, sendMessage, isMessageSending } = useRoom( + const { + isMessageSending, + messageLog, + peerRoom, + sendMessage, + showVideoDisplay, + } = useRoom( { appId, trackerUrls, @@ -53,38 +61,49 @@ export function Room({ sx={{ height: '100%', display: 'flex', - flexDirection: 'column', + flexGrow: '1', + overflow: 'auto', }} > - - } - aria-controls="panel1a-content" - id="panel1a-header" - > - - - - - - - - - + {showVideoDisplay && } + + + } + aria-controls="panel1a-content" + id="panel1a-header" + > + + + + + + + + + + + ) } diff --git a/src/components/Room/RoomAudioControls.tsx b/src/components/Room/RoomAudioControls.tsx index db810bb..1805291 100644 --- a/src/components/Room/RoomAudioControls.tsx +++ b/src/components/Room/RoomAudioControls.tsx @@ -8,6 +8,7 @@ import ListItemText from '@mui/material/ListItemText' import Menu from '@mui/material/Menu' import MenuItem from '@mui/material/MenuItem' import Fab from '@mui/material/Fab' +import Tooltip from '@mui/material/Tooltip' import { PeerRoom } from 'services/PeerRoom/PeerRoom' @@ -53,30 +54,35 @@ export function RoomAudioControls({ peerRoom }: RoomAudioControlsProps) { } return ( - <> - + - {isSpeakingToRoom ? ( - <> - - Stop speaking to room - - ) : ( - <> - - Start speaking to room - - )} - + + {isSpeakingToRoom ? : } + + {audioDevices.length > 0 && ( @@ -116,6 +122,6 @@ export function RoomAudioControls({ peerRoom }: RoomAudioControlsProps) { )} - + ) } diff --git a/src/components/Room/RoomVideoControls.tsx b/src/components/Room/RoomVideoControls.tsx new file mode 100644 index 0000000..613beae --- /dev/null +++ b/src/components/Room/RoomVideoControls.tsx @@ -0,0 +1,121 @@ +import { useState } from 'react' +import Box from '@mui/material/Box' +import Videocam from '@mui/icons-material/Videocam' +import VideocamOff from '@mui/icons-material/VideocamOff' +import List from '@mui/material/List' +import ListItem from '@mui/material/ListItem' +import ListItemText from '@mui/material/ListItemText' +import Menu from '@mui/material/Menu' +import MenuItem from '@mui/material/MenuItem' +import Fab from '@mui/material/Fab' +import Tooltip from '@mui/material/Tooltip' + +import { PeerRoom } from 'services/PeerRoom/PeerRoom' + +import { useRoomVideo } from './useRoomVideo' + +export interface RoomVideoControlsProps { + peerRoom: PeerRoom +} + +export function RoomVideoControls({ peerRoom }: RoomVideoControlsProps) { + const { + videoDevices, + isCameraEnabled, + setIsCameraEnabled, + handleVideoDeviceSelect, + } = useRoomVideo({ peerRoom }) + + const [videoAnchorEl, setVideoAnchorEl] = useState(null) + const isVideoDeviceSelectOpen = Boolean(videoAnchorEl) + const [selectedVideoDeviceIdx, setSelectedVideoDeviceIdx] = useState(0) + + const handleEnableCameraClick = () => { + setIsCameraEnabled(!isCameraEnabled) + } + + const handleVideoDeviceListItemClick = ( + event: React.MouseEvent + ) => { + setVideoAnchorEl(event.currentTarget) + } + + const handleVideoDeviceMenuItemClick = ( + _event: React.MouseEvent, + idx: number + ) => { + setSelectedVideoDeviceIdx(idx) + handleVideoDeviceSelect(videoDevices[idx]) + setVideoAnchorEl(null) + } + + const handleVideoInputSelectMenuClose = () => { + setVideoAnchorEl(null) + } + + return ( + + + + {isCameraEnabled ? : } + + + {videoDevices.length > 0 && ( + + + + + + + + {videoDevices.map((videoDevice, idx) => ( + handleVideoDeviceMenuItemClick(event, idx)} + > + {videoDevice.label} + + ))} + + + )} + + ) +} diff --git a/src/components/Room/RoomVideoDisplay.tsx b/src/components/Room/RoomVideoDisplay.tsx new file mode 100644 index 0000000..581d536 --- /dev/null +++ b/src/components/Room/RoomVideoDisplay.tsx @@ -0,0 +1,64 @@ +import { useContext } from 'react' +import Paper from '@mui/material/Paper' + +import { Peer } from 'models/chat' +import { ShellContext } from 'contexts/ShellContext' + +import { PeerVideo } from './PeerVideo' + +type PeerWithVideo = { peer: Peer; videoStream: MediaStream } + +export interface RoomVideoDisplayProps { + userId: string +} + +export const RoomVideoDisplay = ({ userId }: RoomVideoDisplayProps) => { + const shellContext = useContext(ShellContext) + + const peersWithVideo: PeerWithVideo[] = shellContext.peerList.reduce( + (acc: PeerWithVideo[], peer: Peer) => { + const videoStream = shellContext.peerVideoStreams[peer.peerId] + if (videoStream) { + acc.push({ + peer, + videoStream, + }) + } + + return acc + }, + [] + ) + + return ( + + {shellContext.selfVideoStream && ( + + )} + {peersWithVideo.map(peerWithVideo => ( + + ))} + + ) +} diff --git a/src/components/Room/useRoom.ts b/src/components/Room/useRoom.ts index f5298ab..87ec59c 100644 --- a/src/components/Room/useRoom.ts +++ b/src/components/Room/useRoom.ts @@ -11,6 +11,7 @@ import { Message, ReceivedMessage, UnsentMessage, + VideoState, isMessageReceived, } from 'models/chat' import { funAnimalName } from 'fun-animal-names' @@ -39,8 +40,6 @@ export function useRoom( () => new PeerRoom({ password: password ?? roomId, ...roomConfig }, roomId) ) - peerRoom.flush() - const [numberOfPeers, setNumberOfPeers] = useState(1) // Includes this peer const shellContext = useContext(ShellContext) const settingsContext = useContext(SettingsContext) @@ -110,7 +109,12 @@ export function useRoom( if (peerIndex === -1) { shellContext.setPeerList([ ...shellContext.peerList, - { peerId: peerId, userId: userId, audioState: AudioState.STOPPED }, + { + peerId, + userId, + audioState: AudioState.STOPPED, + videoState: VideoState.STOPPED, + }, ]) } else { const newPeerList = [...shellContext.peerList] @@ -195,10 +199,15 @@ export function useRoom( } }) + const showVideoDisplay = + shellContext.selfVideoStream || + Object.values(shellContext.peerVideoStreams).length > 0 + return { - peerRoom, - messageLog, - sendMessage, isMessageSending, + messageLog, + peerRoom, + sendMessage, + showVideoDisplay, } } diff --git a/src/components/Room/useRoomAudio.ts b/src/components/Room/useRoomAudio.ts index 9a657ec..c32b9d6 100644 --- a/src/components/Room/useRoomAudio.ts +++ b/src/components/Room/useRoomAudio.ts @@ -58,6 +58,10 @@ export function useRoomAudio({ peerRoom }: UseRoomAudioConfig) { }) peerRoom.onPeerStream(PeerStreamType.AUDIO, (stream, peerId) => { + const audioTracks = stream.getAudioTracks() + + if (audioTracks.length === 0) return + const audio = new Audio() audio.srcObject = stream audio.autoplay = true diff --git a/src/components/Room/useRoomVideo.ts b/src/components/Room/useRoomVideo.ts new file mode 100644 index 0000000..2aa570c --- /dev/null +++ b/src/components/Room/useRoomVideo.ts @@ -0,0 +1,230 @@ +import { useContext, useEffect, useCallback, useState } from 'react' + +import { ShellContext } from 'contexts/ShellContext' +import { PeerActions } from 'models/network' +import { VideoState, Peer } from 'models/chat' +import { PeerRoom, PeerHookType, PeerStreamType } from 'services/PeerRoom' + +import { usePeerRoomAction } from './usePeerRoomAction' + +interface UseRoomVideoConfig { + peerRoom: PeerRoom +} + +export function useRoomVideo({ peerRoom }: UseRoomVideoConfig) { + const shellContext = useContext(ShellContext) + const [isCameraEnabled, setIsCameraEnabled] = useState(false) + const [videoDevices, setVideoDevices] = useState([]) + const [selectedVideoDeviceId, setSelectedVideoDeviceId] = useState< + string | null + >(null) + + const { + peerList, + peerVideoStreams, + selfVideoStream, + setPeerList, + setPeerVideoStreams, + setSelfVideoStream, + setVideoState, + } = shellContext + + useEffect(() => { + ;(async () => { + if (!selfVideoStream) return + + const devices = await window.navigator.mediaDevices.enumerateDevices() + const rawVideoDevices = devices.filter( + ({ kind }) => kind === 'videoinput' + ) + + // Sometimes duplicate devices are provided by enumerateDevices, so + // dedupe them by ID. + const newVideoDevices = [ + ...rawVideoDevices + .reduce((acc, videoDevice) => { + return acc.set(videoDevice.deviceId, videoDevice) + }, new Map()) + .values(), + ] + + setVideoDevices(newVideoDevices) + + if (newVideoDevices.length > 0 && !selfVideoStream) { + const [firstVideoDevice] = newVideoDevices + const newSelfStream = await navigator.mediaDevices.getUserMedia({ + audio: false, + video: { + deviceId: firstVideoDevice.deviceId, + }, + }) + + peerRoom.addStream(newSelfStream) + setSelfVideoStream(newSelfStream) + + setSelfVideoStream(newSelfStream) + } + })() + }, [peerRoom, selfVideoStream, setSelfVideoStream]) + + const [sendVideoChange, receiveVideoChange] = usePeerRoomAction( + peerRoom, + PeerActions.VIDEO_CHANGE + ) + + receiveVideoChange((videoState, peerId) => { + const newPeerList = peerList.map(peer => { + const newPeer: Peer = { ...peer } + + if (peer.peerId === peerId) { + newPeer.videoState = videoState + + if (videoState === VideoState.STOPPED) { + deletePeerVideo(peerId) + } + } + + return newPeer + }) + + setPeerList(newPeerList) + }) + + peerRoom.onPeerStream(PeerStreamType.VIDEO, (stream, peerId) => { + const videoTracks = stream.getVideoTracks() + + if (videoTracks.length === 0) return + + setPeerVideoStreams({ + ...peerVideoStreams, + [peerId]: stream, + }) + }) + + const cleanupVideo = useCallback(() => { + if (!selfVideoStream) return + + for (const videoTrack of selfVideoStream.getTracks()) { + videoTrack.stop() + selfVideoStream.removeTrack(videoTrack) + } + }, [selfVideoStream]) + + useEffect(() => { + ;(async () => { + if (isCameraEnabled) { + if (!selfVideoStream) { + const newSelfStream = await navigator.mediaDevices.getUserMedia({ + audio: false, + video: selectedVideoDeviceId + ? { deviceId: selectedVideoDeviceId } + : true, + }) + + peerRoom.addStream(newSelfStream) + sendVideoChange(VideoState.PLAYING) + setVideoState(VideoState.PLAYING) + setSelfVideoStream(newSelfStream) + } + } else { + if (selfVideoStream) { + cleanupVideo() + + peerRoom.removeStream(selfVideoStream, peerRoom.getPeers()) + sendVideoChange(VideoState.STOPPED) + setVideoState(VideoState.STOPPED) + setSelfVideoStream(null) + setSelfVideoStream(null) + } + } + })() + }, [ + isCameraEnabled, + peerRoom, + selfVideoStream, + selectedVideoDeviceId, + sendVideoChange, + cleanupVideo, + setSelfVideoStream, + setVideoState, + ]) + + useEffect(() => { + return () => { + cleanupVideo() + } + }, [cleanupVideo]) + + useEffect(() => { + return () => { + if (selfVideoStream) { + setSelfVideoStream(null) + setVideoState(VideoState.STOPPED) + } + } + }, [selfVideoStream, setSelfVideoStream, setVideoState]) + + useEffect(() => { + return () => { + setPeerVideoStreams({}) + } + }, [setPeerVideoStreams]) + + const handleVideoDeviceSelect = async (videoDevice: MediaDeviceInfo) => { + const { deviceId } = videoDevice + setSelectedVideoDeviceId(deviceId) + + if (!selfVideoStream) return + + for (const videoTrack of selfVideoStream.getTracks()) { + videoTrack.stop() + selfVideoStream.removeTrack(videoTrack) + } + + peerRoom.removeStream(selfVideoStream, peerRoom.getPeers()) + + const newSelfStream = await navigator.mediaDevices.getUserMedia({ + audio: false, + video: { + deviceId, + }, + }) + + peerRoom.addStream(newSelfStream) + setSelfVideoStream(newSelfStream) + } + + const deletePeerVideo = (peerId: string) => { + const newPeerVideos = { ...peerVideoStreams } + delete newPeerVideos[peerId] + setPeerVideoStreams(newPeerVideos) + } + + const handleVideoForNewPeer = (peerId: string) => { + if (selfVideoStream) { + peerRoom.addStream(selfVideoStream, peerId) + } + } + + const handleVideoForLeavingPeer = (peerId: string) => { + if (selfVideoStream) { + peerRoom.removeStream(selfVideoStream, peerId) + deletePeerVideo(peerId) + } + } + + peerRoom.onPeerJoin(PeerHookType.VIDEO, (peerId: string) => { + handleVideoForNewPeer(peerId) + }) + + peerRoom.onPeerLeave(PeerHookType.VIDEO, (peerId: string) => { + handleVideoForLeavingPeer(peerId) + }) + + return { + videoDevices, + isCameraEnabled, + setIsCameraEnabled, + handleVideoDeviceSelect, + } +} diff --git a/src/components/Shell/Shell.tsx b/src/components/Shell/Shell.tsx index be2a3a7..554957b 100644 --- a/src/components/Shell/Shell.tsx +++ b/src/components/Shell/Shell.tsx @@ -15,7 +15,7 @@ import { AlertColor } from '@mui/material/Alert' import { ShellContext } from 'contexts/ShellContext' import { SettingsContext } from 'contexts/SettingsContext' import { AlertOptions } from 'models/shell' -import { AudioState, Peer } from 'models/chat' +import { AudioState, VideoState, Peer } from 'models/chat' import { ErrorBoundary } from 'components/ErrorBoundary' import { Drawer } from './Drawer' @@ -45,6 +45,13 @@ export const Shell = ({ appNeedsUpdate, children, userPeerId }: ShellProps) => { const [peerList, setPeerList] = useState([]) // except you const [tabHasFocus, setTabHasFocus] = useState(true) const [audioState, setAudioState] = useState(AudioState.STOPPED) + const [videoState, setVideoState] = useState(VideoState.STOPPED) + const [selfVideoStream, setSelfVideoStream] = useState( + null + ) + const [peerVideoStreams, setPeerVideoStreams] = useState< + Record + >({}) const showAlert = useCallback< (message: string, options?: AlertOptions) => void @@ -69,6 +76,12 @@ export const Shell = ({ appNeedsUpdate, children, userPeerId }: ShellProps) => { setPeerList, audioState, setAudioState, + videoState, + setVideoState, + selfVideoStream, + setSelfVideoStream, + peerVideoStreams, + setPeerVideoStreams, }), [ isPeerListOpen, @@ -82,6 +95,12 @@ export const Shell = ({ appNeedsUpdate, children, userPeerId }: ShellProps) => { showAlert, audioState, setAudioState, + videoState, + setVideoState, + selfVideoStream, + setSelfVideoStream, + peerVideoStreams, + setPeerVideoStreams, ] ) diff --git a/src/contexts/ShellContext.ts b/src/contexts/ShellContext.ts index 34efa6a..3c777d2 100644 --- a/src/contexts/ShellContext.ts +++ b/src/contexts/ShellContext.ts @@ -1,7 +1,7 @@ import { createContext, Dispatch, SetStateAction } from 'react' import { AlertOptions } from 'models/shell' -import { AudioState, Peer } from 'models/chat' +import { AudioState, VideoState, Peer } from 'models/chat' interface ShellContextProps { numberOfPeers: number @@ -16,6 +16,12 @@ interface ShellContextProps { setPeerList: Dispatch> audioState: AudioState setAudioState: Dispatch> + videoState: VideoState + setVideoState: Dispatch> + selfVideoStream: MediaStream | null + setSelfVideoStream: Dispatch> + peerVideoStreams: Record + setPeerVideoStreams: Dispatch>> } export const ShellContext = createContext({ @@ -31,4 +37,10 @@ export const ShellContext = createContext({ setPeerList: () => {}, audioState: AudioState.STOPPED, setAudioState: () => {}, + videoState: VideoState.STOPPED, + setVideoState: () => {}, + selfVideoStream: null, + setSelfVideoStream: () => {}, + peerVideoStreams: {}, + setPeerVideoStreams: () => {}, }) diff --git a/src/models/chat.ts b/src/models/chat.ts index 793ad5b..a0812ab 100644 --- a/src/models/chat.ts +++ b/src/models/chat.ts @@ -10,10 +10,16 @@ export enum AudioState { STOPPED = 'STOPPED', } +export enum VideoState { + PLAYING = 'PLAYING', + STOPPED = 'STOPPED', +} + export interface Peer { peerId: string userId: string audioState: AudioState + videoState: VideoState } export interface ReceivedMessage extends UnsentMessage { diff --git a/src/models/network.ts b/src/models/network.ts index 92576a1..1ce0f99 100644 --- a/src/models/network.ts +++ b/src/models/network.ts @@ -4,4 +4,5 @@ export enum PeerActions { MESSAGE_TRANSCRIPT = 'MSG_XSCRIPT', PEER_NAME = 'PEER_NAME', AUDIO_CHANGE = 'AUDIO_CHANGE', + VIDEO_CHANGE = 'VIDEO_CHANGE', } diff --git a/src/services/PeerRoom/PeerRoom.ts b/src/services/PeerRoom/PeerRoom.ts index 75c335d..a7c157a 100644 --- a/src/services/PeerRoom/PeerRoom.ts +++ b/src/services/PeerRoom/PeerRoom.ts @@ -4,10 +4,12 @@ import { TorrentRoomConfig } from 'trystero/torrent' export enum PeerHookType { NEW_PEER = 'NEW_PEER', AUDIO = 'AUDIO', + VIDEO = 'VIDEO', } export enum PeerStreamType { AUDIO = 'AUDIO', + VIDEO = 'VIDEO', } export class PeerRoom {