feat: [closes #33] Render inline media (#73)

* refactor: pass inline media upload data to useRoom
* feat: render inline media
* fix: don't rescind inline media file offers
* refactor: send file offer metadata object
* fix: enable re-seeding of inline media files
* feat: show loading indicator for inline media
* feat: rescind any evicted inline media
* feat: display media rendering failure message
* feat: prevent user from uploading file if message is sending
This commit is contained in:
Jeremy Kahn 2022-11-28 21:18:41 -06:00 committed by GitHub
parent ac4cb2d7e0
commit 5d3d019cd6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 321 additions and 89 deletions

View File

@ -2,11 +2,11 @@ import { HTMLAttributes, useRef, useEffect, useState } from 'react'
import cx from 'classnames'
import Box from '@mui/material/Box'
import { Message as IMessage } from 'models/chat'
import { Message as IMessage, InlineMedia } from 'models/chat'
import { Message } from 'components/Message'
export interface ChatTranscriptProps extends HTMLAttributes<HTMLDivElement> {
messageLog: Array<IMessage>
messageLog: Array<IMessage | InlineMedia>
userId: string
}

View File

@ -0,0 +1,67 @@
import { useEffect, useRef, useState } from 'react'
import CircularProgress from '@mui/material/CircularProgress'
import { fileTransfer } from 'services/FileTransfer'
import { Typography } from '@mui/material'
type TorrentFiles = Awaited<ReturnType<typeof fileTransfer.download>>
interface InlineMediaProps {
magnetURI: string
}
interface InlineFileProps {
file: TorrentFiles[0]
}
export const InlineFile = ({ file }: InlineFileProps) => {
const containerRef = useRef(null)
const [didRenderingMediaFail, setDidRenderingMediaFail] = useState(false)
useEffect(() => {
const { current: container } = containerRef
if (!container) return
try {
file.appendTo(container)
} catch (e) {
setDidRenderingMediaFail(true)
}
}, [file, containerRef])
return (
<div ref={containerRef}>
{didRenderingMediaFail && (
<Typography sx={{ fontStyle: 'italic' }}>
Media failed to render
</Typography>
)}
</div>
)
}
export const InlineMedia = ({ magnetURI }: InlineMediaProps) => {
const [hasDownloadInitiated, setHasDownloadInitiated] = useState(false)
const [downloadedFiles, setDownloadedFiles] = useState<TorrentFiles>([])
useEffect(() => {
;(async () => {
if (hasDownloadInitiated) return
setHasDownloadInitiated(true)
const files = await fileTransfer.download(magnetURI)
setDownloadedFiles(files)
})()
}, [hasDownloadInitiated, magnetURI])
return (
<>
{hasDownloadInitiated && downloadedFiles.length === 0 ? (
<CircularProgress variant="indeterminate" color="inherit" />
) : (
downloadedFiles.map(file => <InlineFile file={file} key={file.name} />)
)}
</>
)
}

View File

@ -20,13 +20,19 @@ import { CodeProps } from 'react-markdown/lib/ast-to-react'
// @ts-ignore
import remarkGfm from 'remark-gfm'
import { Message as IMessage, isMessageReceived } from 'models/chat'
import {
InlineMedia as I_InlineMedia,
Message as IMessage,
isMessageReceived,
isInlineMedia,
} from 'models/chat'
import { PeerNameDisplay } from 'components/PeerNameDisplay'
import { InlineMedia } from './InlineMedia'
import './Message.sass'
export interface MessageProps {
message: IMessage
message: IMessage | I_InlineMedia
showAuthor: boolean
userId: string
}
@ -123,6 +129,9 @@ export const Message = ({ message, showAuthor, userId }: MessageProps) => {
}}
maxWidth="85%"
>
{isInlineMedia(message) ? (
<InlineMedia magnetURI={message.magnetURI} />
) : (
<Markdown
components={componentMap}
remarkPlugins={[remarkGfm]}
@ -130,6 +139,7 @@ export const Message = ({ message, showAuthor, userId }: MessageProps) => {
>
{message.text}
</Markdown>
)}
</Box>
</Tooltip>
</Box>

View File

@ -37,6 +37,7 @@ export function Room({
}: RoomProps) {
const {
isMessageSending,
handleInlineMediaUpload,
messageLog,
peerRoom,
roomContextValue,
@ -107,7 +108,10 @@ export function Room({
<RoomAudioControls peerRoom={peerRoom} />
<RoomVideoControls peerRoom={peerRoom} />
<RoomScreenShareControls peerRoom={peerRoom} />
<RoomFileUploadControls peerRoom={peerRoom} />
<RoomFileUploadControls
peerRoom={peerRoom}
onInlineMediaUpload={handleInlineMediaUpload}
/>
</Box>
</AccordionDetails>
</Accordion>

View File

@ -1,31 +1,38 @@
import { ChangeEventHandler, useRef } from 'react'
import { ChangeEventHandler, useContext, useRef } from 'react'
import Box from '@mui/material/Box'
import UploadFile from '@mui/icons-material/UploadFile'
import Cancel from '@mui/icons-material/Cancel'
import Fab from '@mui/material/Fab'
import Tooltip from '@mui/material/Tooltip'
import { RoomContext } from 'contexts/RoomContext'
import { PeerRoom } from 'services/PeerRoom/PeerRoom'
import { useRoomFileShare } from './useRoomFileShare'
export interface RoomFileUploadControlsProps {
onInlineMediaUpload: (files: File[]) => void
peerRoom: PeerRoom
}
export function RoomFileUploadControls({
peerRoom,
onInlineMediaUpload,
}: RoomFileUploadControlsProps) {
const roomContext = useContext(RoomContext)
const fileInputRef = useRef<HTMLInputElement>(null)
const { isMessageSending } = roomContext
const {
isFileShareButtonEnabled,
isFileSharingEnabled,
isSharingFile,
handleFileShareStart,
handleFileShareStop,
sharedFiles,
} = useRoomFileShare({
peerRoom,
onInlineMediaUpload,
})
const handleToggleScreenShareButtonClick = () => {
@ -51,6 +58,8 @@ export function RoomFileUploadControls({
const shareFileLabel =
(sharedFiles && sharedFiles.length === 1 && sharedFiles[0].name) || 'files'
const disableFileUpload = !isFileSharingEnabled || isMessageSending
return (
<Box
sx={{
@ -80,7 +89,7 @@ export function RoomFileUploadControls({
color={isSharingFile ? 'error' : 'success'}
aria-label="share screen"
onClick={handleToggleScreenShareButtonClick}
disabled={!isFileShareButtonEnabled}
disabled={disableFileUpload}
>
{isSharingFile ? <Cancel /> : <UploadFile />}
</Fab>

View File

@ -11,14 +11,20 @@ import {
Message,
ReceivedMessage,
UnsentMessage,
InlineMedia,
ReceivedInlineMedia,
UnsentInlineMedia,
VideoState,
ScreenShareState,
isMessageReceived,
isInlineMedia,
FileOfferMetadata,
} from 'models/chat'
import { getPeerName } from 'components/PeerNameDisplay'
import { NotificationService } from 'services/Notification'
import { Audio as AudioService } from 'services/Audio'
import { PeerRoom, PeerHookType } from 'services/PeerRoom'
import { fileTransfer } from 'services/FileTransfer'
import { messageTranscriptSizeLimit } from 'config/messaging'
@ -52,14 +58,30 @@ export function useRoom(
} = useContext(ShellContext)
const settingsContext = useContext(SettingsContext)
const [isMessageSending, setIsMessageSending] = useState(false)
const [messageLog, _setMessageLog] = useState<
Array<ReceivedMessage | UnsentMessage>
>([])
const [messageLog, _setMessageLog] = useState<Array<Message | InlineMedia>>(
[]
)
const [newMessageAudio] = useState(
() => new AudioService(process.env.PUBLIC_URL + '/sounds/new-message.aac')
)
const setMessageLog = (messages: Message[]) => {
const setMessageLog = (messages: Array<Message | InlineMedia>) => {
if (messages.length > messageTranscriptSizeLimit) {
const evictedMessages = messages.slice(
0,
messages.length - messageTranscriptSizeLimit
)
for (const message of evictedMessages) {
if (
isInlineMedia(message) &&
fileTransfer.isOffering(message.magnetURI)
) {
fileTransfer.rescind(message.magnetURI)
}
}
}
_setMessageLog(messages.slice(-messageTranscriptSizeLimit))
}
@ -77,12 +99,13 @@ export function useRoom(
Record<string, MediaStream>
>({})
const [peerOfferedFileIds, setPeerOfferedFileIds] = useState<
Record<string, string>
const [peerOfferedFileMetadata, setPeerOfferedFileMetadata] = useState<
Record<string, FileOfferMetadata>
>({})
const roomContextValue = useMemo(
() => ({
isMessageSending,
selfVideoStream,
setSelfVideoStream,
peerVideoStreams,
@ -91,10 +114,11 @@ export function useRoom(
setSelfScreenStream,
peerScreenStreams,
setPeerScreenStreams,
peerOfferedFileIds,
setPeerOfferedFileIds,
peerOfferedFileMetadata,
setPeerOfferedFileMetadata,
}),
[
isMessageSending,
selfVideoStream,
setSelfVideoStream,
peerVideoStreams,
@ -103,8 +127,8 @@ export function useRoom(
setSelfScreenStream,
peerScreenStreams,
setPeerScreenStreams,
peerOfferedFileIds,
setPeerOfferedFileIds,
peerOfferedFileMetadata,
setPeerOfferedFileMetadata,
]
)
@ -129,12 +153,15 @@ export function useRoom(
)
const [sendMessageTranscript, receiveMessageTranscript] = usePeerRoomAction<
ReceivedMessage[]
Array<ReceivedMessage | ReceivedInlineMedia>
>(peerRoom, PeerActions.MESSAGE_TRANSCRIPT)
const [sendPeerMessage, receivePeerMessage] =
usePeerRoomAction<UnsentMessage>(peerRoom, PeerActions.MESSAGE)
const [sendPeerInlineMedia, receivePeerInlineMedia] =
usePeerRoomAction<UnsentInlineMedia>(peerRoom, PeerActions.MEDIA_MESSAGE)
const sendMessage = async (message: string) => {
if (isMessageSending) return
@ -254,7 +281,48 @@ export function useRoom(
Object.values({ ...peerVideoStreams, ...peerScreenStreams }).length > 0
)
const handleInlineMediaUpload = async (files: File[]) => {
const fileOfferId = await fileTransfer.offer(files)
const unsentInlineMedia: UnsentInlineMedia = {
authorId: userId,
magnetURI: fileOfferId,
timeSent: Date.now(),
id: getUuid(),
}
setIsMessageSending(true)
setMessageLog([...messageLog, unsentInlineMedia])
await sendPeerInlineMedia(unsentInlineMedia)
setMessageLog([
...messageLog,
{ ...unsentInlineMedia, timeReceived: Date.now() },
])
setIsMessageSending(false)
}
receivePeerInlineMedia(inlineMedia => {
const userSettings = settingsContext.getUserSettings()
if (!tabHasFocus) {
if (userSettings.playSoundOnNewMessage) {
newMessageAudio.play()
}
if (userSettings.showNotificationOnNewMessage) {
NotificationService.showNotification(
`${getPeerName(inlineMedia.authorId)} shared media`
)
}
}
setMessageLog([...messageLog, { ...inlineMedia, timeReceived: Date.now() }])
})
return {
handleInlineMediaUpload,
isMessageSending,
messageLog,
peerRoom,

View File

@ -4,7 +4,7 @@ import { sleep } from 'utils'
import { RoomContext } from 'contexts/RoomContext'
import { ShellContext } from 'contexts/ShellContext'
import { PeerActions } from 'models/network'
import { Peer } from 'models/chat'
import { FileOfferMetadata, Peer } from 'models/chat'
import { PeerRoom, PeerHookType } from 'services/PeerRoom'
import { fileTransfer } from 'services/FileTransfer/index'
@ -12,44 +12,61 @@ import { fileTransfer } from 'services/FileTransfer/index'
import { usePeerRoomAction } from './usePeerRoomAction'
interface UseRoomFileShareConfig {
onInlineMediaUpload: (files: File[]) => void
peerRoom: PeerRoom
}
export function useRoomFileShare({ peerRoom }: UseRoomFileShareConfig) {
const isInlineMediaFile = (file: File) => {
return ['image', 'audio', 'video'].includes(file.type.split('/')[0])
}
export function useRoomFileShare({
onInlineMediaUpload,
peerRoom,
}: UseRoomFileShareConfig) {
const shellContext = useContext(ShellContext)
const roomContext = useContext(RoomContext)
const [sharedFiles, setSharedFiles] = useState<FileList | null>(null)
const [selfFileOfferId, setFileOfferId] = useState<string | null>(null)
const [isFileShareButtonEnabled, setIsFileShareButtonEnabled] = useState(true)
const [selfFileOfferMagnetUri, setFileOfferMagnetUri] = useState<
string | null
>(null)
const [isFileSharingEnabled, setIsFileSharingEnabled] = useState(true)
const { peerList, setPeerList } = shellContext
const { peerOfferedFileIds, setPeerOfferedFileIds } = roomContext
const { peerOfferedFileMetadata, setPeerOfferedFileMetadata } = roomContext
const [sendFileOfferId, receiveFileOfferId] = usePeerRoomAction<
string | null
>(peerRoom, PeerActions.FILE_OFFER)
const [sendFileOfferMetadata, receiveFileOfferMetadata] =
usePeerRoomAction<FileOfferMetadata | null>(
peerRoom,
PeerActions.FILE_OFFER
)
receiveFileOfferId((fileOfferId, peerId) => {
if (fileOfferId) {
setPeerOfferedFileIds({ [peerId]: fileOfferId })
receiveFileOfferMetadata((fileOfferMetadata, peerId) => {
if (fileOfferMetadata) {
setPeerOfferedFileMetadata({ [peerId]: fileOfferMetadata })
} else {
const fileOfferId = peerOfferedFileIds[peerId]
const fileOfferMetadata = peerOfferedFileMetadata[peerId]
const { magnetURI, isAllInlineMedia } = fileOfferMetadata
if (fileOfferId && fileTransfer.isOffering(fileOfferId)) {
fileTransfer.rescind(fileOfferId)
if (
fileOfferMetadata &&
fileTransfer.isOffering(magnetURI) &&
!isAllInlineMedia
) {
fileTransfer.rescind(magnetURI)
}
const newFileOfferIds = { ...peerOfferedFileIds }
delete newFileOfferIds[peerId]
const newFileOfferMetadata = { ...peerOfferedFileMetadata }
delete newFileOfferMetadata[peerId]
setPeerOfferedFileIds(newFileOfferIds)
setPeerOfferedFileMetadata(newFileOfferMetadata)
}
const newPeerList = peerList.map(peer => {
const newPeer: Peer = { ...peer }
if (peer.peerId === peerId) {
newPeer.offeredFileId = fileOfferId
newPeer.offeredFileId = fileOfferMetadata?.magnetURI ?? null
}
return newPeer
@ -58,68 +75,93 @@ export function useRoomFileShare({ peerRoom }: UseRoomFileShareConfig) {
setPeerList(newPeerList)
})
const isEveryFileInlineMedia = (files: FileList | null) =>
Boolean(files && [...files].every(isInlineMediaFile))
peerRoom.onPeerJoin(PeerHookType.FILE_SHARE, async (peerId: string) => {
if (!selfFileOfferId) return
if (!selfFileOfferMagnetUri) return
// This sleep is needed to prevent this peer from not appearing on other
// peers' peer lists. This is because Trystero's interaction between
// onPeerJoin and its actions is not totally compatible with React's
// lifecycle hooks. In this case, the reference to peerList in
// receiveFileOfferId is out of date and prevents this peer from ever being
// added to the receiver's peer list. Deferring the sendFileOfferId call to
// the next tick serves as a workaround.
// receiveFileOfferMetadata is out of date and prevents this peer from ever
// being added to the receiver's peer list. Deferring the
// sendFileOfferMetadata call to the next tick serves as a workaround.
await sleep(1)
sendFileOfferId(selfFileOfferId, peerId)
sendFileOfferMetadata(
{
magnetURI: selfFileOfferMagnetUri,
isAllInlineMedia: isEveryFileInlineMedia(sharedFiles),
},
peerId
)
})
peerRoom.onPeerLeave(PeerHookType.FILE_SHARE, (peerId: string) => {
const fileOfferId = peerOfferedFileIds[peerId]
const fileOfferMetadata = peerOfferedFileMetadata[peerId]
if (!fileOfferId) return
if (!fileOfferMetadata) return
if (fileTransfer.isOffering(fileOfferId)) {
fileTransfer.rescind(fileOfferId)
const { magnetURI, isAllInlineMedia } = fileOfferMetadata
if (fileTransfer.isOffering(magnetURI) && !isAllInlineMedia) {
fileTransfer.rescind(magnetURI)
}
const newPeerFileOfferIds = { ...peerOfferedFileIds }
delete newPeerFileOfferIds[peerId]
setPeerOfferedFileIds(newPeerFileOfferIds)
const newPeerFileOfferMetadata = { ...peerOfferedFileMetadata }
delete newPeerFileOfferMetadata[peerId]
setPeerOfferedFileMetadata(newPeerFileOfferMetadata)
})
const handleFileShareStart = async (files: FileList) => {
const inlineMediaFiles = [...files].filter(isInlineMediaFile)
setSharedFiles(files)
setIsFileShareButtonEnabled(false)
setIsFileSharingEnabled(false)
const fileOfferId = await fileTransfer.offer(files)
sendFileOfferId(fileOfferId)
setFileOfferId(fileOfferId)
const magnetURI = await fileTransfer.offer(files)
setIsFileShareButtonEnabled(true)
if (inlineMediaFiles.length > 0) {
onInlineMediaUpload(inlineMediaFiles)
}
sendFileOfferMetadata({
magnetURI,
isAllInlineMedia: isEveryFileInlineMedia(files),
})
setFileOfferMagnetUri(magnetURI)
setIsFileSharingEnabled(true)
}
const handleFileShareStop = () => {
sendFileOfferId(null)
setFileOfferId(null)
sendFileOfferMetadata(null)
setFileOfferMagnetUri(null)
if (selfFileOfferId && fileTransfer.isOffering(selfFileOfferId)) {
fileTransfer.rescind(selfFileOfferId)
if (
selfFileOfferMagnetUri &&
fileTransfer.isOffering(selfFileOfferMagnetUri) &&
!isEveryFileInlineMedia(sharedFiles)
) {
fileTransfer.rescind(selfFileOfferMagnetUri)
}
}
useEffect(() => {
return () => {
fileTransfer.rescindAll()
sendFileOfferId(null)
sendFileOfferMetadata(null)
}
}, [sendFileOfferId])
}, [sendFileOfferMetadata])
const isSharingFile = Boolean(selfFileOfferId)
const isSharingFile = Boolean(selfFileOfferMagnetUri)
return {
handleFileShareStart,
handleFileShareStop,
isFileShareButtonEnabled,
isFileSharingEnabled,
isSharingFile,
sharedFiles,
}

View File

@ -38,7 +38,7 @@ export const PeerDownloadFileButton = ({
setDownloadProgress(null)
try {
await fileTransfer.download(offeredFileId, { onProgress })
await fileTransfer.download(offeredFileId, { doSave: true, onProgress })
} catch (e) {
if (isError(e)) {
shellContext.showAlert(e.message, {

View File

@ -1,6 +1,8 @@
import { createContext, Dispatch, SetStateAction } from 'react'
import { FileOfferMetadata } from 'models/chat'
interface RoomContextProps {
isMessageSending: boolean
selfVideoStream: MediaStream | null
setSelfVideoStream: Dispatch<SetStateAction<MediaStream | null>>
peerVideoStreams: Record<string, MediaStream>
@ -9,11 +11,14 @@ interface RoomContextProps {
setSelfScreenStream: Dispatch<SetStateAction<MediaStream | null>>
peerScreenStreams: Record<string, MediaStream>
setPeerScreenStreams: Dispatch<SetStateAction<Record<string, MediaStream>>>
peerOfferedFileIds: Record<string, string>
setPeerOfferedFileIds: Dispatch<SetStateAction<Record<string, string>>>
peerOfferedFileMetadata: Record<string, FileOfferMetadata>
setPeerOfferedFileMetadata: Dispatch<
SetStateAction<Record<string, FileOfferMetadata>>
>
}
export const RoomContext = createContext<RoomContextProps>({
isMessageSending: false,
selfVideoStream: null,
setSelfVideoStream: () => {},
peerVideoStreams: {},
@ -22,6 +27,6 @@ export const RoomContext = createContext<RoomContextProps>({
setSelfScreenStream: () => {},
peerScreenStreams: {},
setPeerScreenStreams: () => {},
peerOfferedFileIds: {},
setPeerOfferedFileIds: () => {},
peerOfferedFileMetadata: {},
setPeerOfferedFileMetadata: () => {},
})

View File

@ -5,6 +5,22 @@ export interface UnsentMessage {
authorId: string
}
export interface ReceivedMessage extends UnsentMessage {
timeReceived: number
}
export type Message = UnsentMessage | ReceivedMessage
export interface UnsentInlineMedia extends Omit<UnsentMessage, 'text'> {
magnetURI: string
}
export interface ReceivedInlineMedia extends UnsentInlineMedia {
timeReceived: number
}
export type InlineMedia = UnsentInlineMedia | ReceivedInlineMedia
export enum AudioState {
PLAYING = 'PLAYING',
STOPPED = 'STOPPED',
@ -34,12 +50,17 @@ export interface Peer {
offeredFileId: string | null
}
export interface ReceivedMessage extends UnsentMessage {
timeReceived: number
export const isMessageReceived = (
message: Message | InlineMedia
): message is ReceivedMessage | ReceivedInlineMedia => 'timeReceived' in message
export const isInlineMedia = (
message: Message | InlineMedia
): message is InlineMedia => {
return 'magnetURI' in message
}
export const isMessageReceived = (
message: Message
): message is ReceivedMessage => 'timeReceived' in message
export type Message = UnsentMessage | ReceivedMessage
export interface FileOfferMetadata {
magnetURI: string
isAllInlineMedia: boolean
}

View File

@ -1,6 +1,7 @@
// NOTE: Action names are limited to 12 characters, otherwise Trystero breaks.
export enum PeerActions {
MESSAGE = 'MESSAGE',
MEDIA_MESSAGE = 'MEDIA_MSG',
MESSAGE_TRANSCRIPT = 'MSG_XSCRIPT',
PEER_NAME = 'PEER_NAME',
AUDIO_CHANGE = 'AUDIO_CHANGE',

View File

@ -10,6 +10,7 @@ import { streamSaverUrl } from 'config/streamSaverUrl'
streamSaver.mitm = streamSaverUrl
interface DownloadOpts {
doSave?: boolean
onProgress?: (progress: number) => void
}
@ -72,7 +73,7 @@ export class FileTransfer {
window.addEventListener('beforeunload', this.handleBeforePageUnload)
}
async download(magnetURI: string, { onProgress }: DownloadOpts = {}) {
async download(magnetURI: string, { onProgress, doSave }: DownloadOpts = {}) {
let torrent = this.torrents[magnetURI]
if (!torrent) {
@ -104,6 +105,7 @@ export class FileTransfer {
torrent.on('download', handleDownload)
if (doSave) {
try {
await this.saveTorrentFiles(torrent)
} catch (e) {
@ -114,7 +116,10 @@ export class FileTransfer {
}
}
async offer(files: FileList) {
return torrent.files
}
async offer(files: Parameters<typeof this.webTorrentClient.seed>[0]) {
const { isPrivate } = await detectIncognito()
const torrent = await new Promise<Torrent>(res => {