diff --git a/README.md b/README.md index 2886218..a4f13e7 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ mumh5 keeps the foundation and replaces the experience. - Share a screen or a window with the mumh5 users in your channel. A dialog lets you choose what to show, with a live preview, which sound goes with it and the quality, before anything is sent - Sound: on Linux one program or everything except mumh5 itself (through PipeWire, so viewers do not hear the voice chat twice); on Windows the whole system; in a browser what the browser offers -- People in the channel see an indicator next to your name, on your tile and in your profile, and click to watch. Your tile shows the picture; a click makes it large with everyone else in a strip below, like a meeting. Viewers set the stream's volume +- People in the channel see an indicator next to your name, on your tile and in your profile, and click to watch. The tiles above the chat fill a stage you can drag taller or shorter. A stream opens large, with everyone as a strip of tiles below, like a meeting, and can move to a window of its own; in the stacked layout it is just the picture, as wide as the chat. Sounds announce streams and viewers. Viewers set the stream's volume - No server setup and no extra account: the setup messages travel through the Mumble server, the stream goes directly between the two clients (WebRTC), up to 8 viewers. Regular Mumble clients do not see streams - Direct connections mean sharer and viewer see each other's IP address; mumh5 says so before the first use. Across the internet both sides need a STUN server. The browser version uses the one built into its proxy; in the desktop app you enter one in Settings, Voice (the proxy's address works there too), and none is contacted unless you do - Tested between two desktop instances on one machine with a test picture. Sound capture, real screens, connections across the internet, Windows and the browser build are untested diff --git a/electron/main.ts b/electron/main.ts index 2c5465e..8c12706 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -201,7 +201,11 @@ function createWindow() { }); // Links from chat open in the system browser, never inside the app - win.webContents.setWindowOpenHandler(({ url }) => { + win.webContents.setWindowOpenHandler(({ url, frameName }) => { + // The stream window: an empty page the app fills with the video itself + if (frameName === 'mumh5-stream' && (url === 'about:blank' || url === '')) { + return { action: 'allow', overrideBrowserWindowOptions: { width: 960, height: 600, minWidth: 320, minHeight: 200, backgroundColor: '#000000', autoHideMenuBar: true, title: 'mumh5' } }; + } if (/^https?:\/\//i.test(url)) shell.openExternal(url); return { action: 'deny' }; }); diff --git a/src/App.svelte b/src/App.svelte index eef5bc1..b12d3da 100644 --- a/src/App.svelte +++ b/src/App.svelte @@ -28,8 +28,8 @@ const layout = $derived(width >= 1100 ? 'wide' : width >= 760 ? 'medium' : 'narrow'); // Arrangement (Appearance setting): side is the default, classic puts the channel list right - // of the chat, stacked above it. Phones always use drawers. - const arrangement = $derived(layout === 'narrow' ? 'side' : store.settings.arrangement ?? 'side'); + // of the chat, stacked above it. Narrow windows use drawers, except stacked, which works at any width. + const arrangement = $derived(store.settings.arrangement === 'stacked' ? 'stacked' : layout === 'narrow' ? 'side' : store.settings.arrangement ?? 'side'); // The member panel sits inline only in the side arrangement on wide windows, otherwise it overlays const panelInline = $derived(layout === 'wide' && arrangement === 'side'); let treeHeight = $state(Math.max(120, Number((() => { try { return localStorage.getItem('mumh5.treeHeight'); } catch { return null; } })()) || 280)); @@ -76,7 +76,8 @@ onsettings={() => (ui.settingsOpen = true)} /> {/snippet} {#snippet chat()} - (navOpen = true)} onpanel={togglePanel} /> + (navOpen = true)} onpanel={togglePanel} /> {/snippet}
diff --git a/src/lib/audio/sounds.svelte.ts b/src/lib/audio/sounds.svelte.ts index 00ff015..8e1c054 100644 --- a/src/lib/audio/sounds.svelte.ts +++ b/src/lib/audio/sounds.svelte.ts @@ -27,6 +27,10 @@ export const SOUND_EVENTS = [ { id: 'userBanned', group: 'Your channel', name: 'Someone in your channel was banned', ts: 'CLIENT_WAS_BANNED_CURRENT_CHANNEL', keys: ['banned_server_currentchannel'] }, { id: 'recordingStarted', group: 'Your channel', name: 'Someone started recording', ts: 'CLIENT_RECORDING_START', keys: ['recording_started'] }, { id: 'recordingStopped', group: 'Your channel', name: 'Someone stopped recording', ts: 'CLIENT_RECORDING_STOP', keys: ['recording_stopped'] }, + { id: 'shareStarted', group: 'Screen sharing', name: 'A screen share started (yours or in your channel)', ts: 'MUMH5_SHARE_STARTED', keys: ['share_started', 'stream_started'] }, + { id: 'shareStopped', group: 'Screen sharing', name: 'A screen share stopped', ts: 'MUMH5_SHARE_STOPPED', keys: ['share_stopped', 'stream_stopped'] }, + { id: 'viewerJoined', group: 'Screen sharing', name: 'Someone started watching your screen', ts: 'MUMH5_VIEWER_JOINED', keys: ['viewer_joined'] }, + { id: 'viewerLeft', group: 'Screen sharing', name: 'Someone stopped watching your screen', ts: 'MUMH5_VIEWER_LEFT', keys: ['viewer_left'] }, { id: 'message', group: 'Chat', name: 'Channel message', ts: 'CHAT_RECEIVED_MESSAGE_CHANNEL', keys: ['chat_message_inbound', 'message'] }, { id: 'privateMessage', group: 'Chat', name: 'Direct message', ts: 'CHAT_RECEIVED_MESSAGE_CLIENT', keys: ['private', 'direct', 'pm'] }, { id: 'messageSent', group: 'Chat', name: 'You sent a message', ts: 'CHAT_SENT_MESSAGE_CHANNEL', keys: ['chat_message_outbound', 'sent'], off: true }, @@ -79,6 +83,10 @@ const TONES: Record = { userBanned: [[294, 0, 0.12], [220, 0.13, 0.16]], recordingStarted: [[1175, 0, 0.08], [1175, 0.12, 0.08]], recordingStopped: [[880, 0, 0.12]], + shareStarted: [[523, 0, 0.07], [784, 0.08, 0.07], [1047, 0.16, 0.12]], + shareStopped: [[1047, 0, 0.07], [784, 0.08, 0.07], [523, 0.16, 0.12]], + viewerJoined: [[880, 0, 0.05], [1109, 0.06, 0.08]], + viewerLeft: [[1109, 0, 0.05], [880, 0.06, 0.08]], message: [[1047, 0, 0.06]], privateMessage: [[1047, 0, 0.05], [1319, 0.07, 0.07]], messageSent: [[1319, 0, 0.04]], diff --git a/src/lib/icons.ts b/src/lib/icons.ts index d9a0f6e..cffde88 100644 --- a/src/lib/icons.ts +++ b/src/lib/icons.ts @@ -7,6 +7,7 @@ export const ICON_PATHS: Record = { settings: 'M4 21v-7M4 10V3M12 21v-9M12 8V3M20 21v-5M20 12V3M1 14h6M9 8h6M17 16h6', plus: 'M12 5v14M5 12h14', screen: 'M3 4h18v12H3zM8 20h8M12 16v4', + popout: 'M14 4h6v6M20 4l-9 9M18 14v6H4V6h6', expand: 'M4 9V4h5M20 9V4h-5M4 15v5h5M20 15v5h-5', filter: 'M3 4h18l-7 8v7l-4 2v-9z', info: 'M12 2a10 10 0 1 0 0 20 10 10 0 0 0 0-20zM12 11v6M12 7h.01', diff --git a/src/lib/share.svelte.ts b/src/lib/share.svelte.ts index c32e893..0414836 100644 --- a/src/lib/share.svelte.ts +++ b/src/lib/share.svelte.ts @@ -9,6 +9,7 @@ import { desktop, isWeb } from './native.ts'; import { proxyStun } from './web.svelte.ts'; import { store } from './settings.svelte.ts'; import { ui } from './ui.svelte.ts'; +import { sounds } from './audio/sounds.svelte.ts'; // Every viewer is a separate upload from the sharer const MAX_VIEWERS = 8; @@ -44,6 +45,10 @@ class ScreenShare { private watchPc: RTCPeerConnection | null = null; private watchTimer: ReturnType | undefined; private assemblers = new WeakMap(); + // Viewers whose connection came up, so leaving is only announced for those + private live = new Set(); + // When we entered the channel: streams already running there are not announced with a sound + private enteredAt = new WeakMap(); // Plugin data needs a 1.4 server; older ones would not pass the signals on supported(s: Session): boolean { @@ -81,7 +86,7 @@ class ScreenShare { let answered = false; ui.prompt = { title: 'Screen sharing', label: '', confirm: 'Continue', input: 'none', - text: 'Screen sharing connects you directly to the other person. You see each other\'s IP address, like in a video call. Only mumh5 users in your channel can watch.', + text: 'Screen sharing connects you directly to other participants in this channel. You see each other\'s IP address. Only mumh5 users in your channel can watch.', onsubmit: () => { answered = true; store.settings.shareNoticeSeen = true; store.saveSettings(); resolve(true); }, oncancel: () => { if (!answered) resolve(false); } }; @@ -114,6 +119,7 @@ class ScreenShare { this.stream = stream; this.send(s, this.others(s), SHARE.on); s.setSharing(s.client.session!, true); + sounds.play('shareStarted'); } private release(): void { @@ -127,6 +133,7 @@ class ScreenShare { stop(): void { const s = this.host; + if (this.stream) sounds.play('shareStopped'); if (s?.client) { this.send(s, this.others(s), SHARE.off); if (s.client.session != null) s.setSharing(s.client.session, false); @@ -137,12 +144,14 @@ class ScreenShare { private dropPeers(): void { for (const pc of this.peers.values()) pc.close(); this.peers.clear(); + this.live.clear(); this.viewers = 0; } private dropPeer(session: number): void { this.peers.get(session)?.close(); this.peers.delete(session); + if (this.live.delete(session)) sounds.play('viewerLeft'); this.viewers = this.peers.size; } @@ -195,7 +204,11 @@ class ScreenShare { this.peers.set(viewer, pc); this.viewers = this.peers.size; pc.addEventListener('connectionstatechange', () => { - if ((pc.connectionState === 'failed' || pc.connectionState === 'closed') && this.peers.get(viewer) === pc) this.dropPeer(viewer); + if (this.peers.get(viewer) !== pc) return; + if (pc.connectionState === 'connected' && !this.live.has(viewer)) { + this.live.add(viewer); + sounds.play('viewerJoined'); + } else if (pc.connectionState === 'failed' || pc.connectionState === 'closed') this.dropPeer(viewer); }); for (const track of stream.getTracks()) { const sender = pc.addTrack(track, stream); @@ -241,8 +254,12 @@ class ScreenShare { // Only people in our channel take part if (!msg || !s.client || !this.inMyChannel(s, sender)) return; switch (msg.type) { - case SHARE.on: s.setSharing(sender, true); break; + case SHARE.on: + if (!s.sharing[sender] && Date.now() - (this.enteredAt.get(s) ?? 0) > 3000) sounds.play('shareStarted'); + s.setSharing(sender, true); + break; case SHARE.off: + if (s.sharing[sender]) sounds.play('shareStopped'); s.setSharing(sender, false); if (this.watching?.host === s && this.watching.session === sender) this.endWatch(); break; @@ -260,6 +277,7 @@ class ScreenShare { entered(s: Session): void { if (!this.supported(s)) return; const self = s.client!.session!; + this.enteredAt.set(s, Date.now()); s.clearSharing(); if (this.watching?.host === s) this.unwatch(); if (this.host === s) { @@ -277,6 +295,8 @@ class ScreenShare { if (this.host === s) this.send(s, [user.session], SHARE.on); return; } + // A sharer who left or disconnected: their stream is over for us + if (s.sharing[user.session]) sounds.play('shareStopped'); s.setSharing(user.session, false); if (this.host === s) this.dropPeer(user.session); if (this.watching?.host === s && this.watching.session === user.session) this.endWatch(); diff --git a/src/ui/AppearanceSettings.svelte b/src/ui/AppearanceSettings.svelte index 0f823fb..5bcd4af 100644 --- a/src/ui/AppearanceSettings.svelte +++ b/src/ui/AppearanceSettings.svelte @@ -63,7 +63,7 @@ {/each}
-

Narrow windows always use the phone layout.

+

Narrow windows use the phone layout, except Stacked, which stays as it is.

diff --git a/test/e2e/share.e2e.ts b/test/e2e/share.e2e.ts index 3956cee..1827506 100644 --- a/test/e2e/share.e2e.ts +++ b/test/e2e/share.e2e.ts @@ -72,16 +72,44 @@ try { }; await picker.getByLabel('Quality').selectOption('low'); await pickAndStart(); - await alice.page.getByText('You are sharing your screen').waitFor(); + await alice.page.locator('.stage .status', { hasText: '0 watching' }).waitFor(); await alice.page.locator('.sidebar .row.user.self').getByTitle('You are sharing your screen').waitFor(); - // The sharer's own tile shows what is being sent; clicking it makes it large - await alice.page.waitForFunction(() => (document.querySelector('.stage .tile.self video') as HTMLVideoElement | null)?.videoWidth! > 0, null, { timeout: 15000 }); - await alice.page.locator('.stage .tile.self').click(); + // The sharer sees what is being sent, large by default; the tiles themselves never carry the picture const ownView = alice.page.getByRole('region', { name: 'Screen share' }); await ownView.getByText('Your screen').waitFor(); + await alice.page.waitForFunction(() => (document.querySelector('.spot video') as HTMLVideoElement | null)?.videoWidth! > 0, null, { timeout: 15000 }); + assert.equal(await alice.page.locator('.stage .tile video').count(), 0); await ownView.getByRole('button', { name: 'Back to tiles' }).click(); await ownView.waitFor({ state: 'detached' }); - console.log('ok: sharing started from the dialog, own tile shows the stream'); + await alice.page.locator('.stage .tile.self').click(); + await ownView.waitFor(); + console.log('ok: sharing started from the dialog, own stream shown large'); + + // Stacked layout: only the picture, as wide as the chat, and the layout stays stacked on narrow windows + const setLayout = async (name: string) => { + await alice.page.getByTitle('Settings').click(); + await alice.page.getByRole('tab', { name: 'Appearance' }).click(); + await alice.page.getByRole('radio', { name }).click(); + await alice.page.getByRole('button', { name: 'Done', exact: true }).click(); + }; + await setLayout('Stacked'); + await alice.page.locator('.app.stacked .stage.solo').waitFor(); + assert.equal(await alice.page.locator('.stage .tiles').count(), 0, 'no participant tiles in the stacked layout'); + const fits = () => alice.page.evaluate(() => { + const v = document.querySelector('.stage.solo video') as HTMLVideoElement, c = document.querySelector('.chat')!; + return v.videoWidth > 0 && Math.abs(v.getBoundingClientRect().width - c.getBoundingClientRect().width) < 2; + }); + await alice.page.waitForFunction(() => (document.querySelector('.stage.solo video') as HTMLVideoElement | null)?.videoWidth! > 0); + assert.ok(await fits(), 'picture as wide as the chat'); + await alice.page.setViewportSize({ width: 600, height: 800 }); + await alice.page.locator('.app.stacked .stage.solo video').waitFor(); + assert.ok(await fits(), 'still as wide as the chat on a narrow window'); + if (process.env.SHOTS_DIR) await alice.page.screenshot({ path: `${process.env.SHOTS_DIR}/stacked.png` }); + await alice.page.getByText('0 watching').waitFor(); + await alice.page.setViewportSize({ width: 1280, height: 800 }); + await setLayout('Modern'); + await alice.page.locator('.stage .tiles').waitFor(); + console.log('ok: stacked layout shows only the picture and stays stacked when narrow'); // Bob connects afterwards and still learns about the stream await join(bob.page, bobName); @@ -92,27 +120,52 @@ try { await tile.waitFor(); console.log('ok: a late joiner sees who is sharing'); + // The tiles fill the stage, and the stage can be made taller or shorter + const tileWidth = async () => Math.round((await tile.boundingBox())!.width); + const before = await tileWidth(); + const handle = bob.page.getByRole('separator', { name: 'Resize the tiles' }); + await handle.focus(); + for (let i = 0; i < 8; i++) await bob.page.keyboard.press('ArrowDown'); + await bob.page.waitForFunction(w => Math.round(document.querySelector('.stage .tile')!.getBoundingClientRect().width) > w, before); + const taller = await tileWidth(); + for (let i = 0; i < 8; i++) await bob.page.keyboard.press('ArrowUp'); + await bob.page.waitForFunction(w => Math.round(document.querySelector('.stage .tile')!.getBoundingClientRect().width) < w, taller); + if (process.env.SHOTS_DIR) await bob.page.screenshot({ path: `${process.env.SHOTS_DIR}/tiles.png` }); + console.log(`ok: tiles scale with the stage height (${before} to ${taller} px wide)`); + await tile.click(); await bob.page.getByRole('button', { name: 'Continue' }).click(); const view = bob.page.getByRole('region', { name: 'Screen share' }); await view.getByText(aliceName).waitFor(); await bob.page.waitForFunction(() => (document.querySelector('.spot video') as HTMLVideoElement | null)?.videoWidth! > 0, null, { timeout: 20000 }); const size = await bob.page.evaluate(() => { const v = document.querySelector('.spot video') as HTMLVideoElement; return [v.videoWidth, v.videoHeight]; }); - await alice.page.getByText('You are sharing your screen, 1 watching').waitFor(); + await alice.page.locator('.stage .status', { hasText: '1 watching' }).waitFor(); console.log(`ok: bob receives the picture (${size[0]}x${size[1]}), alice sees one viewer`); - // Like a meeting: back to tiles keeps the picture in the tile, a click makes it large again + if (process.env.SHOTS_DIR) { await bob.page.waitForTimeout(500); await bob.page.screenshot({ path: `${process.env.SHOTS_DIR}/meeting.png` }); } + // The large picture can move to a window of its own + const opened = bob.app.waitForEvent('window'); + await view.getByRole('button', { name: 'Open in a separate window' }).click(); + const streamWindow = await opened; + await streamWindow.waitForFunction(() => (document.querySelector('video') as HTMLVideoElement | null)?.videoWidth! > 0, null, { timeout: 15000 }); + await view.getByText('Shown in a separate window.').waitFor(); + await streamWindow.close(); + await view.getByRole('button', { name: 'Open in a separate window' }).waitFor(); + await bob.page.waitForFunction(() => (document.querySelector('.spot video') as HTMLVideoElement | null)?.videoWidth! > 0); + console.log('ok: stream in a separate window, back when it is closed'); + + // Back to tiles hides the picture; the sharer's tile brings it back await view.getByRole('button', { name: 'Back to tiles' }).click(); await view.waitFor({ state: 'detached' }); - await bob.page.waitForFunction(() => (document.querySelector('.stage .tile.video video') as HTMLVideoElement | null)?.videoWidth! > 0); + assert.equal(await bob.page.locator('.stage .tile video').count(), 0); await tile.click(); await view.waitFor(); - console.log('ok: stream in the tile, large on click'); + console.log('ok: large view hidden and shown again from the tile'); // Leaving and coming back await view.getByRole('button', { name: 'Stop watching' }).click(); await view.waitFor({ state: 'detached' }); - await alice.page.getByText('You are sharing your screen', { exact: true }).waitFor(); + await alice.page.locator('.stage .status', { hasText: '0 watching' }).waitFor(); // The sharer's profile says so and offers to watch await bob.page.locator('.sidebar .row.user', { hasText: aliceName }).click(); const profile = bob.page.locator('.panel .sharing'); @@ -127,7 +180,7 @@ try { await view.waitFor({ state: 'detached' }); await watch.waitFor({ state: 'detached' }); await profile.waitFor({ state: 'detached' }); - assert.equal(await alice.page.getByText('You are sharing your screen').count(), 0); + assert.equal(await alice.page.locator('.stage .status').count(), 0); console.log('ok: stopping ends the stream for viewers'); // A sharer who disconnects takes the stream with them @@ -138,6 +191,13 @@ try { await watch.waitFor({ state: 'detached' }); console.log('ok: disconnecting ends the stream'); + // Sounds: the sharer hears the stream start and stop and viewers come and go; the channel hears streams start and stop + const sounds = (p: Page) => p.evaluate(() => [...((window as any).__mumh5Sounds ?? [])] as string[]); + const heardA = await sounds(alice.page), heardB = await sounds(bob.page); + for (const ev of ['shareStarted', 'viewerJoined', 'viewerLeft', 'shareStopped']) assert.ok(heardA.includes(ev), `sharer hears ${ev}: ${heardA}`); + for (const ev of ['shareStarted', 'shareStopped']) assert.ok(heardB.includes(ev), `viewer hears ${ev}: ${heardB}`); + console.log('ok: sounds for streams and viewers'); + console.log('SHARE E2E PASSED'); } finally { await alice.app.close().catch(() => {}); diff --git a/test/e2e/web.e2e.ts b/test/e2e/web.e2e.ts index 5f34787..8dddc65 100644 --- a/test/e2e/web.e2e.ts +++ b/test/e2e/web.e2e.ts @@ -183,11 +183,11 @@ try { await dialog.getByRole('button', { name: 'Choose a screen or window...' }).click(); await page.waitForFunction(() => (document.querySelector('.preview video') as HTMLVideoElement | null)?.videoWidth! > 0, null, { timeout: 60000 }); await dialog.getByRole('button', { name: 'Start sharing' }).click(); - await page.getByText('You are sharing your screen').waitFor(); + await page.locator('.stage .status', { hasText: '0 watching' }).waitFor(); await page2.locator('.stage .tile', { hasText: name }).click(); await page2.getByRole('button', { name: 'Continue' }).click(); await page2.waitForFunction(() => (document.querySelector('.spot video') as HTMLVideoElement | null)?.videoWidth! > 0, null, { timeout: 30000 }); - await page.getByText('You are sharing your screen, 1 watching').waitFor(); + await page.locator('.stage .status', { hasText: '1 watching' }).waitFor(); console.log('ok: screen sharing between two browsers'); console.log('WEB E2E PASSED');