kibi/mumh5
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Built-in Opus codec for browsers without one; cameras on by default with per-person hide

Browse Source 
- libopus in WebAssembly takes over voice where WebCodecs audio is missing
  (Firefox on Android); tested for tone, timing and bitrate, and in a browser
  forced onto it
- Cameras in the channel show by themselves once the IP note was confirmed;
  a person's camera can be hidden from the right-click menu, all of them with
  one button; an enlarged camera leaves its tile
- CSP allows wasm-unsafe-eval for the codec

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
kibi
2026-10-02 00:36:43 +02:00
co-authored by Claude Opus 5.5
parent 6368e63b0e
commit 5d43b8e4ae
17 changed files with 353 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/e2e/share.e2e.ts
+15 -4
View File
@@ -268,13 +268,20 @@ try {
assert.equal(await picker.getByLabel('Sound').count(), 0, 'no extra sound with a camera');
await startBtn.click();
await alice.page.locator('.sidebar .row.user.self').getByTitle('You are sharing your camera').waitFor();
// Cameras play in the tiles. Bob joins the camera round with one click and sees Alice's
// Cameras play in the tiles and come on by themselves, like in a video call
const cam = bob.page.locator('.sidebar').getByRole('button', { name: `Watch the camera of ${aliceName}`, exact: true });
await cam.waitFor();
const liveCams = (p: Page) => p.evaluate(() => [...document.querySelectorAll('.stage .tile video')].filter(v => (v as HTMLVideoElement).videoWidth > 0).length);
const camsShown = (p: Page, n: number) => p.waitForFunction(
want => [...document.querySelectorAll('.stage .tile video')].filter(v => (v as HTMLVideoElement).videoWidth > 0).length === want, n, { timeout: 30000 });
await bob.page.locator('.stage').getByRole('button', { name: 'Watch all cameras' }).click();
await camsShown(bob.page, 1);
// One person's camera can be hidden, and shown again
await bob.page.locator('.stage .tile', { hasText: aliceName }).click({ button: 'right' });
await bob.page.getByRole('menuitem', { name: 'Hide camera' }).click();
await camsShown(bob.page, 0);
await alice.page.locator('.stage .status', { hasText: '0 watching' }).waitFor();
await bob.page.locator('.stage .tile', { hasText: aliceName }).click({ button: 'right' });
await bob.page.getByRole('menuitem', { name: 'Show camera' }).click();
await camsShown(bob.page, 1);
assert.equal(await bob.page.getByRole('region', { name: 'Screen share' }).count(), 0, 'a camera does not take over the stage by itself');
@@ -295,13 +302,17 @@ try {
// A click enlarges one camera; the statistics show it came through the relay
await bob.page.locator('.stage .tile', { hasText: aliceName }).click();
await bob.page.waitForFunction(() => (document.querySelector('.spot video') as HTMLVideoElement | null)?.videoWidth! > 0, null, { timeout: 20000 });
assert.equal(await bob.page.locator('.stage .tile', { hasText: aliceName }).locator('video').count(), 0, 'the enlarged camera is not also in its tile');
assert.equal(await bob.page.locator('.stage .tile.self video').count(), 1, 'the other tiles keep theirs');
const camStats = await nerd(bob.page, [/^relayed \(TURN\), reached over TCP/, /^0 direct, 0 through STUN, [1-9]\d* relayed/]);
await bob.page.getByRole('region', { name: 'Screen share' }).getByRole('button', { name: 'Back to tiles' }).click();
// Leaving the round keeps Bob's own camera on and drops the others
await bob.page.locator('.stage').getByRole('button', { name: 'Stop watching the cameras' }).click();
// Everyone's cameras off at once keeps Bob's own camera on and drops the others; on again brings them back
await bob.page.locator('.stage').getByRole('button', { name: 'Hide all cameras' }).click();
await camsShown(bob.page, 1);
await alice.page.locator('.stage .status', { hasText: '0 watching' }).waitFor();
await bob.page.locator('.stage').getByRole('button', { name: 'Show all cameras' }).click();
await camsShown(bob.page, 2);
await bob.page.locator('.me').getByRole('button', { name: 'Stop sharing your camera' }).click();
await camsShown(alice.page, 1);
await alice.page.locator('.me').getByRole('button', { name: 'Stop sharing your camera' }).click();
test/e2e/web.e2e.ts
+35 -2
View File
@@ -9,7 +9,9 @@ import { mkdtempSync, existsSync, statSync } from 'node:fs';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { createCodec } from '../../src/core/proto.ts';
import { decodeVoice, type VoicePacket } from '../../src/core/voice-packet.ts';
import { decodeVoice, encodeVoice, type VoicePacket } from '../../src/core/voice-packet.ts';
import { readFileSync } from 'node:fs';
import { instantiateOpus, WasmDecoder } from '../../src/lib/audio/opus-wasm.ts';
import { MumbleClient } from '../../src/core/client.ts';
import type { Transport } from '../../src/core/transport.ts';
import { openTls } from '../../electron/tls-transport.ts';
@@ -171,7 +173,10 @@ try {
page2.on('console', m => { if (m.type() === 'error') console.log('[page2]', m.text()); });
await page2.setViewportSize({ width: 1280, height: 800 });
await page2.getByRole('button', { name: /Create a new identity/ }).click();
await page2.getByLabel('Name', { exact: true }).fill(`webcarol${Date.now() % 100000}`);
const carol = `webcarol${Date.now() % 100000}`;
// This browser is made to use the built-in Opus codec, as one without WebCodecs audio would
await page2.evaluate(() => localStorage.setItem('mumh5.forceWasmOpus', '1'));
await page2.getByLabel('Name', { exact: true }).fill(carol);
await page2.getByRole('button', { name: 'Create', exact: true }).click();
await page2.getByRole('button', { name: 'Skip for now' }).click();
await page2.getByRole('button', { name: 'Done' }).click();
@@ -179,6 +184,34 @@ try {
await page2.getByRole('button', { name: 'Save and connect' }).click();
await page2.getByText(/Connected/).first().waitFor();
// Talking with the built-in codec: its packets are real Opus, checked by decoding them here
const carolSession = () => [...bob.users.values()].find(u => u.name === carol)!.session;
const carolVoice = new Promise<VoicePacket[]>(res => {
const packets: VoicePacket[] = [];
const off = bob.on('voice', raw => {
const p = decodeVoice(raw);
if (!p || p.session !== carolSession() || !p.opus.length) return;
packets.push(p);
if (packets.length >= 20) { off(); res(packets); }
});
});
await page2.getByTitle('Settings').click();
await page2.getByRole('radio', { name: /Always on/ }).click();
await page2.getByRole('button', { name: 'Done', exact: true }).click();
const fromCarol = await within(carolVoice, 'voice from the built-in codec', 20000);
const opus = await instantiateOpus(readFileSync(path.join(root, 'node_modules/@evan/opus/wasm/opus.wasm')));
const check = new WasmDecoder(opus);
const decoded = fromCarol.map(p => check.decode(p.opus));
assert.ok(decoded.every(pcm => pcm && pcm.length === 960), 'every packet is 20 ms of decodable Opus');
assert.ok(decoded.some(pcm => pcm!.some(v => Math.abs(v) > 0.01)), 'and carries the test tone, not silence');
// Hearing: her own packets sent back as bob light him up, and no codec error shows
const back = fromCarol.map((p, i) => encodeVoice({ target: 0, frame: 9000 + i * 2, opus: p.opus, last: i === fromCarol.length - 1 }, bob.protobufVoice));
const pump = setInterval(() => { const pk = back.shift(); if (pk) bob.sendVoiceTunnel(pk); }, 20);
await page2.locator('.sidebar .row.user', { hasText: 'bob' }).locator('.label.talking').waitFor({ timeout: 5000 });
clearInterval(pump);
assert.equal(await page2.getByText('Voice is not working').count(), 0);
console.log(`ok: voice with the built-in Opus codec (${fromCarol[0].opus.length} bytes per packet)`);
await page.locator('.me').getByRole('button', { name: 'Share your screen' }).click();
await page.getByRole('button', { name: 'Continue' }).click();
const dialog = page.getByRole('dialog', { name: 'Share your screen' });