Built-in Opus codec for browsers without one; cameras on by default with per-person hide

- libopus in WebAssembly takes over voice where WebCodecs audio is missing
  (Firefox on Android); tested for tone, timing and bitrate, and in a browser
  forced onto it
- Cameras in the channel show by themselves once the IP note was confirmed;
  a person's camera can be hidden from the right-click menu, all of them with
  one button; an enlarged camera leaves its tile
- CSP allows wasm-unsafe-eval for the codec

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/lib/actions.ts

@@ -69,7 +69,12 @@ export function userMenu(u: User): MenuItem[] {
     );
     if (u.userId == null) items.push({ label: 'Register on this server', action: () => session.register(u.session) });
   } else {
-    if (session.sharing[u.session]) items.push({ label: `Watch ${session.sharing[u.session]}`, action: () => share.watch(sessions.active, u.session) });
+    const sharing = session.sharing[u.session];
+    if (sharing === 'camera') {
+      // Cameras show by themselves; this is the per-person switch
+      const hidden = share.isHidden(sessions.active, u.session) || !share.watchOf(u.session);
+      items.push({ label: hidden ? 'Show camera' : 'Hide camera', action: () => share.hideCamera(sessions.active, u.session, !hidden) });
+    } else if (sharing) items.push({ label: `Watch ${sharing}`, action: () => share.watch(sessions.active, u.session) });
     items.push(
       { label: 'Send message', action: () => session.selectView(`dm:${u.session}`) },
       { label: 'Mute for me', checked: !!voice.settings.localMutes[userKey(u)], action: () => voice.setLocalMute(u, !voice.settings.localMutes[userKey(u)]) },

src/lib/audio/opus-load.ts

@@ -0,0 +1,8 @@
+import wasmUrl from '@evan/opus/wasm/opus.wasm?url';
+import { instantiateOpus, type Opus } from './opus-wasm.ts';
+
+// Fetched once, and only in browsers that need it
+let loading: Promise<Opus> | null = null;
+export function loadOpus(): Promise<Opus> {
+  return loading ??= fetch(wasmUrl).then(r => r.arrayBuffer()).then(instantiateOpus).catch(e => { loading = null; throw e; });
+}

src/lib/audio/opus-wasm.ts

@@ -0,0 +1,126 @@
+// Opus in WebAssembly (libopus, from the @evan/opus package), for browsers without WebCodecs
+// audio, such as Firefox on Android. Only loaded there; everywhere else the browser's own
+// codec is used. The two classes offer the small part of the WebCodecs interface the voice
+// engine needs. No DOM or bundler imports here, so the tests can run it in Node.
+
+interface Exports {
+  memory: WebAssembly.Memory;
+  malloc(size: number): number;
+  free(ptr: number): void;
+  opus_encoder_get_size(channels: number): number;
+  opus_encoder_init(ptr: number, rate: number, channels: number, application: number): number;
+  opus_encoder_ctl_set(ptr: number, request: number, value: number): number;
+  opus_encode(ptr: number, pcm: number, frameSize: number, out: number, max: number): number;
+  opus_decoder_get_size(channels: number): number;
+  opus_decoder_init(ptr: number, rate: number, channels: number): number;
+  opus_decode(ptr: number, data: number, length: number, pcm: number, frameSize: number, fec: number): number;
+}
+
+export interface Opus { wasm: Exports }
+
+const RATE = 48000;
+const MAX_PACKET = 4000;
+// The longest Opus frame is 120 ms
+const MAX_SAMPLES = 5760;
+const APPLICATION_VOIP = 2048, SET_BITRATE = 4002, SET_COMPLEXITY = 4010, SET_INBAND_FEC = 4012, SET_PACKET_LOSS = 4014, SET_SIGNAL = 4024, SIGNAL_VOICE = 3001;
+
+// bytes: the contents of opus.wasm (fetched by opus-load.ts in the app, read from disk in tests)
+export async function instantiateOpus(bytes: BufferSource): Promise<Opus> {
+  const imports = {
+    wasi_snapshot_preview1: { fd_seek() { return 0; }, fd_write() { return 0; }, fd_close() { return 0; }, proc_exit() {} },
+    // Memory is read through fresh views every call, so growth needs no bookkeeping
+    env: { emscripten_notify_memory_growth() {} }
+  };
+  const { instance } = await WebAssembly.instantiate(bytes, imports);
+  return { wasm: instance.exports as unknown as Exports };
+}
+
+// Takes the engine's 10 ms frames and gives out one packet per frame duration
+export class WasmEncoder {
+  state: 'configured' | 'closed' = 'configured';
+  private wasm: Exports;
+  private ptr: number;
+  private pcm: number;
+  private out: number;
+  private samples: number;
+  private filled = 0;
+  private output: (packet: Uint8Array) => void;
+
+  constructor(opus: Opus, bitrate: number, frameMs: number, output: (packet: Uint8Array) => void) {
+    const w = this.wasm = opus.wasm;
+    this.output = output;
+    this.samples = RATE * frameMs / 1000;
+    this.ptr = w.malloc(w.opus_encoder_get_size(1));
+    this.pcm = w.malloc(this.samples * 2);
+    this.out = w.malloc(MAX_PACKET);
+    if (w.opus_encoder_init(this.ptr, RATE, 1, APPLICATION_VOIP) < 0) { this.close(); throw new Error('Could not start the Opus encoder'); }
+    w.opus_encoder_ctl_set(this.ptr, SET_BITRATE, Math.round(bitrate));
+    // A notch below the maximum: phones are the ones without a codec of their own
+    w.opus_encoder_ctl_set(this.ptr, SET_COMPLEXITY, 8);
+    w.opus_encoder_ctl_set(this.ptr, SET_INBAND_FEC, 1);
+    w.opus_encoder_ctl_set(this.ptr, SET_PACKET_LOSS, 10);
+    w.opus_encoder_ctl_set(this.ptr, SET_SIGNAL, SIGNAL_VOICE);
+  }
+
+  push(frame: Float32Array): void {
+    if (this.state !== 'configured') return;
+    let view = new Int16Array(this.wasm.memory.buffer, this.pcm, this.samples);
+    for (let i = 0; i < frame.length; i++) {
+      const v = Math.max(-1, Math.min(1, frame[i]));
+      view[this.filled++] = v < 0 ? v * 0x8000 : v * 0x7fff;
+      if (this.filled < this.samples) continue;
+      this.filled = 0;
+      const length = this.wasm.opus_encode(this.ptr, this.pcm, this.samples, this.out, MAX_PACKET);
+      if (length > 0) this.output(new Uint8Array(this.wasm.memory.buffer, this.out, length).slice());
+      // Encoding may have grown the memory, which detaches the view
+      view = new Int16Array(this.wasm.memory.buffer, this.pcm, this.samples);
+    }
+  }
+
+  // An unfinished frame at the end of a transmission is dropped
+  flush(): Promise<void> {
+    this.filled = 0;
+    return Promise.resolve();
+  }
+
+  close(): void {
+    if (this.state === 'closed') return;
+    this.state = 'closed';
+    for (const p of [this.ptr, this.pcm, this.out]) if (p) this.wasm.free(p);
+  }
+}
+
+export class WasmDecoder {
+  private wasm: Exports;
+  private ptr: number;
+  private data: number;
+  private pcm: number;
+  private closed = false;
+
+  constructor(opus: Opus) {
+    const w = this.wasm = opus.wasm;
+    this.ptr = w.malloc(w.opus_decoder_get_size(1));
+    this.data = w.malloc(MAX_PACKET);
+    this.pcm = w.malloc(MAX_SAMPLES * 2);
+    if (w.opus_decoder_init(this.ptr, RATE, 1) < 0) { this.close(); throw new Error('Could not start the Opus decoder'); }
+  }
+
+  // Mono samples for the playback worklet, or null for a packet that does not decode
+  decode(packet: Uint8Array): Float32Array<ArrayBuffer> | null {
+    // An empty packet would make libopus invent audio (loss concealment)
+    if (this.closed || !packet.length || packet.length > MAX_PACKET) return null;
+    new Uint8Array(this.wasm.memory.buffer, this.data, packet.length).set(packet);
+    const n = this.wasm.opus_decode(this.ptr, this.data, packet.length, this.pcm, MAX_SAMPLES, 0);
+    if (n <= 0) return null;
+    const samples = new Int16Array(this.wasm.memory.buffer, this.pcm, n);
+    const out = new Float32Array(n);
+    for (let i = 0; i < n; i++) out[i] = samples[i] / 0x8000;
+    return out;
+  }
+
+  close(): void {
+    if (this.closed) return;
+    this.closed = true;
+    for (const p of [this.ptr, this.data, this.pcm]) if (p) this.wasm.free(p);
+  }
+}

src/lib/audio/voice.svelte.ts

@@ -1,3 +1,5 @@
+import { WasmEncoder, WasmDecoder, type Opus } from './opus-wasm.ts';
+import { loadOpus } from './opus-load.ts';
 import captureUrl from './capture.worklet.ts?worker&url';
 import playbackUrl from './playback.worklet.ts?worker&url';
 import { encodeVoice, decodeVoice } from '../../core/voice-packet.ts';
@@ -77,8 +79,10 @@ class VoiceEngine {
   private outputGainNode: GainNode | null = null;
   private capture: AudioWorkletNode | null = null;
   private playback: AudioWorkletNode | null = null;
-  private encoder: AudioEncoder | null = null;
-  private decoders = new Map<number, { decoder: AudioDecoder; lastFrame: number; timer: ReturnType<typeof setTimeout> | null }>();
+  // The browser's Opus codec (WebCodecs), or our own in WebAssembly where the browser has none
+  private opus: Opus | null = null;
+  private encoder: AudioEncoder | WasmEncoder | null = null;
+  private decoders = new Map<number, { decoder: AudioDecoder | WasmDecoder; lastFrame: number; timer: ReturnType<typeof setTimeout> | null }>();
   private client: MumbleClient | null = null;
   private sessionGains = new Map<number, number>();
   private unsubscribe: (() => void)[] = [];
@@ -139,8 +143,18 @@ class VoiceEngine {
     if (this.ctx) return;
     this.error = '';
     try {
-      if (typeof AudioEncoder === 'undefined' || !(await AudioEncoder.isConfigSupported(this.encoderConfig())).supported) {
-        throw new Error('This browser cannot encode Opus audio (WebCodecs is missing), so you cannot talk or hear others here. Chromium-based browsers and the desktop app can.');
+      const builtIn = typeof AudioEncoder !== 'undefined' && typeof AudioDecoder !== 'undefined' &&
+        (await AudioEncoder.isConfigSupported(this.encoderConfig()).catch(() => ({ supported: false }))).supported;
+      // mumh5.forceWasmOpus in localStorage: use the built-in codec regardless, for testing it
+      let forced = false;
+      try { forced = localStorage.getItem('mumh5.forceWasmOpus') === '1'; } catch { /* storage unavailable */ }
+      if (!builtIn || forced) {
+        // Firefox on Android, for one, has no audio codec for pages; bring our own
+        try {
+          this.opus = await loadOpus();
+        } catch (e) {
+          throw new Error(`This browser has no Opus audio codec, and the built-in one could not be loaded (${(e as Error).message})`);
+        }
       }
       const ctx = new AudioContext({ sampleRate: SAMPLE_RATE, latencyHint: 'interactive' });
       this.ctx = ctx;
@@ -281,11 +295,16 @@ class VoiceEngine {
     this.effectiveBitrate = this.allowedBitrate();
     if (!this.ctx) return;
     try { this.encoder?.close(); } catch { /* closed */ }
-    this.encoder = new AudioEncoder({
+    if (this.opus) {
+      this.encoder = new WasmEncoder(this.opus, this.effectiveBitrate, this.settings.frameMs, packet => this.onPacket(packet));
+      return;
+    }
+    const encoder = new AudioEncoder({
       output: chunk => this.onEncoded(chunk),
       error: e => { this.error = `Encoder: ${e.message}`; }
     });
-    this.encoder.configure(this.encoderConfig());
+    encoder.configure(this.encoderConfig());
+    this.encoder = encoder;
   }
 
   private canTalk(): boolean {
@@ -320,18 +339,24 @@ class VoiceEngine {
     if (!wanted && this.open) { this.endTransmission(); return; }
     if (!this.open) return;
 
+    const encoder = this.encoder!;
+    if (encoder instanceof WasmEncoder) return encoder.push(frame);
     const data = new AudioData({
       format: 'f32-planar', sampleRate: SAMPLE_RATE, numberOfFrames: FRAME, numberOfChannels: 1,
       timestamp: this.sampleTime, data: frame
     });
     this.sampleTime += 10000;
-    this.encoder!.encode(data);
+    encoder.encode(data);
     data.close();
   }
 
   private onEncoded(chunk: EncodedAudioChunk): void {
     const opus = new Uint8Array(chunk.byteLength);
     chunk.copyTo(opus);
+    this.onPacket(opus);
+  }
+
+  private onPacket(opus: Uint8Array): void {
     if (this.stopping) { this.held.push(opus); return; }
     this.sendPacket(opus, false);
   }
@@ -389,11 +414,15 @@ class VoiceEngine {
     let d = this.decoders.get(p.session);
     if (!d) {
       const session = p.session;
-      const decoder = new AudioDecoder({
-        output: data => this.onDecoded(session, data),
-        error: () => this.dropSession(session)
-      });
-      decoder.configure({ codec: 'opus', sampleRate: SAMPLE_RATE, numberOfChannels: 1 });
+      let decoder: AudioDecoder | WasmDecoder;
+      if (this.opus) decoder = new WasmDecoder(this.opus);
+      else {
+        decoder = new AudioDecoder({
+          output: data => this.onDecoded(session, data),
+          error: () => this.dropSession(session)
+        });
+        decoder.configure({ codec: 'opus', sampleRate: SAMPLE_RATE, numberOfChannels: 1 });
+      }
       d = { decoder, lastFrame: -1, timer: null };
       this.decoders.set(session, d);
       if (user) this.setSessionGain(session, this.settings.userVolumes[userKey(user)] ?? 1);
@@ -402,7 +431,13 @@ class VoiceEngine {
     d.lastFrame = p.frame;
 
     if (p.opus.length) {
-      d.decoder.decode(new EncodedAudioChunk({ type: 'key', timestamp: p.frame * 10000, data: p.opus }));
+      if (d.decoder instanceof WasmDecoder) {
+        const pcm = d.decoder.decode(p.opus);
+        if (pcm) {
+          this.stats.decoded++;
+          this.playback?.port.postMessage({ session: p.session, pcm }, [pcm.buffer]);
+        }
+      } else d.decoder.decode(new EncodedAudioChunk({ type: 'key', timestamp: p.frame * 10000, data: p.opus }));
     }
     if (!this.talking[p.session]) this.talking[p.session] = true;
     if (d.timer) clearTimeout(d.timer);

src/lib/settings.svelte.ts

@@ -25,6 +25,7 @@ export interface Settings {
   publicListUrl: string;  // where the public server list comes from
   arrangement: Arrangement;
   shareQuality: 'low' | 'medium' | 'high'; // bitrate and frame rate of our own stream
+  autoCameras: boolean;   // show other people's cameras as they come on
   shareVolume: number;    // volume of streams we watch, 0 to 1
   relayServer: string;    // a mumh5 web proxy whose STUN and relay to use for screen sharing
   stunServer: string;     // helps screen sharing connect through routers; empty contacts nobody
@@ -58,6 +59,7 @@ const defaults: Settings = {
   publicListUrl: 'https://publist.mumble.info/v1/list?version=1.5.0',
   arrangement: 'side',
   shareQuality: 'medium',
+  autoCameras: true,
   shareVolume: 1,
   relayServer: '',
   stunServer: '',

src/lib/share.svelte.ts

@@ -46,9 +46,8 @@ class ScreenShare {
   viewers = $state(0);
   // Streams we receive: one screen at a time, and any number of cameras next to it
   watches = $state.raw<Watching[]>([]);
-  // In the camera round: everyone's camera in the channel is watched as it comes on
-  cams = $state(false);
-  private camHost: Session | null = null;
+  // People whose camera we chose not to see, by certificate (or name); until the app restarts
+  hidden = $state<Record<string, boolean>>({});
   error = $state('');
   // The server the start dialog is open for
   dialog = $state.raw<Session | null>(null);
@@ -148,8 +147,6 @@ class ScreenShare {
     this.send(s, this.others(s), SHARE.on, kind);
     s.setSharing(s.client.session!, kind);
     sounds.play('shareStarted');
-    // Turning the camera on joins the round
-    if (kind === 'camera') this.setCams(s, true);
   }
 
   private release(): void {
@@ -193,6 +190,28 @@ class ScreenShare {
     return this.watches.find(w => w.session === session);
   }
 
+  // Cameras come on by themselves, like in a video call, once the note about IP addresses was
+  // confirmed; each person's can be hidden, and the whole thing switched off
+  get cams(): boolean {
+    return store.settings.autoCameras;
+  }
+  private who(s: Session, session: number): string {
+    const u = s.client?.users.get(session);
+    return u ? u.hash || `name:${u.name}` : '';
+  }
+  isHidden(s: Session, session: number): boolean {
+    return !!this.hidden[this.who(s, session)];
+  }
+  hideCamera(s: Session, session: number, hide: boolean): void {
+    const key = this.who(s, session);
+    if (!key) return;
+    if (hide) { this.hidden[key] = true; this.unwatch(session); }
+    else { delete this.hidden[key]; this.watch(s, session); }
+  }
+  private autoWatch(s: Session, session: number): void {
+    if (this.cams && store.settings.shareNoticeSeen && s.sharing[session] === 'camera' && !this.isHidden(s, session)) this.watch(s, session);
+  }
+
   async watch(s: Session, session: number): Promise<void> {
     const kind = s.sharing[session];
     if (!this.supported(s) || !kind || this.watchOf(session) || !(await this.notice())) return;
@@ -200,6 +219,8 @@ class ScreenShare {
     for (const w of this.watches) if (w.host !== s || (kind === 'screen' && w.kind === 'screen')) this.unwatch(w.session);
     if (kind === 'camera' && this.watches.filter(w => w.kind === 'camera').length >= MAX_CAMERAS) return;
     if (!s.sharing[session] || this.watchOf(session)) return;
+    // Asking for it shows it again
+    delete this.hidden[this.who(s, session)];
     this.watches = [...this.watches, { host: s, session, kind, stream: null, state: 'connecting' }];
     this.send(s, [session], SHARE.watch);
     this.watchTimers.set(session, setTimeout(() => this.setWatchState(session, 'failed'), 20000));
@@ -228,20 +249,16 @@ class ScreenShare {
     this.watches = this.watches.map(w => w.session === session ? { ...w, state, stream: stream ?? w.stream } : w);
   }
 
-  // The camera round: watch every camera in the channel, now and as they come on
+  // All cameras on or off, remembered
   async setCams(s: Session, on: boolean): Promise<void> {
+    if (on && (!this.supported(s) || !(await this.notice()))) return;
+    store.settings.autoCameras = on;
+    store.saveSettings();
     if (!on) {
-      this.cams = false;
-      this.camHost = null;
       for (const w of this.watches) if (w.kind === 'camera') this.unwatch(w.session);
       return;
     }
-    if (!this.supported(s) || !(await this.notice())) return;
-    this.cams = true;
-    this.camHost = s;
-    for (const [session, kind] of Object.entries(s.sharing)) {
-      if (kind === 'camera' && Number(session) !== s.client?.session) this.watch(s, Number(session));
-    }
+    for (const session of Object.keys(s.sharing)) if (Number(session) !== s.client?.session) this.autoWatch(s, Number(session));
   }
 
   // Candidates travel inside the description, so wait for them instead of sending each one
@@ -338,7 +355,7 @@ class ScreenShare {
       case SHARE.on:
         if (!s.sharing[sender] && Date.now() - (this.enteredAt.get(s) ?? 0) > 3000) sounds.play('shareStarted');
         s.setSharing(sender, msg.text === 'camera' ? 'camera' : 'screen');
-        if (msg.text === 'camera' && this.cams && this.camHost === s) this.watch(s, sender);
+        this.autoWatch(s, sender);
         break;
       case SHARE.off:
         if (s.sharing[sender]) sounds.play('shareStopped');
@@ -388,7 +405,6 @@ class ScreenShare {
     if (this.host === s) this.release();
     if (this.dialog === s) this.dialog = null;
     for (const w of this.watches) if (w.host === s) this.endWatch(w.session);
-    if (this.camHost === s) { this.cams = false; this.camHost = null; }
   }
 }
 

src/ui/VoiceStage.svelte

@@ -288,7 +288,8 @@
           {@const stream = streamOf(u.session)}
           {@const sharing = session.sharing[u.session]}
           {@const isSelf = u.session === self?.session}
-          {@const cam = camOf(u.session)}
+          <!-- While a camera is shown large, its tile goes back to the avatar: one picture, not two -->
+          {@const cam = focus === u.session ? null : camOf(u.session)}
           {@const w = watchOf(u.session)}
           <button class="tile" class:talking={talking(u.session)} class:self={isSelf} class:focused={focus === u.session}
             onclick={() => tileClick(u.session)}
@@ -359,8 +360,8 @@
         {/if}
       </button>
       {#if anyCamera || share.cams}
-        <button class="icon-btn" class:live={share.cams} aria-pressed={share.cams} title={share.cams ? 'Stop watching the cameras' : 'Watch everyone\'s camera'}
-          aria-label={share.cams ? 'Stop watching the cameras' : 'Watch all cameras'} onclick={() => share.setCams(sessions.active, !share.cams)}>
+        <button class="icon-btn" class:live={share.cams} aria-pressed={share.cams} title={share.cams ? 'Hide everyone\'s camera' : 'Show everyone\'s camera'}
+          aria-label={share.cams ? 'Hide all cameras' : 'Show all cameras'} onclick={() => share.setCams(sessions.active, !share.cams)}>
           <Icon name="camera" size={18} />
         </button>
       {/if}