implement global mic, muting, get chrome audio fully working again

src/lib/audio.ts

@@ -1,8 +1,9 @@
 import { derived, type Readable, type Unsubscriber, writable } from "svelte/store";
 import { Player } from "./client";
+import { localStore, subscribe } from "./stores";
 
 export class AudioManager {
-  globalGain = writable(1.0);
+  micGain: Readable<number>;
   useMusic: boolean;
   ac: AudioContext;
   dest: MediaStreamAudioDestinationNode;
@@ -10,7 +11,7 @@ export class AudioManager {
   source: AudioNode;
   public readonly output: MediaStream;
 
-  constructor(media: HTMLAudioElement | MediaStream) {
+  constructor(media: HTMLAudioElement | MediaStream, micVolume: Readable<number>, muted: Readable<boolean>) {
     this.ac = new AudioContext();
     this.dest = this.ac.createMediaStreamDestination();
     this.gain = this.ac.createGain();
@@ -26,68 +27,82 @@ export class AudioManager {
       this.source = this.ac.createMediaStreamSource(media);
       this.useMusic = false;
     }
-    this.gain.gain.value /= 32;
+    subscribe([micVolume, muted], ([micVolume, muted]) => {
+      this.gain.gain.value = micVolume * +!muted;
+    });
     this.source.connect(this.gain).connect(this.dest)
     this.output = this.dest.stream;
   }
 
-  createPlayerAudio(outputDiv: HTMLDivElement, localPlayer: Readable<Player | undefined>, peerPlayer: Readable<Player | undefined>) {
+  createPlayerAudio(
-    const panner = this.ac.createPanner();
+    outputDiv: HTMLDivElement,
-    panner.distanceModel = "linear";
+    localPlayer: Readable<Player | undefined>,
-    panner.refDistance = 1500;
+    peerPlayer: Readable<Player | undefined>,
-    panner.maxDistance = 2500;
+    globalSpeak: Readable<boolean>,
+  ) {
     const gain = this.ac.createGain();
     const dest = this.ac.createMediaStreamDestination();
-    gain.connect(dest); //.connect(panner)
 
+    const isDev = ((globalThis as any).isDev = "hot" in import.meta);
     let audio = document.createElement("audio");
     audio.srcObject = dest.stream;
     audio.autoplay = true;
-    audio.controls = false;
+    audio.controls = isDev;
+    audio.play();
     outputDiv.appendChild(audio);
 
-    return new PlayerAudio(this.ac, panner, gain, localPlayer, peerPlayer);
+    return new PlayerAudio(this.ac, dest, gain, localPlayer, peerPlayer, globalSpeak);
   }
 }
 
 export class PlayerAudio {
   private source: MediaStreamAudioSourceNode | undefined;
-  private playerWatcher: Readable<[number[], boolean]>;
+  // private playerWatcher: Readable<[number[], boolean]>;
   constructor(
     private context: AudioContext,
-    private panner: PannerNode,
+    private dest: MediaStreamAudioDestinationNode,
     private gain: GainNode,
     localPlayer: Readable<Player | undefined>,
-    peerPlayer: Readable<Player | undefined>
+    peerPlayer: Readable<Player | undefined>,
+    globalSpeak: Readable<boolean>,
   ) {
-    this.playerWatcher = derived([localPlayer, peerPlayer], ([localPlayer, peerPlayer], set) => {
+    console.log("created player audio");
+
+    subscribe([localPlayer, peerPlayer], ([localPlayer, peerPlayer]) => {
       if (!localPlayer || !peerPlayer) {
-        console.log("got player binding")
+        this.gain.gain.value = 0;
-        set([[0, 0, 0], false]);
+        // console.warn("missing a player...");
         return () => { };
       }
 
-      return derived([localPlayer.stageInfo, peerPlayer.stageInfo], ([localInfo, peerInfo]): [number[], boolean] => {
+      console.log("got player binding for (local, peer)", localPlayer.name, peerPlayer.name);
-        return [
-          localInfo.position.map((localPosition, i) => peerInfo.position[i] - localPosition),
-          localInfo.stage === peerInfo.stage
-        ];
-      }).subscribe(set);
-    });
 
-    this.playerWatcher.subscribe(([[x, y, z], sameStage]) => {
+      return subscribe([localPlayer.proximityInfo, peerPlayer.proximityInfo, globalSpeak], ([localInfo, peerInfo, globalSpeak]) => {
-      // this.panner.positionX.value = x;
+        const [x, y, z] = localInfo.position.map((localPosition, i) => peerInfo.position[i] - localPosition);
-      // this.panner.positionY.value = y;
+        const sameStage = localInfo.stage === peerInfo.stage;
-      // this.panner.positionZ.value = z;
         let magnitude = Math.sqrt(Math.pow(x, 2) + Math.pow(y, 2) + Math.pow(z, 2));
-      const max = 2500, min = 1500;
+        const max = 3500, min = 750;
         let scaledGain = 1 - (magnitude - min) / max;
-      this.gain.gain.value = scaledGain * +sameStage;
+        let t = Math.max(Math.min(scaledGain * +sameStage, 1), 0);
+        function easeInExpo(x: number): number {
+          return x === 0 ? 0 : Math.pow(2, 10 * x - 10);
+        }
+        this.gain.gain.value = globalSpeak ? 1 : easeInExpo(t);
+      });
     });
   }
 
   gotTrack(stream: MediaStream) {
+    let a: HTMLAudioElement | null = new Audio();
+    a.muted = true;
+    a.srcObject = stream;
+    a.addEventListener('canplaythrough', () => {
+      a = null;
+    });
+    a.play();
     this.source = this.context.createMediaStreamSource(stream);
-    this.source.connect(this.gain);
+    let gainOut = this.source.connect(this.gain);
+    gainOut.connect(this.dest);
+    // gainOut.connect(this.context.destination);
   }
 }

src/lib/client.ts

@@ -6,14 +6,14 @@ import { ReadKind, readEvent } from "./protocol/serverEvent";
 import { AudioManager, PlayerAudio } from "./audio";
 import { type WriteEvent, writeEvent, WriteKind } from "./protocol/clientEvent";
 import { BinaryWriter } from "./binary/writer";
-import { cached, type CachedWritable, type Cached, cachedWritable } from "./stores";
+import { cached, type CachedWritable, type Cached, cachedWritable, subscribe } from "./stores";
 import { BinaryReader } from "./binary/reader";
-import { base } from "$app/paths";
+// @ts-ignore
+import { PUBLIC_WT_URL } from "$env/static/public";
 
 export class Client {
-  static async connect(name: Readable<string>, audioManager: AudioManager, outputAudio: HTMLDivElement) {
+  static async connect(name: Readable<string>, globalSpeak: Readable<boolean>, audioManager: AudioManager, outputAudio: HTMLDivElement) {
-    const settings = await (await fetch("settings.json")).json();
+    const transport = new WebTransport(PUBLIC_WT_URL, { allowPooling: false });
-    const transport = new WebTransport(`https://${settings.host}:${settings.port}`, { allowPooling: false });
     await transport.ready;
     console.log("ready!");
     const stream = await transport.createBidirectionalStream();
@@ -24,17 +24,20 @@ export class Client {
     const array = new Uint8Array(0x20);
     new TextEncoder().encodeInto(get(name), array);
     await writer.write(array);
+    writer.write(new Uint8Array([+get(globalSpeak)]))
 
     const reader = BinaryStreamReader.fromReader(stream.readable.getReader());
     const { players, peers } = await readHello(reader);
     console.log("done reading hello");
 
-    const client = new Client(audioManager, outputAudio, name, cachedWritable(players), uuid, transport, stream, writer);
+    const client = new Client(audioManager, outputAudio, name, globalSpeak, cachedWritable(players), uuid, transport, stream, writer);
     globalThis.client = client;
+    globalThis.get = get;
 
-    for (const [id, name] of peers) {
+    for (const [id, name, globalSpeak] of peers) {
       const peer = client.getPeer(id);
       peer.targetPlayerName.set(name);
+      peer.globalSpeak.set(globalSpeak)
       peer.offer();
     }
 
@@ -49,6 +52,7 @@ export class Client {
   private async closeHandler() {
     await this.transport.closed;
     console.error("closed");
+    this.connected.set(false);
   }
 
   private getPeer(id: string) {
@@ -93,25 +97,27 @@ export class Client {
             return x;
           })
           break;
-          break;
         }
         case ReadKind.Moved: {
           const player = this.players.get()[event.id];
           if (player)
-            player.stageInfo.update(info => Object.assign(info, { position: event.position }));
+            player.proximityInfo.update(info => Object.assign(info, { position: event.position }));
           break;
         }
         case ReadKind.StageChanged: {
           console.log("goodbye", event.id);
           const player = this.players.get()[event.id];
           if (player)
-            player.stageInfo.update(info => Object.assign(info, { stage: event.stage }));
+            player.proximityInfo.update(info => Object.assign(info, { stage: event.stage }));
           break;
         }
         case ReadKind.PeerConnectionChanged: {
           console.log("peer changed", event.id, event.connected, event.target);
-          if (event.connected)
+          if (event.connected) {
-            this.getPeer(event.id).targetPlayerName.set(event.target);
+            const peer = this.getPeer(event.id);
+            peer.targetPlayerName.set(event.target);
+            peer.globalSpeak.set(event.globalSpeak);
+          }
           else
             delete this.peers[event.id];
           break;
@@ -135,11 +141,13 @@ export class Client {
   private peers: Record<string, Peer> = {};
   public boundPlayer: Readable<Player | undefined>;
   public playersByName: Cached<Record<string, Player>>;
+  public connected: Writable<boolean> = writable(true);
 
   private constructor(
     public audioManager: AudioManager,
     public outputAudio: HTMLDivElement,
     boundPlayerName: Readable<string>,
+    globalSpeak: Readable<boolean>,
     public players: CachedWritable<Record<number, Player>>,
     public uuid: string,
     public transport: WebTransport,
@@ -149,26 +157,30 @@ export class Client {
     this.playersByName = cached(this.players, (players) => {
       console.log("players by name derivation", Object.values(players), Object.fromEntries(Object.values(players).map(x => [x.name, x])))
       return Object.fromEntries(Object.values(players).map(x => [x.name, x]))
-    })
+    });
+    subscribe([boundPlayerName, globalSpeak], ([name, globalSpeak]) => {
+      console.log("writing target change");
+      this.writeEvent({
+        kind: WriteKind.TargetChanged,
+        name,
+        globalSpeak
+      });
+    });
     this.boundPlayer = derived([boundPlayerName, this.playersByName], ([playerName, players]) => players[playerName]);
   }
-
-  closed() {
-    return this.transport.closed;
-  }
 }
 
 export class Player {
-  public stageInfo: Writable<{ position: number[], stage: string }>;
+  public proximityInfo: CachedWritable<{ position: number[], stage: string }>;
-  constructor(public name: string, public id: number, stageInfo?: { position: number[], stage: string }) {
+  constructor(public name: string, public id: number, proximityInfo?: { position: number[], stage: string }) {
-
+    this.proximityInfo = cachedWritable(Object.assign({ position: [0, 0, 0], stage: "nostage" + Math.random() }, proximityInfo ?? {}));
-    this.stageInfo = writable(Object.assign({ position: [0, 0, 0], stage: "nostage" + Math.random() }, stageInfo ?? {}));
   }
 }
 
 export class Peer {
   private connection: RTCPeerConnection;
   private playerAudio: PlayerAudio;
+  public globalSpeak: Writable<boolean> = writable(false);
   public targetPlayerName: Writable<string>;
   private targetPlayer: Readable<Player | undefined>;
   private state: Writable<string> = writable("not connected");
@@ -191,7 +203,7 @@ export class Peer {
     });
     this.targetPlayerName = writable("");
     this.targetPlayer = derived([this.targetPlayerName, this.client.playersByName], ([playerName, players]) => players[playerName]);
-    this.playerAudio = this.client.audioManager.createPlayerAudio(this.client.outputAudio, this.client.boundPlayer, this.targetPlayer);
+    this.playerAudio = this.client.audioManager.createPlayerAudio(this.client.outputAudio, this.client.boundPlayer, this.targetPlayer, this.globalSpeak);
     for (const track of this.client.audioManager.output.getAudioTracks()) {
       this.connection.addTrack(track, this.client.audioManager.output);
     }
@@ -205,7 +217,12 @@ export class Peer {
       else
         console.log(this.peerId, "finished candidate gathering")
     });
+    // this.connection.addEventListener("negotiationneeded", () => {
+    // console.warn("Needed renegotiation")
+    // this.offer();
+    // });
     this.connection.addEventListener("track", (ev) => {
+      console.log("got track with it", ev.streams.length, ev.streams);
       this.playerAudio.gotTrack(ev.streams[0]);
     });
   }
@@ -228,7 +245,7 @@ export class Peer {
     this.client.writeEvent({
       kind: WriteKind.Answer,
       to: this.peerId,
-      answerSdp: answer.sdp ?? throwExpr("it didn't create an offer???"),
+      answerSdp: answer.sdp ?? throwExpr("it didn't create an answer???"),
     });
   }
 

src/lib/protocol/clientEvent.ts

@@ -20,12 +20,14 @@ export type WriteEvent = {
   answerSdp: string
 } | {
   kind: WriteKind.TargetChanged,
-  name: string
+  name: string,
+  globalSpeak: boolean,
 }
 export function writeEvent(writer: BinaryWriter, event: WriteEvent) {
   writer.writeUInt8(event.kind);
   if (event.kind === WriteKind.TargetChanged) {
     writer.writeFixedString(0x20, event.name);
+    writer.writeBoolean(event.globalSpeak);
     return;
   }
 

src/lib/protocol/hello.ts

@@ -16,8 +16,8 @@ export async function readHello(reader: BinaryStreamReader) {
       { position, stage },
     );
   };
-  const peers = await reader.readArray(await reader.readUInt32(), async (reader): Promise<[string, string]> => {
+  const peers = await reader.readArray(await reader.readUInt32(), async (reader): Promise<[string, string, boolean]> => {
-    return [await reader.readFixedString(36), await reader.readFixedString(0x20)];
+    return [await reader.readFixedString(36), await reader.readFixedString(0x20), await reader.readBoolean()];
   });
 
   return { players, peers };

src/lib/protocol/serverEvent.ts

@@ -38,7 +38,8 @@ export async function readEvent(reader: BinaryReader | BinaryStreamReader) {
       const id = await reader.readFixedString(36);
       const connected = await reader.readBoolean();
       const target = await reader.readFixedString(0x20);
-      return { kind, id, connected, target };
+      const globalSpeak = await reader.readBoolean();
+      return { kind, id, connected, target, globalSpeak };
     }
     case ReadKind.Offer: {
       const id = await reader.readFixedString(36);

src/lib/stores.ts

@@ -1,4 +1,4 @@
-import { writable, type Writable, type Readable, derived } from "svelte/store";
+import { writable, type Writable, type Readable, derived, type Unsubscriber } from "svelte/store";
 
 export function localStore<T>(key: string, defaultValue: T, initialConverter: (value: string) => T, stringifier: (value: T) => string): Writable<T> {
   let baseWritable = writable(defaultValue);
@@ -67,6 +67,13 @@ export function cachedWritable<T>(initial: T): CachedWritable<T> {
         return newValue;
       });
     },
-    subscribe: store.subscribe
+    subscribe(run, invalidate) {
+      console.log('fucyk');
+      return store.subscribe(run, invalidate)
     }
   }
+}
+
+export function subscribe<S extends Stores>(stores: S, subscriber: (values: StoresValues<S>) => void): Unsubscriber {
+  return derived(stores, identity).subscribe(subscriber);
+}

src/routes/+page.svelte

@@ -3,20 +3,41 @@
   import { Client, Player } from "$lib/client";
   import { AudioManager } from "$lib/audio";
   import { identity, localStore } from "$lib/stores";
-  import { readable, readonly, type Writable } from "svelte/store";
+  import {
+    readable,
+    readonly,
+    type Readable,
+    type Writable,
+  } from "svelte/store";
 
-  const isDev = ((globalThis as any).isDev = location.pathname.includes("dev"));
+  const setupName = new URLSearchParams(location.search).get("name");
+  const isDev = ((globalThis as any).isDev = "hot" in import.meta);
   let name = localStore("name", "", identity, identity);
+  if (setupName) name.set(setupName);
   let useMusic = isDev
     ? localStore(
         "useMusic",
         false,
         (v) => v == "true",
-        (v) => v.toString()
+        (v) => v.toString(),
       )
     : readable(false);
+  let micVolume = localStore("micVolume", 0.4, parseFloat, (v) => v.toString());
+  let muted = localStore(
+    "muted",
+    false,
+    (v) => v == "true",
+    (v) => v.toString(),
+  );
+  let globalSpeak = localStore(
+    "globalSpeak",
+    true,
+    (v) => v == "true",
+    (v) => v.toString(),
+  );
   let client: Client = $state(undefined as any);
   let players: Writable<Record<number, Player>> = undefined as any;
+  let connected: Readable<boolean> = undefined as any;
   let ac: AudioManager;
   let wyrm: HTMLAudioElement;
   let outputAudio: HTMLDivElement;
@@ -27,18 +48,30 @@
           audio: true,
           video: false,
         });
-    ac = new AudioManager(media);
+    ac = new AudioManager(media, micVolume, muted);
   }
   async function connect() {
     await audioPrep();
-    client = await Client.connect(readonly(name), ac, outputAudio);
+    client = await Client.connect(
+      readonly(name),
+      readonly(globalSpeak),
+      ac,
+      outputAudio,
+    );
     players = client.players;
+    players.subscribe(() => {
+      console.log("players", $players);
+      players = client.players;
+    });
+    connected = readonly(client.connected);
     console.log(client);
   }
 </script>
 
-<div bind:this={outputAudio}>audio outputs</div>
+{#if "WebTransport" in globalThis}
+  <div bind:this={outputAudio}>{void 0}</div>
 
+  <div class="controls">
     <div>
       I am <input
         type="text"
@@ -46,16 +79,44 @@
         list="connectedPlayers"
         bind:value={$name}
       />
+    </div>
+    <div>
+      <label for="micVolume">Mic Volume: </label>
+      <input
+        id="micVolume"
+        type="range"
+        min="0"
+        max="1"
+        step="0.1"
+        bind:value={$micVolume}
+      />
+    </div>
+    <div>
+      <label for="muted">Muted: </label>
+      <input id="muted" type="checkbox" bind:checked={$muted} />
+    </div>
+    <div>
+      <label for="globalSpeak">Global Mic: </label>
+      <input id="globalSpeak" type="checkbox" bind:checked={$globalSpeak} />
+    </div>
+    <div>
       <button onclick={() => connect()}>Connect</button>
+    </div>
     {#if isDev}
-    <label for="useMusic">Use Music</label>
+      <div>
+        <label for="useMusic">Use Music (dev only!)</label>
         <input id="useMusic" type="checkbox" bind:checked={$useMusic} />
-    <audio bind:this={wyrm} src="/wyrmjewelbox.wav" autoplay={false} controls
+        <audio
-      >audio of the wyrm's houseki box</audio
+          bind:this={wyrm}
+          src="/wyrmjewelbox.wav"
+          autoplay={false}
+          controls>audio of the wyrm's houseki box</audio
         >
+      </div>
     {/if}
   </div>
   {#if client !== undefined}
+    <input type="checkbox" disabled bind:checked={$connected} />
     <datalist>
       {#each Object.values($players) as player}
         <option>{player.name}</option>
@@ -70,7 +131,19 @@
   {/if}
 
   <style>
+    .controls {
+      display: flex;
+      flex-basis: 1 1 auto;
+      flex-direction: column;
+    }
     .name {
       font-style: italic;
     }
   </style>
+{:else}
+  !secureContext... was the correct link shared? <a
+    href="chrome://flags/#unsafely-treat-insecure-origin-as-secure"
+    >make sure you have the insecure flag set in case you need it</a
+  >
+  to <b>{window.location}</b>
+{/if}

static/settings.json

@@ -1,4 +0,0 @@
-{
-  "host": "example.com",
-  "port": 4433
-}

svelte.config.js

@@ -21,7 +21,10 @@ const config = {
 			fallback: undefined,
 			precompress: false,
 			strict: true
-		})
+		}),
+		// paths: {
+		// 	base: "/dev"
+		// }
 	}
 };
 

vite.config.ts

@@ -8,6 +8,11 @@ export default defineConfig({
 		// 	key: readFileSync('key.pem'),
 		// 	cert: readFileSync('cert.pem'),
 		// },
+		// hmr: {
+		// 	path: "ws",
+		// 	host: "smosh.eepy.engineering"
+		// },
+		// origin: "https://smosh.eepy.engineering"
 	},
 	plugins: [
 		sveltekit(),