implement global mic, muting, get chrome audio fully working again

This commit is contained in:
Aubrey 2025-01-04 12:08:00 -06:00
parent e35e5ae2fe
commit 3c54a5aafd
No known key found for this signature in database
10 changed files with 225 additions and 106 deletions

View file

@ -1,8 +1,9 @@
import { derived, type Readable, type Unsubscriber, writable } from "svelte/store";
import { Player } from "./client";
import { localStore, subscribe } from "./stores";
export class AudioManager {
globalGain = writable(1.0);
micGain: Readable<number>;
useMusic: boolean;
ac: AudioContext;
dest: MediaStreamAudioDestinationNode;
@ -10,7 +11,7 @@ export class AudioManager {
source: AudioNode;
public readonly output: MediaStream;
constructor(media: HTMLAudioElement | MediaStream) {
constructor(media: HTMLAudioElement | MediaStream, micVolume: Readable<number>, muted: Readable<boolean>) {
this.ac = new AudioContext();
this.dest = this.ac.createMediaStreamDestination();
this.gain = this.ac.createGain();
@ -26,68 +27,82 @@ export class AudioManager {
this.source = this.ac.createMediaStreamSource(media);
this.useMusic = false;
}
this.gain.gain.value /= 32;
subscribe([micVolume, muted], ([micVolume, muted]) => {
this.gain.gain.value = micVolume * +!muted;
});
this.source.connect(this.gain).connect(this.dest)
this.output = this.dest.stream;
}
createPlayerAudio(outputDiv: HTMLDivElement, localPlayer: Readable<Player | undefined>, peerPlayer: Readable<Player | undefined>) {
const panner = this.ac.createPanner();
panner.distanceModel = "linear";
panner.refDistance = 1500;
panner.maxDistance = 2500;
createPlayerAudio(
outputDiv: HTMLDivElement,
localPlayer: Readable<Player | undefined>,
peerPlayer: Readable<Player | undefined>,
globalSpeak: Readable<boolean>,
) {
const gain = this.ac.createGain();
const dest = this.ac.createMediaStreamDestination();
gain.connect(dest); //.connect(panner)
const isDev = ((globalThis as any).isDev = "hot" in import.meta);
let audio = document.createElement("audio");
audio.srcObject = dest.stream;
audio.autoplay = true;
audio.controls = false;
audio.controls = isDev;
audio.play();
outputDiv.appendChild(audio);
return new PlayerAudio(this.ac, panner, gain, localPlayer, peerPlayer);
return new PlayerAudio(this.ac, dest, gain, localPlayer, peerPlayer, globalSpeak);
}
}
export class PlayerAudio {
private source: MediaStreamAudioSourceNode | undefined;
private playerWatcher: Readable<[number[], boolean]>;
// private playerWatcher: Readable<[number[], boolean]>;
constructor(
private context: AudioContext,
private panner: PannerNode,
private dest: MediaStreamAudioDestinationNode,
private gain: GainNode,
localPlayer: Readable<Player | undefined>,
peerPlayer: Readable<Player | undefined>
peerPlayer: Readable<Player | undefined>,
globalSpeak: Readable<boolean>,
) {
this.playerWatcher = derived([localPlayer, peerPlayer], ([localPlayer, peerPlayer], set) => {
console.log("created player audio");
subscribe([localPlayer, peerPlayer], ([localPlayer, peerPlayer]) => {
if (!localPlayer || !peerPlayer) {
console.log("got player binding")
set([[0, 0, 0], false]);
this.gain.gain.value = 0;
// console.warn("missing a player...");
return () => { };
}
return derived([localPlayer.stageInfo, peerPlayer.stageInfo], ([localInfo, peerInfo]): [number[], boolean] => {
return [
localInfo.position.map((localPosition, i) => peerInfo.position[i] - localPosition),
localInfo.stage === peerInfo.stage
];
}).subscribe(set);
});
console.log("got player binding for (local, peer)", localPlayer.name, peerPlayer.name);
this.playerWatcher.subscribe(([[x, y, z], sameStage]) => {
// this.panner.positionX.value = x;
// this.panner.positionY.value = y;
// this.panner.positionZ.value = z;
return subscribe([localPlayer.proximityInfo, peerPlayer.proximityInfo, globalSpeak], ([localInfo, peerInfo, globalSpeak]) => {
const [x, y, z] = localInfo.position.map((localPosition, i) => peerInfo.position[i] - localPosition);
const sameStage = localInfo.stage === peerInfo.stage;
let magnitude = Math.sqrt(Math.pow(x, 2) + Math.pow(y, 2) + Math.pow(z, 2));
const max = 2500, min = 1500;
const max = 3500, min = 750;
let scaledGain = 1 - (magnitude - min) / max;
this.gain.gain.value = scaledGain * +sameStage;
let t = Math.max(Math.min(scaledGain * +sameStage, 1), 0);
function easeInExpo(x: number): number {
return x === 0 ? 0 : Math.pow(2, 10 * x - 10);
}
this.gain.gain.value = globalSpeak ? 1 : easeInExpo(t);
});
});
}
gotTrack(stream: MediaStream) {
let a: HTMLAudioElement | null = new Audio();
a.muted = true;
a.srcObject = stream;
a.addEventListener('canplaythrough', () => {
a = null;
});
a.play();
this.source = this.context.createMediaStreamSource(stream);
this.source.connect(this.gain);
let gainOut = this.source.connect(this.gain);
gainOut.connect(this.dest);
// gainOut.connect(this.context.destination);
}
}

View file

@ -6,14 +6,14 @@ import { ReadKind, readEvent } from "./protocol/serverEvent";
import { AudioManager, PlayerAudio } from "./audio";
import { type WriteEvent, writeEvent, WriteKind } from "./protocol/clientEvent";
import { BinaryWriter } from "./binary/writer";
import { cached, type CachedWritable, type Cached, cachedWritable } from "./stores";
import { cached, type CachedWritable, type Cached, cachedWritable, subscribe } from "./stores";
import { BinaryReader } from "./binary/reader";
import { base } from "$app/paths";
// @ts-ignore
import { PUBLIC_WT_URL } from "$env/static/public";
export class Client {
static async connect(name: Readable<string>, audioManager: AudioManager, outputAudio: HTMLDivElement) {
const settings = await (await fetch("settings.json")).json();
const transport = new WebTransport(`https://${settings.host}:${settings.port}`, { allowPooling: false });
static async connect(name: Readable<string>, globalSpeak: Readable<boolean>, audioManager: AudioManager, outputAudio: HTMLDivElement) {
const transport = new WebTransport(PUBLIC_WT_URL, { allowPooling: false });
await transport.ready;
console.log("ready!");
const stream = await transport.createBidirectionalStream();
@ -24,17 +24,20 @@ export class Client {
const array = new Uint8Array(0x20);
new TextEncoder().encodeInto(get(name), array);
await writer.write(array);
writer.write(new Uint8Array([+get(globalSpeak)]))
const reader = BinaryStreamReader.fromReader(stream.readable.getReader());
const { players, peers } = await readHello(reader);
console.log("done reading hello");
const client = new Client(audioManager, outputAudio, name, cachedWritable(players), uuid, transport, stream, writer);
const client = new Client(audioManager, outputAudio, name, globalSpeak, cachedWritable(players), uuid, transport, stream, writer);
globalThis.client = client;
globalThis.get = get;
for (const [id, name] of peers) {
for (const [id, name, globalSpeak] of peers) {
const peer = client.getPeer(id);
peer.targetPlayerName.set(name);
peer.globalSpeak.set(globalSpeak)
peer.offer();
}
@ -49,6 +52,7 @@ export class Client {
private async closeHandler() {
await this.transport.closed;
console.error("closed");
this.connected.set(false);
}
private getPeer(id: string) {
@ -93,25 +97,27 @@ export class Client {
return x;
})
break;
break;
}
case ReadKind.Moved: {
const player = this.players.get()[event.id];
if (player)
player.stageInfo.update(info => Object.assign(info, { position: event.position }));
player.proximityInfo.update(info => Object.assign(info, { position: event.position }));
break;
}
case ReadKind.StageChanged: {
console.log("goodbye", event.id);
const player = this.players.get()[event.id];
if (player)
player.stageInfo.update(info => Object.assign(info, { stage: event.stage }));
player.proximityInfo.update(info => Object.assign(info, { stage: event.stage }));
break;
}
case ReadKind.PeerConnectionChanged: {
console.log("peer changed", event.id, event.connected, event.target);
if (event.connected)
this.getPeer(event.id).targetPlayerName.set(event.target);
if (event.connected) {
const peer = this.getPeer(event.id);
peer.targetPlayerName.set(event.target);
peer.globalSpeak.set(event.globalSpeak);
}
else
delete this.peers[event.id];
break;
@ -135,11 +141,13 @@ export class Client {
private peers: Record<string, Peer> = {};
public boundPlayer: Readable<Player | undefined>;
public playersByName: Cached<Record<string, Player>>;
public connected: Writable<boolean> = writable(true);
private constructor(
public audioManager: AudioManager,
public outputAudio: HTMLDivElement,
boundPlayerName: Readable<string>,
globalSpeak: Readable<boolean>,
public players: CachedWritable<Record<number, Player>>,
public uuid: string,
public transport: WebTransport,
@ -149,26 +157,30 @@ export class Client {
this.playersByName = cached(this.players, (players) => {
console.log("players by name derivation", Object.values(players), Object.fromEntries(Object.values(players).map(x => [x.name, x])))
return Object.fromEntries(Object.values(players).map(x => [x.name, x]))
})
});
subscribe([boundPlayerName, globalSpeak], ([name, globalSpeak]) => {
console.log("writing target change");
this.writeEvent({
kind: WriteKind.TargetChanged,
name,
globalSpeak
});
});
this.boundPlayer = derived([boundPlayerName, this.playersByName], ([playerName, players]) => players[playerName]);
}
closed() {
return this.transport.closed;
}
}
export class Player {
public stageInfo: Writable<{ position: number[], stage: string }>;
constructor(public name: string, public id: number, stageInfo?: { position: number[], stage: string }) {
this.stageInfo = writable(Object.assign({ position: [0, 0, 0], stage: "nostage" + Math.random() }, stageInfo ?? {}));
public proximityInfo: CachedWritable<{ position: number[], stage: string }>;
constructor(public name: string, public id: number, proximityInfo?: { position: number[], stage: string }) {
this.proximityInfo = cachedWritable(Object.assign({ position: [0, 0, 0], stage: "nostage" + Math.random() }, proximityInfo ?? {}));
}
}
export class Peer {
private connection: RTCPeerConnection;
private playerAudio: PlayerAudio;
public globalSpeak: Writable<boolean> = writable(false);
public targetPlayerName: Writable<string>;
private targetPlayer: Readable<Player | undefined>;
private state: Writable<string> = writable("not connected");
@ -191,7 +203,7 @@ export class Peer {
});
this.targetPlayerName = writable("");
this.targetPlayer = derived([this.targetPlayerName, this.client.playersByName], ([playerName, players]) => players[playerName]);
this.playerAudio = this.client.audioManager.createPlayerAudio(this.client.outputAudio, this.client.boundPlayer, this.targetPlayer);
this.playerAudio = this.client.audioManager.createPlayerAudio(this.client.outputAudio, this.client.boundPlayer, this.targetPlayer, this.globalSpeak);
for (const track of this.client.audioManager.output.getAudioTracks()) {
this.connection.addTrack(track, this.client.audioManager.output);
}
@ -205,7 +217,12 @@ export class Peer {
else
console.log(this.peerId, "finished candidate gathering")
});
// this.connection.addEventListener("negotiationneeded", () => {
// console.warn("Needed renegotiation")
// this.offer();
// });
this.connection.addEventListener("track", (ev) => {
console.log("got track with it", ev.streams.length, ev.streams);
this.playerAudio.gotTrack(ev.streams[0]);
});
}
@ -228,7 +245,7 @@ export class Peer {
this.client.writeEvent({
kind: WriteKind.Answer,
to: this.peerId,
answerSdp: answer.sdp ?? throwExpr("it didn't create an offer???"),
answerSdp: answer.sdp ?? throwExpr("it didn't create an answer???"),
});
}

View file

@ -20,12 +20,14 @@ export type WriteEvent = {
answerSdp: string
} | {
kind: WriteKind.TargetChanged,
name: string
name: string,
globalSpeak: boolean,
}
export function writeEvent(writer: BinaryWriter, event: WriteEvent) {
writer.writeUInt8(event.kind);
if (event.kind === WriteKind.TargetChanged) {
writer.writeFixedString(0x20, event.name);
writer.writeBoolean(event.globalSpeak);
return;
}

View file

@ -16,8 +16,8 @@ export async function readHello(reader: BinaryStreamReader) {
{ position, stage },
);
};
const peers = await reader.readArray(await reader.readUInt32(), async (reader): Promise<[string, string]> => {
return [await reader.readFixedString(36), await reader.readFixedString(0x20)];
const peers = await reader.readArray(await reader.readUInt32(), async (reader): Promise<[string, string, boolean]> => {
return [await reader.readFixedString(36), await reader.readFixedString(0x20), await reader.readBoolean()];
});
return { players, peers };

View file

@ -38,7 +38,8 @@ export async function readEvent(reader: BinaryReader | BinaryStreamReader) {
const id = await reader.readFixedString(36);
const connected = await reader.readBoolean();
const target = await reader.readFixedString(0x20);
return { kind, id, connected, target };
const globalSpeak = await reader.readBoolean();
return { kind, id, connected, target, globalSpeak };
}
case ReadKind.Offer: {
const id = await reader.readFixedString(36);

View file

@ -1,4 +1,4 @@
import { writable, type Writable, type Readable, derived } from "svelte/store";
import { writable, type Writable, type Readable, derived, type Unsubscriber } from "svelte/store";
export function localStore<T>(key: string, defaultValue: T, initialConverter: (value: string) => T, stringifier: (value: T) => string): Writable<T> {
let baseWritable = writable(defaultValue);
@ -67,6 +67,13 @@ export function cachedWritable<T>(initial: T): CachedWritable<T> {
return newValue;
});
},
subscribe: store.subscribe
subscribe(run, invalidate) {
console.log('fucyk');
return store.subscribe(run, invalidate)
}
}
}
export function subscribe<S extends Stores>(stores: S, subscriber: (values: StoresValues<S>) => void): Unsubscriber {
return derived(stores, identity).subscribe(subscriber);
}

View file

@ -3,20 +3,41 @@
import { Client, Player } from "$lib/client";
import { AudioManager } from "$lib/audio";
import { identity, localStore } from "$lib/stores";
import { readable, readonly, type Writable } from "svelte/store";
import {
readable,
readonly,
type Readable,
type Writable,
} from "svelte/store";
const isDev = ((globalThis as any).isDev = location.pathname.includes("dev"));
const setupName = new URLSearchParams(location.search).get("name");
const isDev = ((globalThis as any).isDev = "hot" in import.meta);
let name = localStore("name", "", identity, identity);
if (setupName) name.set(setupName);
let useMusic = isDev
? localStore(
"useMusic",
false,
(v) => v == "true",
(v) => v.toString()
(v) => v.toString(),
)
: readable(false);
let micVolume = localStore("micVolume", 0.4, parseFloat, (v) => v.toString());
let muted = localStore(
"muted",
false,
(v) => v == "true",
(v) => v.toString(),
);
let globalSpeak = localStore(
"globalSpeak",
true,
(v) => v == "true",
(v) => v.toString(),
);
let client: Client = $state(undefined as any);
let players: Writable<Record<number, Player>> = undefined as any;
let connected: Readable<boolean> = undefined as any;
let ac: AudioManager;
let wyrm: HTMLAudioElement;
let outputAudio: HTMLDivElement;
@ -27,35 +48,75 @@
audio: true,
video: false,
});
ac = new AudioManager(media);
ac = new AudioManager(media, micVolume, muted);
}
async function connect() {
await audioPrep();
client = await Client.connect(readonly(name), ac, outputAudio);
client = await Client.connect(
readonly(name),
readonly(globalSpeak),
ac,
outputAudio,
);
players = client.players;
players.subscribe(() => {
console.log("players", $players);
players = client.players;
});
connected = readonly(client.connected);
console.log(client);
}
</script>
<div bind:this={outputAudio}>audio outputs</div>
{#if "WebTransport" in globalThis}
<div bind:this={outputAudio}>{void 0}</div>
<div>
<div class="controls">
<div>
I am <input
type="text"
maxlength="32"
list="connectedPlayers"
bind:value={$name}
/>
</div>
<div>
<label for="micVolume">Mic Volume: </label>
<input
id="micVolume"
type="range"
min="0"
max="1"
step="0.1"
bind:value={$micVolume}
/>
</div>
<div>
<label for="muted">Muted: </label>
<input id="muted" type="checkbox" bind:checked={$muted} />
</div>
<div>
<label for="globalSpeak">Global Mic: </label>
<input id="globalSpeak" type="checkbox" bind:checked={$globalSpeak} />
</div>
<div>
<button onclick={() => connect()}>Connect</button>
</div>
{#if isDev}
<label for="useMusic">Use Music</label>
<div>
<label for="useMusic">Use Music (dev only!)</label>
<input id="useMusic" type="checkbox" bind:checked={$useMusic} />
<audio bind:this={wyrm} src="/wyrmjewelbox.wav" autoplay={false} controls
>audio of the wyrm's houseki box</audio
<audio
bind:this={wyrm}
src="/wyrmjewelbox.wav"
autoplay={false}
controls>audio of the wyrm's houseki box</audio
>
</div>
{/if}
</div>
{#if client !== undefined}
</div>
{#if client !== undefined}
<input type="checkbox" disabled bind:checked={$connected} />
<datalist>
{#each Object.values($players) as player}
<option>{player.name}</option>
@ -67,10 +128,22 @@
<pre class="name">{player.name}</pre>
{/each}
</div>
{/if}
{/if}
<style>
<style>
.controls {
display: flex;
flex-basis: 1 1 auto;
flex-direction: column;
}
.name {
font-style: italic;
}
</style>
</style>
{:else}
!secureContext... was the correct link shared? <a
href="chrome://flags/#unsafely-treat-insecure-origin-as-secure"
>make sure you have the insecure flag set in case you need it</a
>
to <b>{window.location}</b>
{/if}

View file

@ -1,4 +0,0 @@
{
"host": "example.com",
"port": 4433
}

View file

@ -21,7 +21,10 @@ const config = {
fallback: undefined,
precompress: false,
strict: true
})
}),
// paths: {
// base: "/dev"
// }
}
};

View file

@ -8,6 +8,11 @@ export default defineConfig({
// key: readFileSync('key.pem'),
// cert: readFileSync('cert.pem'),
// },
// hmr: {
// path: "ws",
// host: "smosh.eepy.engineering"
// },
// origin: "https://smosh.eepy.engineering"
},
plugins: [
sveltekit(),