add support for audio streaming

DavidVentura · Feb 2, 2024 · 737f024 · 737f024
1 parent 75f559a
commit 737f024
Show file tree

Hide file tree

Showing 3 changed files with 117 additions and 7 deletions.
diff --git a/README.md b/README.md
@@ -18,6 +18,8 @@ There's a basic UI which can display multiple cameras:
 
 The server will send a broadcast packet every few seconds to discover all the cameras available; this means that it *must* run in the same broadcast domain (VLAN) as your cameras.
 
+Clicking on the image will take you to a page that has audio streaming enabled.
+
 ## Pairing a new camera
 
 Connect to the device's access point and run `SSID=<your ssid> PSK=<your password> make pair`.

diff --git a/asd.html b/asd.html
@@ -0,0 +1,62 @@
+<html>
+    <h2>${id}</h2><a href="/camera/${id}"><img src="/camera/${id}"/></a><hr/>
+	<script>
+		const alaw_to_s16_table = [
+		  -5504, -5248, -6016, -5760, -4480, -4224, -4992, -4736, -7552, -7296, -8064, -7808, -6528, -6272, -7040, -6784, -2752,
+		  -2624, -3008, -2880, -2240, -2112, -2496, -2368, -3776, -3648, -4032, -3904, -3264, -3136, -3520, -3392, -22016,
+		  -20992, -24064, -23040, -17920, -16896, -19968, -18944, -30208, -29184, -32256, -31232, -26112, -25088, -28160,
+		  -27136, -11008, -10496, -12032, -11520, -8960, -8448, -9984, -9472, -15104, -14592, -16128, -15616, -13056, -12544,
+		  -14080, -13568, -344, -328, -376, -360, -280, -264, -312, -296, -472, -456, -504, -488, -408, -392, -440, -424, -88,
+		  -72, -120, -104, -24, -8, -56, -40, -216, -200, -248, -232, -152, -136, -184, -168, -1376, -1312, -1504, -1440, -1120,
+		  -1056, -1248, -1184, -1888, -1824, -2016, -1952, -1632, -1568, -1760, -1696, -688, -656, -752, -720, -560, -528, -624,
+		  -592, -944, -912, -1008, -976, -816, -784, -880, -848, 5504, 5248, 6016, 5760, 4480, 4224, 4992, 4736, 7552, 7296,
+		  8064, 7808, 6528, 6272, 7040, 6784, 2752, 2624, 3008, 2880, 2240, 2112, 2496, 2368, 3776, 3648, 4032, 3904, 3264,
+		  3136, 3520, 3392, 22016, 20992, 24064, 23040, 17920, 16896, 19968, 18944, 30208, 29184, 32256, 31232, 26112, 25088,
+		  28160, 27136, 11008, 10496, 12032, 11520, 8960, 8448, 9984, 9472, 15104, 14592, 16128, 15616, 13056, 12544, 14080,
+		  13568, 344, 328, 376, 360, 280, 264, 312, 296, 472, 456, 504, 488, 408, 392, 440, 424, 88, 72, 120, 104, 24, 8, 56,
+		  40, 216, 200, 248, 232, 152, 136, 184, 168, 1376, 1312, 1504, 1440, 1120, 1056, 1248, 1184, 1888, 1824, 2016, 1952,
+		  1632, 1568, 1760, 1696, 688, 656, 752, 720, 560, 528, 624, 592, 944, 912, 1008, 976, 816, 784, 880, 848,
+		];
+
+		const alaw_to_s16 = (a_val) => {
+		  return alaw_to_s16_table[a_val];
+		};
+
+		const audio_context = new AudioContext();
+		const gain_node = audio_context.createGain(); // Declare gain node
+		const channels =1;
+		const sample_rate = 8000;
+		const audioBuffer = audio_context.createBuffer(channels, 960, sample_rate); // 960??
+		//const audioBuffer = audio_context.createBuffer(channels, decoded.length, sample_rate);
+
+        gain_node.connect(audio_context.destination); // Connect gain node to speakers
+		audio_context.resume();
+
+		const evtSource = new EventSource('/audio/${id}');
+		evtSource.onopen = (e) => {
+			console.log("evtsource open");
+		}
+		let endsAt = 0;
+		let startAt = 0;
+		evtSource.onmessage = (e) => {
+			const nowBuffering = audioBuffer.getChannelData(0);
+			const u8 = Uint8Array.from(atob(e.data), c => c.charCodeAt(0));
+			new Int16Array(u8).map(alaw_to_s16).forEach((el, i) => nowBuffering[i] = el / 0x8000 );
+
+			const source_node = audio_context.createBufferSource();
+			source_node.buffer = audioBuffer;
+			source_node.connect(gain_node);
+			const now = Date.now();
+			if(now > endsAt) { // lost packets
+				startAt = 0;
+			} else {
+				startAt += audioBuffer.duration;
+			}
+			source_node.start(startAt);
+			endsAt = now + audioBuffer.duration * 1000;
+		};
+		evtSource.onerror = (e) => {
+			console.log("evtsource error", e);
+		}
+	</script>
+</html>
diff --git a/http_server.ts b/http_server.ts
@@ -1,5 +1,5 @@
 import { RemoteInfo } from "dgram";
-import { createWriteStream } from "node:fs";
+import { readFileSync } from "node:fs";
 import http from "node:http";
 
 import { discoverDevices } from "./discovery.js";
@@ -11,15 +11,52 @@ const opts = {
   debug: false,
   ansi: false,
   // discovery_ip: "192.168.40.255", //, "192.168.1.255"
-  discovery_ip: "192.168.40.101",
+  discovery_ip: "192.168.40.104",
   attempt_to_fix_packet_loss: false,
+  //attempt_to_fix_packet_loss: true,
 };
 
 let BOUNDARY = "a very good boundary line";
 let responses: Record<string, ServerResponse[]> = {};
+let audioResponses: Record<string, ServerResponse[]> = {};
 let sessions: Record<string, Session> = {};
 
 const server = http.createServer((req, res) => {
+  if (req.url.startsWith("/ui/")) {
+    let devId = req.url.split("/")[2];
+    let s = sessions[devId];
+    if (s === undefined) {
+      res.writeHead(400);
+      res.end("invalid ID");
+      return;
+    }
+    if (!s.connected) {
+      res.writeHead(400);
+      res.end("Nothing online");
+      return;
+    }
+    const ui = readFileSync("asd.html").toString();
+    res.end(ui.replace(/\${id}/g, devId));
+    return;
+  }
+  if (req.url.startsWith("/audio/")) {
+    let devId = req.url.split("/")[2];
+    let s = sessions[devId];
+    if (s === undefined) {
+      res.writeHead(400);
+      res.end("invalid ID");
+      return;
+    }
+    if (!s.connected) {
+      res.writeHead(400);
+      res.end("Nothing online");
+      return;
+    }
+    res.setHeader("Content-Type", `text/event-stream`);
+    audioResponses[devId].push(res);
+    return;
+  }
+
   if (req.url.startsWith("/camera/")) {
     let devId = req.url.split("/")[2];
     console.log("requested for", devId);
@@ -35,6 +72,7 @@ const server = http.createServer((req, res) => {
       res.end("Nothing online");
       return;
     }
+
     res.setHeader("Content-Type", `multipart/x-mixed-replace; boundary="${BOUNDARY}"`);
     responses[devId].push(res);
     res.on("close", () => {
@@ -43,7 +81,9 @@ const server = http.createServer((req, res) => {
     });
   } else {
     res.write(`<html>`);
-    Object.keys(sessions).forEach((id) => res.write(`<a href="/camera/${id}"><img src="/camera/${id}"/></a><hr/>`));
+    Object.keys(sessions).forEach((id) =>
+      res.write(`<h2>${id}</h2><a href="/ui/${id}"><img src="/camera/${id}"/></a><hr/>`),
+    );
     res.write(`</html>`);
     res.end();
   }
@@ -58,9 +98,10 @@ devEv.on("discover", (rinfo: RemoteInfo, dev: DevSerial) => {
 
   console.log(`discovered ${dev.devId} - ${rinfo.address}`);
   responses[dev.devId] = [];
+  audioResponses[dev.devId] = [];
   const s = makeSession(Handlers, dev, rinfo, startVideoStream, opts);
 
-  const withAudio = false;
+  const withAudio = true;
   const header = Buffer.from(`--${BOUNDARY}\r\nContent-Type: image/jpeg\r\n\r\n`);
 
   s.eventEmitter.on("frame", () => {
@@ -77,9 +118,14 @@ devEv.on("discover", (rinfo: RemoteInfo, dev: DevSerial) => {
     delete sessions[dev.devId];
   });
   if (withAudio) {
-    const audioFd = createWriteStream(`audio.pcm`);
-    s.eventEmitter.on("audio", (frame: Buffer) => {
-      audioFd.write(frame);
+    s.eventEmitter.on("audio", ({ gap, data }) => {
+      // ew, maybe WS?
+      var b64encoded = Buffer.from(data).toString("base64");
+      audioResponses[dev.devId].forEach((res) => {
+        res.write("data: ");
+        res.write(b64encoded);
+        res.write("\n\n");
+      });
     });
   }
   sessions[dev.devId] = s;