add audioRenderer + ipc event listeners

src/background.ts

@@ -1,6 +1,6 @@
 "use strict";
 
-import { app, protocol, BrowserWindow } from "electron";
+import { app, protocol, BrowserWindow, ipcMain } from "electron";
 import { createProtocol } from "vue-cli-plugin-electron-builder/lib";
 import installExtension, { VUEJS_DEVTOOLS } from "electron-devtools-installer";
 const isDevelopment = process.env.NODE_ENV !== "production";
@@ -56,7 +56,22 @@ function createWindow() {
     win.loadURL("app://./index.html");
   }
 
-  // win.webContents.on('did-finish-load', () => {
+
+  win.webContents.on('did-finish-load', () => {
+    // ipcMain.on('audio-stream', (Event: any, Payload: any) => {
+    //   if (Payload.stream) {
+    //     console.log('begin streaming');
+    //     recorder.start({
+    //       onAudio: (audio: object, speech: boolean) => {
+    //         console.log(audio)
+    //       }
+    //     });
+    //   } else {
+    //     recorder.stop();
+    //     console.log('stop streaming')
+    //   }
+    // })
+
     // if (win) {
     //   recorder.start({
     //     onAudio: (audio: object, speech: boolean) => {
@@ -75,7 +90,7 @@ function createWindow() {
     //     }
     //   });
     // }
-  // })
+  })
 
   win.on("closed", () => {
     win = null;

src/components/UI/UIDetails/inputItem.vue

@@ -11,31 +11,36 @@
 <script lang="ts">
 import useInputRenderer from "@/composables/inputItem/useInputRenderer";
 import useStreamRecorder from "@/composables/streamRecorder/useStreamRecorder";
-import {
+import { defineComponent, watch, onMounted, ref } from "vue";
-  defineComponent,
-  watch,
-  onMounted
-} from "vue";
 export default defineComponent({
   name: "InputItem",
   setup() {
+    const record = ref(false);
     const { inputXoffset, input } = useInputRenderer();
-    const { initRecorder } = useStreamRecorder();
+    const { initRecorder } = useStreamRecorder(record);
 
     onMounted(async () => {
-
       // step get streamRecorder reference
       const { streamRecorder } = await initRecorder();
 
       // add event listener for space keyup
       window.addEventListener("keyup", (e) => {
         if (e.key === " " && streamRecorder.state === "recording") {
-          console.log("stop recording")
+          console.log("stop recording");
+          record.value = false;
           streamRecorder.stop();
-          console.log('recorder is', streamRecorder.state);
+
+          // window.postMessage(
+          //   {
+          //     myTypeField: "audio-stream",
+          //     stream: false,
+          //   },
+          //   "*"
+          // );
+          console.log("recorder is", streamRecorder.state);
           input.value = "";
         }
-      })
+      });
 
       // begin recording on space
       watch(input, (input, prevInput) => {
@@ -43,12 +48,21 @@ export default defineComponent({
           return;
         }
         if (input === " ") {
-          console.log("record")
+          console.log("record");
+          record.value = true;
+
+          // window.postMessage(
+          //   {
+          //     myTypeField: "audio-stream",
+          //     stream: true,
+          //   },
+          //   "*"
+          // );
           streamRecorder.start();
-          console.log('recorder is', streamRecorder.state);
+          console.log("recorder is", streamRecorder.state);
         }
-      })
+      });
-    })
+    });
 
     return {
       input,

src/composables/audioRenderer/useAudioRenderer.ts

@@ -1,37 +1,22 @@
-export default function useAudioRenderer() {
+export default function useAudioRenderer(stream: MediaStream) {
     const audioCtx = new (window.AudioContext)();
     const analyzer = audioCtx.createAnalyser();
+    const source = audioCtx.createMediaStreamSource(stream);
+    source.connect(analyzer);
+
     analyzer.fftSize = 2048;
     const xInitial = 24;
     const xFinal = 640;
-    // analyzer.minDecibels = -90;
+    analyzer.minDecibels = -90;
     analyzer.smoothingTimeConstant = 0.85;
     const dataArray = new Uint8Array(analyzer.frequencyBinCount);
-    const sampleRate = 16000;
+    // const sampleRate = 16000;
 
-    function convertBlock(buffer: ArrayBuffer): Float32Array {
+    const renderAudio = (audio: Float32Array, sampleRate: number) => {
-        // incoming data is an ArrayBuffer
-        const incomingData = new Uint8Array(buffer); // create a uint8 view on the ArrayBuffer
-        const l = incomingData.length; // length, we need this for the loop
-        const outputData = new Float32Array(incomingData.length); // create the Float32Array for output
-        for (let i = 0; i < l; i++) {
-            outputData[i] = (incomingData[i] - 128) / 128.0; // convert audio to float
-        }
-        return outputData; // return the Float32Array
-    }
-
-    const getAudioFrequencyData = () => {
-
-    }
-
-    const renderAudio = (payload) => {
-
-        // step 1: convert audio from Uint8 to Float32Array
-        const floatArray = convertBlock(payload.audio.buffer);
 
         // step 2: source audio frequency {ffft} data
-        const buffer = audioCtx.createBuffer(1, floatArray.length, sampleRate);
+        const buffer = audioCtx.createBuffer(1, audio.length, sampleRate);
-        buffer.copyToChannel(floatArray, 0, 0);
+        buffer.copyToChannel(audio, 0, 0);
         const source = audioCtx.createBufferSource();
         source.buffer = buffer;
         source.connect(analyzer);
@@ -44,7 +29,7 @@ export default function useAudioRenderer() {
         const average = sum / dataView.length;
         const scaled = average / 255;
         const zeroed = scaled - 0.6;
-        return zeroed;
+        return average;
     }
 
     return {

src/composables/streamRecorder/useStreamRecorder.ts

@@ -1,12 +1,26 @@
-export default function useStreamRecorder() {
+import { watch } from "vue";
+import useAudioRenderer from "@/composables/audioRenderer/useAudioRenderer";
+export default function useStreamRecorder(record: any) {
     let streamRecorder: MediaRecorder;
     const chunks: Blob[] = [];
+
+    window.AudioContext = window.AudioContext;
+    let audio_context = new AudioContext();
+    const analyzer = audio_context.createAnalyser();
+    analyzer.fftSize = 256;
+    var bufferLengthAlt = analyzer.frequencyBinCount;
+    var dataArrayAlt = new Uint8Array(bufferLengthAlt);
+
+    let source;
+
+
     const initRecorder = async () => {
         const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+        handleStream(stream);
         streamRecorder = new MediaRecorder(stream);
         streamRecorder.ondataavailable = (e: any) => { chunks.push(e.data) }
         streamRecorder.onstop = (e: any) => {
-             const audioBlob = new Blob(chunks, {'type': 'audio/ogg; codecs=opus'} );
+            const audioBlob = new Blob(chunks, { 'type': 'audio/ogg; codecs=opus' });
             // do something with blob
         }
         return {
@@ -14,6 +28,73 @@ export default function useStreamRecorder() {
         }
     }
 
+    interface Window {
+        AudioContext: any;
+        webkitAudioContext: any;
+    }
+
+    interface Event {
+        inputBuffer: AudioBuffer;
+    }
+
+    let bufferLen = 4096;
+    let numChannels = 1;
+    function visualize() {
+
+        function drawAlt() {
+
+            const drawVisual = requestAnimationFrame(drawAlt);
+            analyzer.getByteFrequencyData(dataArrayAlt);
+
+            for (var i = 0; i < bufferLengthAlt; i++) {
+                console.log(dataArrayAlt[i])
+            }
+        }
+        drawAlt();
+    }
+
+    function handleStream(stream: MediaStream) {
+        source = audio_context.createMediaStreamSource(stream);
+        source.connect(analyzer);
+        visualize();
+        //
+        // const context = source.context;
+        // const sampleRate = context.sampleRate;
+
+        // const node = context.createScriptProcessor.call(
+        //     context,
+        //     bufferLen,
+        //     numChannels,
+        //     numChannels
+        // );
+
+        // source.connect(node);
+
+        // node.onaudioprocess = (e: Event) => {
+        //     const inputBuffer = new Float32Array(bufferLen);
+
+        //     e.inputBuffer.copyFromChannel(inputBuffer, 0);
+
+        //     // const int16 = new Int16Array(inputBuffer.buffer);
+
+        //     // window.postMessage(
+        //     //     {
+        //     //         myTypeField: "handle-audio-stream",
+        //     //         audioData: int16
+        //     //     },
+        //     //     "*"
+        //     // );
+        // };
+
+        // watch(record, (rec) => {
+        //     if (rec) {
+        //         node.connect(context.destination);
+        //     } else {
+        //         node.disconnect(context.destination);
+        //     }
+        // })
+    }
+
     return {
         initRecorder
     }

src/preload.ts

@@ -6,3 +6,15 @@ interface Window {
   ipcRenderer: typeof ipcRenderer;
 }
 window.ipcRenderer = ipcRenderer;
+
+process.once("loaded", () => {
+  window.addEventListener("message", event => {
+    // do something with custom event
+    const message = event.data;
+
+    if (message.myTypeField === "audio-stream") {
+      ipcRenderer.send("audio-stream", message);
+    }
+  });
+});
+