add base64 audio recorder

src/background.ts

@@ -5,20 +5,29 @@ import { createProtocol } from "vue-cli-plugin-electron-builder/lib";
 import installExtension, { VUEJS_DEVTOOLS } from "electron-devtools-installer";
 const isDevelopment = process.env.NODE_ENV !== "production";
 import * as path from "path";
-import { SpeechRecorder } from "speech-recorder";
 const fs = require('fs');
 
-// TODO: connect To Crimata-BE
+const portAudio = require('naudiodon');
+
+// Create an instance of AudioIO with inOptions (defaults are as below), which will return a ReadableStream
+const audioOptions = {
+  channelCount: 2,
+  sampleFormat: portAudio.SampleFormat16Bit,
+  sampleRate: 44100,
+  deviceId: -1, // Use -1 or omit the deviceId to select the default device
+  closeOnError: true // Close the stream if an audio error is detected, if set false then just log the error
+}
+
+let ai = new portAudio.AudioIO({
+  inOptions: audioOptions
+});
+
 import WebSocket from "ws";
-
-
-const recorder = new SpeechRecorder({ sampleRate: 16000, framesPerBuffer: 320, level: 0 });
-// const writeStream = fs.createWriteStream("audio.raw");
+const ws = new WebSocket("ws://localhost:8080");
 
 // Keep a global reference of the window object, if you don't, the window will
 // be closed automatically when the JavaScript object is garbage collected.
 let win: BrowserWindow | null;
-let ws: any;
 
 // Scheme must be registered before the app is ready
 protocol.registerSchemesAsPrivileged([
@@ -53,7 +62,6 @@ function createWindow() {
 
   win.webContents.on('did-finish-load', () => {
 
-    ws = new WebSocket("ws://localhost:8080");
     ws.on("open", function open() {
       // ws.send("hernandeze2@xavier.edu");
     });
@@ -67,44 +75,40 @@ function createWindow() {
       }
     });
 
-    ipcMain.on('send-message', (Event: any, message: any) => {
-      const stringMessage = JSON.stringify(message)
-      ws.send(stringMessage);
-    })
-
     ipcMain.on('update-menu-bar', (Event: any, payload: any) => {
     // win.setTitle("Listening...");
     })
 
-    let audioData: any[] = [];
+    ipcMain.on('send-message', (Event: any, payload: any) => {
+      ws.send(JSON.stringify(payload));
+    // win.setTitle("Listening...");
+    })
+
+    let audioData = "";
     ipcMain.on('update-recorder', (Event: any, record: boolean) => {
-     console.log('updating recorder', record)
      if (record) {
-       recorder.start({
-         onAudio: (audio: any) => {
-           audioData.push(audio.buffer)
-           // writeStream.write(audio);
-         }
-       })
+       ai.start();
+       ai.on('data', (buf: Buffer) => {
+          const base64Data = buf.toString('base64');
+          audioData += base64Data;
+       }); 
      } else {
-       recorder.stop()
-       // send audio to BE
+       ai.quit();
        const message = {
-         text: '',
+         text: "",
          audio: audioData
        }
        ws.send(JSON.stringify(message))
-       console.log('sent', message)
-
-       // clear audio.raw file
-       audioData = [];
+       ai =  new portAudio.AudioIO({
+         inOptions: audioOptions
+       })
+       audioData = "";
      }
     })
   })
 
   win.on("closed", () => {
     win = null;
-    ws = null;
   });
 }
 

src/core/UI/input/audio/useAudioRecorder.ts

@@ -17,13 +17,12 @@ export default function useAudioRecorder(record: Ref<boolean>, stream: MediaStre
   const bufferLen = 4096;
   const numChannels = 1;
 
-  window.AudioContext = window.AudioContext;
+  //window.AudioContext = window.AudioContext;
   const audioContext = new AudioContext({ sampleRate: 16000 });
 
   const source = audioContext.createMediaStreamSource(stream);
   const context = source.context;
   const sampleRate = context.sampleRate;
-  console.log(sampleRate)
 
   const node = context.createScriptProcessor.call(
     context,
@@ -47,12 +46,12 @@ export default function useAudioRecorder(record: Ref<boolean>, stream: MediaStre
       node.disconnect(context.destination);
 
       const message = {
-          text: '',
-          audio: {
-              sampleRate: sampleRate,
-              channelCount: numChannels,
-              audio: audioArray
-          }
+        text: '',
+        audio: {
+          sampleRate: sampleRate,
+          channelCount: numChannels,
+          audio: audioArray
+        }
       }
       window.postMessage({
         myTypeField: 'send-message',
@@ -65,4 +64,4 @@ export default function useAudioRecorder(record: Ref<boolean>, stream: MediaStre
 
   source.connect(node);
 
-}
+}

src/core/UI/input/audio/useStreamRecord.ts

@@ -1,4 +1,4 @@
-import {inject} from 'vue';
+import { inject } from 'vue';
 export default function useStreamRecord(): {
     initRecorder: (stream: MediaStream) => MediaRecorder;
 } {
@@ -9,23 +9,23 @@ export default function useStreamRecord(): {
         mimeType: "audio/webm; codecs=opus",
     }
 
-    console.log('ogg supported',MediaRecorder.isTypeSupported('audio/mpeg; '))
+    // console.log('ogg supported',MediaRecorder.isTypeSupported('audio/mpeg; '))
     const initRecorder = (stream: MediaStream) => {
 
-       const audioTrack = stream.getAudioTracks()[0]; 
-       const audioSettings = audioTrack.getSettings()
+        const audioTrack = stream.getAudioTracks()[0];
+        const audioSettings = audioTrack.getSettings()
 
         streamRecorder = new MediaRecorder(stream, options);
-        streamRecorder.ondataavailable = (e: any) => { 
-              if (e.data.size > 0) {
+        streamRecorder.ondataavailable = (e: any) => {
+            if (e.data.size > 0) {
                 chunks.push(e.data);
-              } else {
+            } else {
                 // ...
-              }
+            }
             //chunks.push(e.data) 
         }
         streamRecorder.onstop = async (e: any) => {
-            const audioBlob = new Blob(chunks, { 
+            const audioBlob = new Blob(chunks, {
                 type: options.mimeType
             });
             const arrayBuffer = await audioBlob.arrayBuffer()
@@ -41,8 +41,8 @@ export default function useStreamRecord(): {
                 }
             }
             window.postMessage({
-              myTypeField: 'send-message',
-              message: message
+                myTypeField: 'send-message',
+                message: message
             }, '*')
 
         }

src/core/UI/input/useInputController.ts

@@ -37,12 +37,10 @@ export default function useInputController() {
   function keyupHandler(e: any) {
     if (e.key === " " && visualizeStream.value) {
       console.log("stop recording");
-      // recorder.stop();
-
-      // window.postMessage({
-      //   myTypeField: 'update-recorder',
-      //   record: false
-      // }, '*')
+      window.postMessage({
+        myTypeField: 'update-recorder',
+        record: false
+      }, '*')
 
       visualizeStream.value = false;
       paths.value = []
@@ -73,12 +71,10 @@ export default function useInputController() {
     if (input === " " && visualizeStream.value === false) {
       console.log("record MediaStream");
       visualizeStream.value = true;
-      // recorder.start();
-
-      // window.postMessage({
-      //   myTypeField: 'update-recorder',
-      //   record: true
-      // }, '*')
+      window.postMessage({
+        myTypeField: 'update-recorder',
+        record: true
+      }, '*')
 
       expandBubble(audioBubbleWidth);
       visualizeStreamAsPaths(stream, paths, audioBubbleWidth);

src/core/UI/scrollView/useScrollView.ts

@@ -1,56 +1,57 @@
-import {ref, computed, onMounted} from "vue";
-export default function useScrollView({ targetId }: {targetId: string}) {
-    const scroll = ref(0);
-    let scrollTarget: HTMLElement | SVGElement | null;
-    const targetHeight = ref(0);
-    let topBound: number;
+import { ref, computed, onMounted } from "vue";
+export default function useScrollView({ targetId }: { targetId: string }) {
+  const scroll = ref(0);
+  let scrollTarget: HTMLElement | SVGElement | null;
+  const targetHeight = ref(0);
+  let topBound: number;
+  const scrollStartHeight = 445;
 
-    const clampedScroll = computed(() => {
-      if (targetHeight.value === 0) { return 0; }
-      topBound = -targetHeight.value + 450;
-      return Math.max(topBound, Math.min(scroll.value, 0)) as number;
-    })
+  const clampedScroll = computed(() => {
+    if (targetHeight.value === 0) { return 0; }
+    topBound = -targetHeight.value + 450;
+    return Math.max(topBound, Math.min(scroll.value, 0)) as number;
+  })
 
-    const scrollView = computed(() => {
-      if (clampedScroll.value === 0) {
-        scroll.value = 0;
-      } else if (clampedScroll.value === topBound) {
-        scroll.value = topBound;
-      }
-      return `0 ${clampedScroll.value} 350 500`;
-    });
-
-    const getHeight = () => {
-      if (scrollTarget) {
-        return scrollTarget.getBoundingClientRect().height as number;
-      }
+  const scrollView = computed(() => {
+    if (clampedScroll.value === 0) {
+      scroll.value = 0;
+    } else if (clampedScroll.value === topBound) {
+      scroll.value = topBound;
     }
+    return `0 ${clampedScroll.value} 350 500`;
+  });
 
-    const handleScrollEvent = (verticalScroll: number) => {
-        if (scrollTarget) {
-          // only scroll if renderer height is greater than main window
-          const heightResult = getHeight();
-          if (heightResult) {
-            if (heightResult > 500) { 
-              targetHeight.value = heightResult;
-              scroll.value += verticalScroll;
-            }
-          }
+  const getHeight = () => {
+    if (scrollTarget) {
+      return scrollTarget.getBoundingClientRect().height as number;
+    }
+  }
+
+  const handleScrollEvent = (verticalScroll: number) => {
+    if (scrollTarget) {
+      const heightResult = getHeight();
+      if (heightResult) {
+        // only scroll if renderer scrollStart
+        if (heightResult > scrollStartHeight) {
+          targetHeight.value = heightResult;
+          scroll.value += verticalScroll;
         }
-    }
-
-    onMounted(() => {
-      scrollTarget = document.getElementById(targetId);
-      if (scrollTarget) {
-        window.addEventListener("wheel", (e) => {
-          const verticalScroll = e.deltaY * 0.25;
-          handleScrollEvent(verticalScroll);
-        })
       }
-    });
-
-    return {
-      scrollView
     }
+  }
 
-}
+  onMounted(() => {
+    scrollTarget = document.getElementById(targetId);
+    if (scrollTarget) {
+      window.addEventListener("wheel", (e) => {
+        const verticalScroll = e.deltaY * 0.25;
+        handleScrollEvent(verticalScroll);
+      })
+    }
+  });
+
+  return {
+    scrollView
+  }
+
+}

src/core/messenger/message/messageDetails/useComputed.ts

@@ -49,7 +49,6 @@ export default function useMessageItemComp(m: string, id: string) {
         if (messageRenderer) {
             const rect = messageRenderer.getBoundingClientRect();
             let Y = 600;
-            console.log('renderer height',rect.height)
             if (rect.height > 500) {
                 Y = rect.height + messageMarginTop;
             }

src/core/messenger/messageView/useMessageView.ts

@@ -54,7 +54,6 @@ export default function useMessageView(): {
         // get message positioning & shift state before animating
         if (message) {
             messageOffset = calculateMessageOffsets(message);
-            console.log(messageOffset.y)
             shift = messageOffset.y > 400 ? true : false;
         }
         if (el) {