add base64 audio recorder

build/config.gypi

@@ -0,0 +1,79 @@
+# Do not edit. File was generated by node-gyp's "configure" step
+{
+  "target_defaults": {
+    "cflags": [],
+    "default_configuration": "Release",
+    "defines": [],
+    "include_dirs": [],
+    "libraries": []
+  },
+  "variables": {
+    "asan": 0,
+    "build_v8_with_gn": "false",
+    "coverage": "false",
+    "dcheck_always_on": 0,
+    "debug_nghttp2": "false",
+    "debug_node": "false",
+    "enable_lto": "false",
+    "enable_pgo_generate": "false",
+    "enable_pgo_use": "false",
+    "error_on_warn": "false",
+    "force_dynamic_crt": 0,
+    "host_arch": "x64",
+    "icu_data_in": "../../deps/icu-tmp/icudt67l.dat",
+    "icu_endianness": "l",
+    "icu_gyp_path": "tools/icu/icu-generic.gyp",
+    "icu_path": "deps/icu-small",
+    "icu_small": "false",
+    "icu_ver_major": "67",
+    "is_debug": 0,
+    "llvm_version": "0.0",
+    "napi_build_version": "6",
+    "node_byteorder": "little",
+    "node_debug_lib": "false",
+    "node_enable_d8": "false",
+    "node_install_npm": "true",
+    "node_module_version": 83,
+    "node_no_browser_globals": "false",
+    "node_prefix": "/",
+    "node_release_urlbase": "https://nodejs.org/download/release/",
+    "node_shared": "false",
+    "node_shared_brotli": "false",
+    "node_shared_cares": "false",
+    "node_shared_http_parser": "false",
+    "node_shared_libuv": "false",
+    "node_shared_nghttp2": "false",
+    "node_shared_openssl": "false",
+    "node_shared_zlib": "false",
+    "node_tag": "",
+    "node_target_type": "executable",
+    "node_use_bundled_v8": "true",
+    "node_use_dtrace": "true",
+    "node_use_etw": "false",
+    "node_use_node_code_cache": "true",
+    "node_use_node_snapshot": "true",
+    "node_use_openssl": "true",
+    "node_use_v8_platform": "true",
+    "node_with_ltcg": "false",
+    "node_without_node_options": "false",
+    "openssl_fips": "",
+    "openssl_is_fips": "false",
+    "shlib_suffix": "83.dylib",
+    "target_arch": "x64",
+    "v8_enable_31bit_smis_on_64bit_arch": 0,
+    "v8_enable_gdbjit": 0,
+    "v8_enable_i18n_support": 1,
+    "v8_enable_inspector": 1,
+    "v8_enable_pointer_compression": 0,
+    "v8_no_strict_aliasing": 1,
+    "v8_optimized_debug": 1,
+    "v8_promise_internal_field_count": 1,
+    "v8_random_seed": 0,
+    "v8_trace_maps": 0,
+    "v8_use_siphash": 1,
+    "want_separate_host_toolset": 0,
+    "xcode_version": "11.0",
+    "nodedir": "/Users/Enrique/Library/Caches/node-gyp/14.4.0",
+    "standalone_static_library": 1
+  }
+}

package.json

@@ -28,7 +28,7 @@
     "animejs": "^3.2.0",
     "core-js": "^3.6.5",
     "mitt": "^2.1.0",
-    "speech-recorder": "^1.2.1",
+    "naudiodon": "^2.3.2",
     "vue": "^3.0.0-0",
     "vue-router": "^4.0.0-0",
     "vuex": "^4.0.0-0",

src/background.ts

@@ -5,20 +5,29 @@ import { createProtocol } from "vue-cli-plugin-electron-builder/lib";
 import installExtension, { VUEJS_DEVTOOLS } from "electron-devtools-installer";
 const isDevelopment = process.env.NODE_ENV !== "production";
 import * as path from "path";
-import { SpeechRecorder } from "speech-recorder";
 const fs = require('fs');
 
-// TODO: connect To Crimata-BE
+const portAudio = require('naudiodon');
+
+// Create an instance of AudioIO with inOptions (defaults are as below), which will return a ReadableStream
+const audioOptions = {
+  channelCount: 2,
+  sampleFormat: portAudio.SampleFormat16Bit,
+  sampleRate: 44100,
+  deviceId: -1, // Use -1 or omit the deviceId to select the default device
+  closeOnError: true // Close the stream if an audio error is detected, if set false then just log the error
+}
+
+let ai = new portAudio.AudioIO({
+  inOptions: audioOptions
+});
+
 import WebSocket from "ws";
-
-
-const recorder = new SpeechRecorder({ sampleRate: 16000, framesPerBuffer: 320, level: 0 });
-// const writeStream = fs.createWriteStream("audio.raw");
+const ws = new WebSocket("ws://localhost:8080");
 
 // Keep a global reference of the window object, if you don't, the window will
 // be closed automatically when the JavaScript object is garbage collected.
 let win: BrowserWindow | null;
-let ws: any;
 
 // Scheme must be registered before the app is ready
 protocol.registerSchemesAsPrivileged([
@@ -53,7 +62,6 @@ function createWindow() {
 
   win.webContents.on('did-finish-load', () => {
 
-    ws = new WebSocket("ws://localhost:8080");
     ws.on("open", function open() {
       // ws.send("hernandeze2@xavier.edu");
     });
@@ -67,44 +75,40 @@ function createWindow() {
       }
     });
 
-    ipcMain.on('send-message', (Event: any, message: any) => {
-      const stringMessage = JSON.stringify(message)
-      ws.send(stringMessage);
-    })
-
     ipcMain.on('update-menu-bar', (Event: any, payload: any) => {
     // win.setTitle("Listening...");
     })
 
-    let audioData: any[] = [];
+    ipcMain.on('send-message', (Event: any, payload: any) => {
+      ws.send(JSON.stringify(payload));
+    // win.setTitle("Listening...");
+    })
+
+    let audioData = "";
     ipcMain.on('update-recorder', (Event: any, record: boolean) => {
-     console.log('updating recorder', record)
      if (record) {
-       recorder.start({
-         onAudio: (audio: any) => {
-           audioData.push(audio.buffer)
-           // writeStream.write(audio);
-         }
-       })
+       ai.start();
+       ai.on('data', (buf: Buffer) => {
+          const base64Data = buf.toString('base64');
+          audioData += base64Data;
+       }); 
      } else {
-       recorder.stop()
-       // send audio to BE
+       ai.quit();
        const message = {
-         text: '',
+         text: "",
          audio: audioData
        }
        ws.send(JSON.stringify(message))
-       console.log('sent', message)
-
-       // clear audio.raw file
-       audioData = [];
+       ai =  new portAudio.AudioIO({
+         inOptions: audioOptions
+       })
+       audioData = "";
      }
     })
   })
 
   win.on("closed", () => {
     win = null;
-    ws = null;
   });
 }
 

src/core/UI/input/audio/useAudioRecorder.ts

@@ -17,13 +17,12 @@ export default function useAudioRecorder(record: Ref<boolean>, stream: MediaStre
   const bufferLen = 4096;
   const numChannels = 1;
 
-  window.AudioContext = window.AudioContext;
+  //window.AudioContext = window.AudioContext;
   const audioContext = new AudioContext({ sampleRate: 16000 });
 
   const source = audioContext.createMediaStreamSource(stream);
   const context = source.context;
   const sampleRate = context.sampleRate;
-  console.log(sampleRate)
 
   const node = context.createScriptProcessor.call(
     context,
@@ -47,12 +46,12 @@ export default function useAudioRecorder(record: Ref<boolean>, stream: MediaStre
       node.disconnect(context.destination);
 
       const message = {
-          text: '',
-          audio: {
-              sampleRate: sampleRate,
-              channelCount: numChannels,
-              audio: audioArray
-          }
+        text: '',
+        audio: {
+          sampleRate: sampleRate,
+          channelCount: numChannels,
+          audio: audioArray
+        }
       }
       window.postMessage({
         myTypeField: 'send-message',
@@ -65,4 +64,4 @@ export default function useAudioRecorder(record: Ref<boolean>, stream: MediaStre
 
   source.connect(node);
 
-}
+}

src/core/UI/input/audio/useStreamRecord.ts

@@ -1,4 +1,4 @@
-import {inject} from 'vue';
+import { inject } from 'vue';
 export default function useStreamRecord(): {
     initRecorder: (stream: MediaStream) => MediaRecorder;
 } {
@@ -9,23 +9,23 @@ export default function useStreamRecord(): {
         mimeType: "audio/webm; codecs=opus",
     }
 
-    console.log('ogg supported',MediaRecorder.isTypeSupported('audio/mpeg; '))
+    // console.log('ogg supported',MediaRecorder.isTypeSupported('audio/mpeg; '))
     const initRecorder = (stream: MediaStream) => {
 
-       const audioTrack = stream.getAudioTracks()[0]; 
-       const audioSettings = audioTrack.getSettings()
+        const audioTrack = stream.getAudioTracks()[0];
+        const audioSettings = audioTrack.getSettings()
 
         streamRecorder = new MediaRecorder(stream, options);
-        streamRecorder.ondataavailable = (e: any) => { 
-              if (e.data.size > 0) {
+        streamRecorder.ondataavailable = (e: any) => {
+            if (e.data.size > 0) {
                 chunks.push(e.data);
-              } else {
+            } else {
                 // ...
-              }
+            }
             //chunks.push(e.data) 
         }
         streamRecorder.onstop = async (e: any) => {
-            const audioBlob = new Blob(chunks, { 
+            const audioBlob = new Blob(chunks, {
                 type: options.mimeType
             });
             const arrayBuffer = await audioBlob.arrayBuffer()
@@ -41,8 +41,8 @@ export default function useStreamRecord(): {
                 }
             }
             window.postMessage({
-              myTypeField: 'send-message',
-              message: message
+                myTypeField: 'send-message',
+                message: message
             }, '*')
 
         }

src/core/UI/input/useInputController.ts

@@ -37,12 +37,10 @@ export default function useInputController() {
   function keyupHandler(e: any) {
     if (e.key === " " && visualizeStream.value) {
       console.log("stop recording");
-      // recorder.stop();
-
-      // window.postMessage({
-      //   myTypeField: 'update-recorder',
-      //   record: false
-      // }, '*')
+      window.postMessage({
+        myTypeField: 'update-recorder',
+        record: false
+      }, '*')
 
       visualizeStream.value = false;
       paths.value = []
@@ -73,12 +71,10 @@ export default function useInputController() {
     if (input === " " && visualizeStream.value === false) {
       console.log("record MediaStream");
       visualizeStream.value = true;
-      // recorder.start();
-
-      // window.postMessage({
-      //   myTypeField: 'update-recorder',
-      //   record: true
-      // }, '*')
+      window.postMessage({
+        myTypeField: 'update-recorder',
+        record: true
+      }, '*')
 
       expandBubble(audioBubbleWidth);
       visualizeStreamAsPaths(stream, paths, audioBubbleWidth);

src/core/UI/scrollView/useScrollView.ts

@@ -1,56 +1,57 @@
-import {ref, computed, onMounted} from "vue";
-export default function useScrollView({ targetId }: {targetId: string}) {
-    const scroll = ref(0);
-    let scrollTarget: HTMLElement | SVGElement | null;
-    const targetHeight = ref(0);
-    let topBound: number;
+import { ref, computed, onMounted } from "vue";
+export default function useScrollView({ targetId }: { targetId: string }) {
+  const scroll = ref(0);
+  let scrollTarget: HTMLElement | SVGElement | null;
+  const targetHeight = ref(0);
+  let topBound: number;
+  const scrollStartHeight = 445;
 
-    const clampedScroll = computed(() => {
-      if (targetHeight.value === 0) { return 0; }
-      topBound = -targetHeight.value + 450;
-      return Math.max(topBound, Math.min(scroll.value, 0)) as number;
-    })
+  const clampedScroll = computed(() => {
+    if (targetHeight.value === 0) { return 0; }
+    topBound = -targetHeight.value + 450;
+    return Math.max(topBound, Math.min(scroll.value, 0)) as number;
+  })
 
-    const scrollView = computed(() => {
-      if (clampedScroll.value === 0) {
-        scroll.value = 0;
-      } else if (clampedScroll.value === topBound) {
-        scroll.value = topBound;
-      }
-      return `0 ${clampedScroll.value} 350 500`;
-    });
-
-    const getHeight = () => {
-      if (scrollTarget) {
-        return scrollTarget.getBoundingClientRect().height as number;
-      }
+  const scrollView = computed(() => {
+    if (clampedScroll.value === 0) {
+      scroll.value = 0;
+    } else if (clampedScroll.value === topBound) {
+      scroll.value = topBound;
     }
+    return `0 ${clampedScroll.value} 350 500`;
+  });
 
-    const handleScrollEvent = (verticalScroll: number) => {
-        if (scrollTarget) {
-          // only scroll if renderer height is greater than main window
-          const heightResult = getHeight();
-          if (heightResult) {
-            if (heightResult > 500) { 
-              targetHeight.value = heightResult;
-              scroll.value += verticalScroll;
-            }
-          }
+  const getHeight = () => {
+    if (scrollTarget) {
+      return scrollTarget.getBoundingClientRect().height as number;
+    }
+  }
+
+  const handleScrollEvent = (verticalScroll: number) => {
+    if (scrollTarget) {
+      const heightResult = getHeight();
+      if (heightResult) {
+        // only scroll if renderer scrollStart
+        if (heightResult > scrollStartHeight) {
+          targetHeight.value = heightResult;
+          scroll.value += verticalScroll;
         }
-    }
-
-    onMounted(() => {
-      scrollTarget = document.getElementById(targetId);
-      if (scrollTarget) {
-        window.addEventListener("wheel", (e) => {
-          const verticalScroll = e.deltaY * 0.25;
-          handleScrollEvent(verticalScroll);
-        })
       }
-    });
-
-    return {
-      scrollView
     }
+  }
 
-}
+  onMounted(() => {
+    scrollTarget = document.getElementById(targetId);
+    if (scrollTarget) {
+      window.addEventListener("wheel", (e) => {
+        const verticalScroll = e.deltaY * 0.25;
+        handleScrollEvent(verticalScroll);
+      })
+    }
+  });
+
+  return {
+    scrollView
+  }
+
+}

src/core/messenger/message/messageDetails/useComputed.ts

@@ -49,7 +49,6 @@ export default function useMessageItemComp(m: string, id: string) {
         if (messageRenderer) {
             const rect = messageRenderer.getBoundingClientRect();
             let Y = 600;
-            console.log('renderer height',rect.height)
             if (rect.height > 500) {
                 Y = rect.height + messageMarginTop;
             }

src/core/messenger/messageView/useMessageView.ts

@@ -54,7 +54,6 @@ export default function useMessageView(): {
         // get message positioning & shift state before animating
         if (message) {
             messageOffset = calculateMessageOffsets(message);
-            console.log(messageOffset.y)
             shift = messageOffset.y > 400 ? true : false;
         }
         if (el) {

testServer.js

@@ -34,10 +34,7 @@ const wss = new WebSocket.Server({
 wss.on("connection", function connection(ws, req) {
   ws.on("message", function incoming(message) {
     const parsed = JSON.parse(message)
-    if (parsed.audio) {
-      console.log(parsed.audio)
-    }
-    console.log("received: %s:", parsed);
+      console.log(parsed)
   });
 
   const ip = req.socket.remoteAddress;

yarn.lock

@@ -2903,7 +2903,7 @@ binaryen@^85.0.0:
   resolved "https://registry.yarnpkg.com/binaryen/-/binaryen-85.0.0.tgz#72c385d558862e52b75918bbce748d4627a98499"
   integrity sha512-01z4ZvA1O1hhNH+R4QQHuqzslecdHyzHYeuPMm4rB3hEiCP6BQlXZhHs68FgKfKT199YOt1uy5bmAIsilaOQ9g==
 
-bindings@^1.3.0, bindings@^1.5.0:
+bindings@^1.2.1, bindings@^1.5.0:
   version "1.5.0"
   resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.5.0.tgz#10353c9e945334bc0511a6d90b38fbc7c9c504df"
   integrity sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==
@@ -8742,7 +8742,7 @@ mz@^2.4.0:
     object-assign "^4.0.1"
     thenify-all "^1.0.0"
 
-nan@^2.12.1, nan@^2.13.2, nan@^2.14.1:
+nan@^2.12.1, nan@^2.13.2, nan@^2.14.0:
   version "2.14.2"
   resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.2.tgz#f5376400695168f4cc694ac9393d0c9585eeea19"
   integrity sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ==
@@ -8769,6 +8769,14 @@ natural-compare@^1.4.0:
   resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7"
   integrity sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=
 
+naudiodon@^2.3.2:
+  version "2.3.2"
+  resolved "https://registry.yarnpkg.com/naudiodon/-/naudiodon-2.3.2.tgz#e1b14902e830e0140dcbfd80d81f7cf75a777c79"
+  integrity sha512-ugxUwZNs9mvY4p4bhKLftX+06cYDN6XLjcXZiW3B72Mzce63OXmynSemeUaCqLpTXQRipyaySc61dNwtLrxfng==
+  dependencies:
+    bindings "^1.5.0"
+    segfault-handler "^1.3.0"
+
 negotiator@0.6.2:
   version "0.6.2"
   resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.2.tgz#feacf7ccf525a77ae9634436a64883ffeca346fb"
@@ -10863,6 +10871,14 @@ scss-tokenizer@^0.2.3:
     js-base64 "^2.1.8"
     source-map "^0.4.2"
 
+segfault-handler@^1.3.0:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/segfault-handler/-/segfault-handler-1.3.0.tgz#054bc847832fa14f218ba6a79e42877501c8870e"
+  integrity sha512-p7kVHo+4uoYkr0jmIiTBthwV5L2qmWtben/KDunDZ834mbos+tY+iO0//HpAJpOFSQZZ+wxKWuRo4DxV02B7Lg==
+  dependencies:
+    bindings "^1.2.1"
+    nan "^2.14.0"
+
 select-hose@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/select-hose/-/select-hose-2.0.0.tgz#625d8658f865af43ec962bfc376a37359a4994ca"
@@ -11282,16 +11298,6 @@ spectron@^11.0.0:
     split "^1.0.0"
     webdriverio "^6.1.20"
 
-speech-recorder@^1.2.1:
-  version "1.2.1"
-  resolved "https://registry.yarnpkg.com/speech-recorder/-/speech-recorder-1.2.1.tgz#db050c39db951db099ae3ee07f4eba574faab770"
-  integrity sha512-kILQF+5ZS6XNgZo6zul+6PPMSYbryPaLs5qWLnjR2KtbCZGxNwuuPTiMaJYvvQRdFmJlJOSgsGylZYk1sAwR4Q==
-  dependencies:
-    bindings "^1.3.0"
-    nan "^2.14.1"
-    uuid "^3.3.3"
-    webrtcvad "^1.0.1"
-
 split-string@^3.0.1, split-string@^3.0.2:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/split-string/-/split-string-3.1.0.tgz#7cb09dda3a86585705c64b39a6466038682e8fe2"
@@ -12445,7 +12451,7 @@ utils-merge@1.0.1:
   resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713"
   integrity sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=
 
-uuid@^3.3.2, uuid@^3.3.3, uuid@^3.4.0:
+uuid@^3.3.2, uuid@^3.4.0:
   version "3.4.0"
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee"
   integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==
@@ -12850,14 +12856,6 @@ webpack@^4.0.0, webpack@^4.18.0:
     watchpack "^1.7.4"
     webpack-sources "^1.4.1"
 
-webrtcvad@^1.0.1:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/webrtcvad/-/webrtcvad-1.0.1.tgz#862b35c4ed2e7de3f39b268f48ef890ac85e02ce"
-  integrity sha512-oLfReCmGMpRducFWKP+o0GpKZLPj0u6qkln3P7wGaEzyjxtBiFuzvK28pmKF5SSihEkC1RGKO7Pi9C4VfY1q4Q==
-  dependencies:
-    bindings "^1.3.0"
-    node-addon-api "^1.7.1"
-
 websocket-driver@0.6.5:
   version "0.6.5"
   resolved "https://registry.yarnpkg.com/websocket-driver/-/websocket-driver-0.6.5.tgz#5cb2556ceb85f4373c6d8238aa691c8454e13a36"