add speech to text transcription

src/background/run.ts

@@ -1,40 +1,34 @@
 import { createWindow } from './createWindow';
-import WebSocket from 'ws';
 import { Recorder } from './recorder';
 import { ipcMain, BrowserWindow } from "electron";
+import WebSocket from 'ws';
 import { windowEmitter } from './windowEmitter';
+const portAudio = require('naudiodon');
 
 /*
  * The main function will be run after electron app is ready.
  */
 export function main(): void {
 
+    let audioInput: string[] = [];
+
     // create main window.
-    let win: BrowserWindow | null;
-    win = createWindow({ width: 350, height: 525, resizable: false });
+    const win = createWindow({ width: 350, height: 525, resizable: false });
 
-    // Instantiate ws session with crimata-platorm.
-    const ws = new WebSocket('ws://localhost:8080');
-
-    // Instantiate portAudio recorder.
-    const recorder = new Recorder({
-        channelCount: 1,
-        sampleFormat: 16,
-        sampleRate: 16000,
-        deviceId: -1,
-        closeOnError: false,
-    }, 'binary');
+    // Instantiate socket session with crimata-platorm.
+    const socket = new WebSocket('socket://localhost:8080');
+    socket.binaryType = 'arraybuffer';
 
     const initSession = (): void => {
-        ws.send('hernandeze2@xavier.edu');
+        socket.send('hernandeze2@xavier.edu');
     }
 
     const sendMessage = (Event: any, payload: any): void => {
-        ws.send(JSON.stringify(payload));
+        socket.send(JSON.stringify(payload));
     }
 
-    const sendAudio = (data: string): void => {
-        ws.send(data);
+    const sendAudio = (data: Buffer): void => {
+        socket.send(data);
     }
 
     const receiveMessage = (message: string): void => {
@@ -45,39 +39,65 @@ export function main(): void {
       });
     }
 
-    const updateRecorder = (Event: any, record: boolean): void => {
+    const updateRecorder = async (Event: any, record: boolean): void => {
       if (record) {
-          recorder.resume();
+          ia.resume();
       } else {
-          recorder.pause();
-          const data = recorder.getData();
-          sendAudio(data);
+          ia.pause();
+          const audio = processAudio();
+          sendAudio(audio);
+          audioInput = [];
       }
     }
 
+    function processAudio(): Buffer {
+        let audio = '';
+        audioInput.forEach(s => {
+            audio += s;
+        });
+        const buffer = Buffer.from(audio, "base64");
+        return buffer;
+    }
+
     const windowMount = (): void => {
-        console.log('testing mount');
-        // ipc event handlers
         windowEmitter.emit('window-active', true);
+        // ipc event handlers
         ipcMain.on('update-recorder', updateRecorder);
         ipcMain.on('send-message', sendMessage);
     }
 
     const windowDismount = (): void => {
-        console.log('testing dismount');
-        // TODO: Gracefully close ws connection
-         win = null;
          windowEmitter.emit('window-active', false);
+        // TODO: Gracefully close socket connection
          ipcMain.removeListener('update-recorder', updateRecorder);
          ipcMain.removeListener('send-message', sendMessage);
     }
 
-    // ws handlers
-    ws.on('connect', initSession);
-    ws.on("message", receiveMessage);
+    // socket handlers
+    socket.on('connect', initSession);
+    socket.on("message", receiveMessage);
 
     // Handle window mount and dismount.
     win.webContents.on('did-finish-load', windowMount);
     win.on("closed", windowDismount);
 
+    let ia = new portAudio.AudioIO({
+      inOptions: {
+        channelCount: 1,
+        sampleFormat: 16,
+        sampleRate: 16000,
+        deviceId: -1,
+        closeOnError: false,
+      }
+    });
+    ia.setEncoding('base64');
+    ia.start();
+    ia.on('error', (e: Error) => {
+        console.log('error recording audio', + e);
+    });
+    ia.pause();
+    ia.on('data', (chunk: string) => {
+      audioInput.push(chunk);
+      console.log('Got %d characters of string data:', chunk.length);
+    });
 }