add audio playback

gcloud.json

@@ -0,0 +1,12 @@
+{
+  "type": "service_account",
+  "project_id": "crimata",
+  "private_key_id": "3a324fa3fcca4cea589e4b5007f4142db4b3a84d",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCxFJOjhIkYlyhn\nHjtpFC5EOwzNFZem6k6SS7zcgWbNLe9mcBsiTKCjpxItzkIo0pnT32yqw+AKsqpv\n6+LVyzzjfB7vtEUflvg6KgmYMS4ixOTCEbN3e2qj1t2RnITXZR2VGSGdoKKyqZm0\ntWrmexQvSN20cByL1vfxD3E69oYFGhVraa0zmm94WXHAzofT6DkvGdrbK1s1G9jI\nG3mdQ1jN71tghyloeHL+R/1iKQoZFYTkVe6yWvccSNAZelt/5sciCKRTCArXj89X\nLwU6jGOKIsyqx3BeUyWaSS59xpnGX3meSf5Q9U+j45ePUiqO6TTleF+kWiBwjUpp\nfeYcwEIDAgMBAAECggEAElkN9R7p75zV6F1XDYu0QYiWyncmx/o2Gu1zC6vywWa1\ni/korpSe/mX0ub9J1p3/p1balRUHlUQu6brYvYs56140fGTC1sOXQ7uQU+8glySs\niTk5TbOBeKluOsSrdP/6oTTB6Wm4AegVz4YOpgPxsvaLVkNZidnDxfvyIQhjGYsO\nsvRaw48eEmGOa1uvDhgvPOxiDp5YEdxqSue/7ylzjBdKecDyLgTSDDf/p+FwIx5B\n2d/0uRgQBaAk6D/DNOvN8GrYyakFmlDjE72nrBGAJq50UyV6THOSDk+8cK+2QiS1\nEDCFufj+7Dk1EBfBFUFY8ZBB/pphpEWyLoFl7NzGCQKBgQDaj/KOOcj6BkkTj1WM\nRW3KhOE3nhsCQEgQGCflpycEAZj6J7ktBG2/4teDt9sr+Epx8ymnQCZa9pmysdrD\nU0aWku1qUx0IUKTtZbAEbB+AnrWYDYkIWU8kRUUvNHNTBg43VyuohV9wprPih138\nHWMeNac82/XICz2A3rv+Xkx8KwKBgQDPaaEEGWVs3nIhf/p3M3/xWNKt8fhAERpG\n0YpTjgHW3FQFYjiCTTIcuLO3qsDGmfIAB9tL37s2Y3e2wKBGUj/o69Fr7sWVPdsX\nJtUc0+KiG2Bs0hsyZPdk9UQiQX8y2eq1NZW7vEoxWso1bM64WURB5N8ocWePNJJx\nL2Oe75rtiQKBgQDLfQQOiRxmNF3rOSMkAywyRr9NQgXRdbniSiszNQotP7OHDF7q\n29m2suOGfjIv4O6m4wdf8WkEfd4hsleETc9Ft6wVtyYrrLGxWWCk7WnzHVDjLY7s\n2AHIOjostf+9R8EKoz1BnFN8laibev71EQNMiBWZow1VX6m2hymurWs2mwKBgQCb\nuY3n2v14qOb92e1+U89KsEq1yMd/qpeU9jwqEaO14wS+aglNY5ItWEuuqWhFdE3q\n0ftHUzpnUnUOZD+xrI1JXsyEgegc7i0xi7lUBI3S8kUKTxGWW5IXXcKDCbPrxQtg\ndFPweSUnOyg4xnHKnVMPOjyGS+bZ8TnF+zOLoBAtKQKBgF6hAOIwF1DheXT4p8xS\nlSGhl8dBxjsJDVwbnFWQceGzGnq1e3ncy/UItt42UCDVRWTH4r7pQ9eKyd/3foUD\n6KDIcCrYo2KJh6ZNFJ67XF30U+TuEg5U/9jTbWqDOwYVXcMcnzQE4ZiGBxVHL6xr\nNc8FRW5+7xh4zNfzhGUGt5kt\n-----END PRIVATE KEY-----\n",
+  "client_email": "crimata-service-account@crimata.iam.gserviceaccount.com",
+  "client_id": "102914612880037864172",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://oauth2.googleapis.com/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/crimata-service-account%40crimata.iam.gserviceaccount.com"
+}

src/background/audio.ts

@@ -1,50 +1,104 @@
-"use strict";
 /* eslint @typescript-eslint/no-var-requires: "off" */
+
+"use strict";
+
 import { sendAudio } from './session'
 const portAudio = require('naudiodon');
 
-let ia: typeof portAudio.AudioIO;
-let audioInput: string[] = [];
-
-function processAudio(audioInput: Array<string>): Buffer {
-    let audio = '';
-    audioInput.forEach(chunk => audio += chunk);
-    return  Buffer.from(audio, "base64");
+let ai: typeof portAudio.AudioIO;
+let ao: typeof portAudio.AudioIO;
+let audioInput: Buffer[] = [];
+const encoding = "base64";
+const audioOptions = {
+    channelCount: 1,
+    sampleFormat: 16,
+    sampleRate: 16000,
+    deviceId: -1,
+    closeOnError: false,
 }
 
-export const updateRecorder = (_event, record: boolean): void => {
-  if (record) {
-      ia.resume();
-  } else {
-      ia.pause();
-      const audio = processAudio(audioInput);
-      sendAudio(audio);
-      audioInput = [];
+// run every time the main function launches.
+export const initAudioIO = () => {
+  // init portAudio readable stream once.
+  if (!ai) {
+    ai = new portAudio.AudioIO({ inOptions: audioOptions });
+
+    // base64 encoding needed for google speech to text.
+    ai.setEncoding(encoding);
+
+    // pause the portAudio data flow as soon as stream is initiated.
+    ai.start();
+    ai.pause();
+
+    ai.on('error', (e: Error) => {
+      console.log('Error recording audio', + e);
+    });
+    ai.on('data', (chunk: string) => {
+      // turn string base64 data into buffer object.
+      audioInput.push(Buffer.from(chunk, encoding));
+    });
+  }
+
+  // init portAudio writable stream once.
+  if (!ao) {
+      ao = new portAudio.AudioIO({ outOptions: audioOptions });
+      ao.start();
   }
 }
 
-export const initRecorder = () => {
-
-    if (!ia) {
-        ia = new portAudio.AudioIO({
-          inOptions: {
-            channelCount: 1,
-            sampleFormat: 16,
-            sampleRate: 16000,
-            deviceId: -1,
-            closeOnError: false,
-          }
-        });
-        ia.setEncoding('base64');
-        ia.start();
-        ia.pause();
-        ia.on('error', (e: Error) => {
-            console.log('Error recording audio', + e);
-        });
-        ia.on('data', (chunk: string) => {
-          audioInput.push(chunk);
-        });
+// run this function on space bar key up and down.
+export const updateRecorder = (_event, record: boolean): void => {
+    if (record) {
+        // resume mic data flow on space bar key down.
+        ai.resume();
+    } else {
+        // stop mic data flow on space bar key up.
+        ai.pause();
+        sendAudio(Buffer.concat(audioInput));
+        audioInput = [];
     }
 }
 
+// play audio buffers.
+export const play = (input: Array<Buffer>) => {
+    let i = 0;
+    // format buffers into half their size to account for writable highwaterMark.
+    const audio = bufSplit(input);
+    // call this fuction after last audio chunk has been written.
+    const callback = () => {
+        // TODO: clear portAudio writable buffer on write end.
+    }
+    write();
+    // iterate through audio array and write buffers to portAudio writable.
+    function write() {
+        let chunk: Buffer;
+        let ok = true;
+        do {
+            chunk = audio[i];
+            if (i === audio.length - 1) {
+                // write last chunk.
+                ao.write(chunk, null, callback);
+            } else {
+                // check for backpreassure.
+                ok = ao.write(chunk, null);
+            }
+            i++;
+        } while (i < audio.length && ok);
 
+        if (i < audio.length) {
+            // Had to stop early!
+            // Write some more once it drains.
+            ao.once('drain', write);
+        }
+    }
+}
+
+// utility function used by play func
+function bufSplit(input: Array<Buffer>): Array<Buffer> {
+    let result: Buffer[] = [];
+    input.forEach((b: Buffer) => {
+        // split buffer into two.
+        result.push(b.slice(0, b.length / 2), b.slice(b.length / 2, b.length));
+    });
+    return result;
+}

src/background/run.ts

@@ -2,7 +2,7 @@
 
 import { createWindow } from './window';
 import { initSession } from './session';
-import { initRecorder } from './audio';
+import { initAudioIO } from './audio';
 
 let socket = false;
 
@@ -23,6 +23,6 @@ export async function main() {
     socket = true;
 
     // Begin audio stream.
-    // initRecorder();
+    initAudioIO();
 
 }

src/background/session.ts

@@ -5,7 +5,10 @@ import { backgroundMitt } from './emitter';
 import { ipcMain } from "electron";
 
 interface RenderMessage {
-    content: string;
+    content: {
+        text: string;
+        audio: boolean | Buffer;
+    };
     avatar: string;
     context: string;
     subContext: string;
@@ -24,7 +27,7 @@ interface TextMessage {
     content: string;
 }
 
-const ip = 'ws://127.0.0.1';
+const ip = 'ws://127.0. 0.1';
 const port = 8760;
 const reconnectTimeout = 3000; //ms
 let socket: WebSocket;
@@ -67,6 +70,9 @@ const receiveMessage = (message: string): void => {
     }
 
     const parsed: RenderMessage = JSON.parse(message);
+    console.log('message received', parsed);
+    if (parsed.content.audio) {
+    }
     backgroundMitt.emit('ipc-renderer', {
         endpoint: 'render-message',
         message: parsed

src/components/messageView/index.vue

@@ -0,0 +1,111 @@
+<template>
+  <svg
+    version="1.1"
+    xmlns="http://www.w3.org/2000/svg"
+    xmlns:xlink="http://www.w3.org/1999/xlink"
+    :viewBox="scrollView"
+    @click="emittMessage"
+  >
+    <g id="messageView" transform="translate(0, 20)">
+      <g v-for="(item, index) in renderArr" :key="index" preserverAspectRatio="none">
+        <transition
+          appear
+          v-on:before-appear="beforeEnter"
+          v-on:appear="enter"
+          v-on:after-enter="afterEnter"
+        >
+          <TextBubble :item="item" :key="index" />
+        </transition>
+      </g>
+    </g>
+  </svg>
+</template>
+
+<script lang="ts">
+import TextBubble from "@/components/textBubble/index.vue";
+import { Message } from "@/types/message/index";
+import { useIpc } from "@/modules/ipc";
+import { Queue } from "@/modules/queue";
+import useTransitions from "./viewDetails/transitions";
+import useScrollView from "@/modules/scrollView";
+import useMitt from "@/modules/mitt";
+import {
+  defineComponent,
+  reactive,
+  onMounted,
+  onUnmounted,
+  watch,
+} from "vue";
+
+export default defineComponent({
+  name: "MessageView",
+  components: { TextBubble },
+
+  setup() {
+    const { emitter } = useMitt();
+    const renderArr: Message[] = reactive([]);
+    const renderQ = new Queue();
+
+    const { invoke } = useIpc("message");
+
+    // Each method will be called on their respective lifecycle event.
+    const { beforeEnter, enter, afterEnter, rendering } = useTransitions();
+    const { scrollView } = useScrollView({
+      targetId: "messageView",
+    });
+
+    const render = (m: Message) => {
+      if (rendering.value) {
+        renderQ.enqueue(m);
+      } else {
+        renderArr.push(m);
+      }
+    };
+
+    watch(rendering, (rendering, _prevRendering) => {
+      if (!rendering) {
+        const m = renderQ.dequeue() as Message | null;
+        if (m) renderArr.push(m);
+      }
+    });
+
+    onMounted(() => {
+      // Call render function on event "render-message."
+      window.ipcRenderer.on("render-message", (event, payload) => {
+        const m = payload.message;
+        if (m.content) render(m);
+      });
+
+      emitter.on("user-message", async (message) => {
+        try {
+          const res = await invoke(message);
+          render(res);
+        } catch (e) {
+          console.log("No response. Do not render.");
+        }
+      });
+    });
+
+    onUnmounted(() => {
+      window.ipcRenderer.removeAllListeners("render-message");
+      emitter.all.clear();
+    });
+
+    return {
+      renderArr,
+      beforeEnter,
+      enter,
+      afterEnter,
+      scrollView,
+    };
+  },
+});
+</script>
+
+<style lang="scss" scoped>
+#messageView {
+  width: 1000px;
+  height: 100vh;
+  color: red;
+}
+</style>

src/components/taInput/inputController.ts

@@ -34,7 +34,7 @@ export default function useInputController() {
         emitter.emit("animate-send");
 
         // send message to backend
-        post('message', message);
+        if (input) post('message', message);
     }
 
     const {

src/components/textBubble/bubbleDetails/drawer.ts

@@ -0,0 +1,50 @@
+import { ref, onMounted } from "vue";
+import { Ref } from "@/types/vueRef/index";
+
+/**
+ * handles visualization of individual svg text bubbles
+ */
+export default function useBubbleDrawer({
+    mr,
+    tr,
+}: {
+    mr: Ref<string>;
+    tr: Ref<string>;
+}) {
+    const signatureTranslate = ref("");
+    let messageRect: SVGSVGElement & SVGRectElement;
+    let messageText: SVGSVGElement & SVGTextElement;
+
+    onMounted(async () => {
+        messageRect = (mr.value as unknown) as SVGSVGElement & SVGRectElement;
+        messageText = (tr.value as unknown) as SVGSVGElement & SVGTextElement;
+    });
+
+    function adjustMessageRect(): void {
+        // must get dimensions of text box to fit bubble around
+        const padding = 12.5;
+        const textBox = messageText.getBBox();
+        messageRect.setAttribute("x", String(textBox.x - padding));
+        messageRect.setAttribute("y", String(textBox.y - padding));
+        messageRect.setAttribute("width", String(textBox.width + 2 * padding));
+        messageRect.setAttribute("height", String(textBox.height + 2 * padding));
+    }
+
+    function translateMsgSignature(): void {
+        if (messageRect.getAttribute("height") !== null) {
+            const height = Number(messageRect.getAttribute("height"));
+            signatureTranslate.value = `translate(-10 ${height})`;
+        }
+    }
+
+    async function drawMessage() {
+        // ignore lint error: await needed for proper render
+        await adjustMessageRect();
+        translateMsgSignature();
+    }
+
+    return {
+        drawMessage,
+        signatureTranslate,
+    };
+}

tests/audio.js

@@ -50,43 +50,90 @@ const ia = new portAudio.AudioIO({
     sampleRate: 16000,
     deviceId: -1,
     closeOnError: false,
-  }
+  },
 });
-ia.setEncoding('base64');
+// ia.setEncoding('base64');
 ia.start();
 ia.on('error', (e) => {
     console.log('error recording audio', + e);
 });
 ia.on('data', (chunk) => {
-  audioInput.push(chunk);
-  console.log('Got %d characters of string data:', chunk.length);
+    audioInput.push(chunk);
 });
 
+const ao = new portAudio.AudioIO({
+  outOptions: {
+    channelCount: 1,
+    sampleFormat: portAudio.SampleFormat16Bit,
+    sampleRate: 16000,
+    deviceId: -1, // Use -1 or omit the deviceId to select the default device
+    closeOnError: true // Close the stream if an audio error is detected, if set false then just log the error
+  }
+});
+
+ao.start();
+
+const callback = () => {
+    console.log('yayayay')
+}
+
+const play = (audio) => {
+
+    console.log('audio length', audio.length);
+    let i = 0;
+    write();
+
+    function write() {
+        let ok = true;
+         do {
+              const chunk = audio[i]
+                console.log(chunk, i)
+
+              i++;
+              if (i === audio.length - 1) {
+                // Last time!
+                ao.write(chunk, null, callback);
+              } else {
+                // See if we should continue, or wait.
+                // Don't pass the callback, because we're not done yet.
+                ok = ao.write(chunk, null);
+              }
+        } while (i < audio.length - 1 && ok);
+        if (i < audio.length - 1) {
+          // Had to stop early!
+          // Write some more once it drains.
+          ao.on('drain', write);
+        }
+    }
+
+}
+
 async function processAudio() {
-    audio = '';
-    audioInput.forEach(s => {
-        audio += s;
-    })
-    const buffer = Buffer.from(audio, 'base64');
+
+    const buf = Buffer.concat(audioInput);
+
+    play(audioInput);
+
     audioInput = [];
-    await transcribeSpeech({
-        content: buffer
-    });
+
+   transcribeSpeech({
+       content: buf
+   });
 }
 
 setTimeout(async () => {
     ia.pause();
-    await processAudio();
+    processAudio();
     // ia.resume();
 }, 3000);
 
 setTimeout(() => {
     ia.resume();
-}, 4000)
+}, 5000)
 
 setTimeout(async () => {
     ia.pause();
-    await processAudio();
+    processAudio();
     // ia.resume();
 }, 8000);
 

tests/server.js

@@ -9,8 +9,12 @@ wss.on("connection", function connection(ws, req) {
 
     console.log(message)
     if (message === 'no-token' || message === 'undefined') {
-      console.log('no token to authenticate');
       ws.send('success');
+    } else if (message instanceof Buffer) {
+      // audio message
+
+    } else {
+      const parsed = JSON.parse(message);
     }
 
   });