successfully imports nodeaudio, add yarnrc.yml for registry config

src/audio.ts

@@ -0,0 +1,74 @@
+const nodeAudio = require('@crimata/nodeaudio');
+import { globalShortcut } from "electron";
+import { backgroundMitt, ipcEmit } from '@/composables/useEmitter';
+
+let inputDevice: number;
+let outputDevice: number;
+
+let setStreamsId: ReturnType<typeof setTimeout>;
+
+let recording = false;
+const chunks: Int16Array[] = [];
+
+// backgroundMitt.on("data", (int16Arr: Int16Array) => {
+//     if (recording) {
+//         chunks.push(int16Arr);
+//         ipcEmit("recording", int16Arr); // for recording animation
+//     }
+// });
+
+// const setStreams = () => {
+
+//     const defaultInput = nodeAudio.GetDefaultInputDevice();
+//     const defaultOutput = nodeAudio.GetDefaultOutputDevice();
+
+//     if (inputDevice !== defaultInput) {
+//         inputDevice = defaultInput;
+//         nodeAudio.CloseInputStream(inputDevice);
+//         nodeAudio.OpenInputStream(inputDevice);
+//     }
+
+//     if (outputDevice !== defaultOutput) {
+//         outputDevice = defaultOutput;
+//         nodeAudio.CloseOutputStream(outputDevice);
+//         nodeAudio.OpenOutputStream(outputDevice);
+//     }
+// }
+
+// const startRecording = () => {
+//     recording = true;
+// }
+
+// const stopRecording = () => {
+//     recording = false;
+//     const audio: Int16Array = nodeAudio.MergeChunks(chunks);
+//     backgroundMitt.emit("audio", audio); // emit audio event
+//     chunks.length = 0;
+// }
+
+export const initAudio = () => {
+    // nodeAudio.Initialize(backgroundMitt.emit.bind(backgroundMitt));
+    // setStreamsId = setInterval(setStreams, 2000); // sense device 
+
+    const ret = globalShortcut.register('CommandOrControl+R', () => {
+        console.log('Toggle record...');
+    });
+
+    if (!ret) {
+        console.log('registration failed');
+    }
+
+    // Check whether a shortcut is registered.
+    console.log(globalShortcut.isRegistered('CommandOrControl+R'));
+
+}
+
+// export const playback = (audio: Int16Array) => {
+//     console.log("Playing audio");
+//     // nodeAudio.WriteToOutputStream(audio);
+// }
+
+// export const terminateAudio = () => {
+//     console.log("Terminating audio");
+//     // clearInterval(setStreamsId);
+// }

src/init.ts

@@ -5,11 +5,11 @@
 
 "use strict";
 
-import { app, protocol } from "electron";
+import { app, protocol, globalShortcut } from "electron";
 import createWindow from "./window";
 import main from "./main";
 import { backgroundMitt } from '@/composables/useEmitter';
-import {setWindowOpen, getWindowOpen } from './store';
+import { setWindowOpen, getWindowOpen } from './store';
 
 console.log('Starting Crimata electron app.');
 
@@ -30,6 +30,10 @@ app.on("ready", async () => {
     await main();
 });
 
+app.on("will-quit", () => {
+    globalShortcut.unregisterAll();
+});
+
 // Must keep to ensure app doesn't quit on close.
 app.on("before-quit", async () => {
 });

src/render/components/login.vue

@@ -122,7 +122,7 @@ export default defineComponent({
 .submit {
   font-family: "SF Compact Display";
   position: fixed;
-  margin-top: 141px;
+  margin-top: 260px;
   background-color: #58C4FD;
   color: white;
   padding: 10px 30px;

src/session.ts

@@ -6,6 +6,7 @@ import useWebsockets from "./composables/useWebsockets";
 import { backgroundMitt, ipcEmit } from "@/composables/useEmitter";
 import { Notification } from 'electron';
 import { getWindowFocus, getWindowOpen } from './store';
+import { initAudio, terminateAudio, playback } from "./audio";
 
 export const CONNECTION_STATUS = {
     connected: 'Connected',
@@ -14,24 +15,8 @@ export const CONNECTION_STATUS = {
     lost: 'Connection Lost',
 };
 
-function showNotification (options: {
-    title: string;
-    subtitle?: string;
-    body: string;
-}) {
-    new Notification(
-        {
-            title: options.title,
-            subtitle: options.subtitle,
-            body: options.body,
-        }).show()
-}
-
-// UI state
 const uiState = new UIState();
 
-// let audio = new Audio();
-
 const onMessageCallback = (payload: string) => {
 
     if (payload === "CLOSE_AUTH_FAIL") {
@@ -41,33 +26,42 @@ const onMessageCallback = (payload: string) => {
 
     const message = JSON.parse(payload) as ClientProtocol;
 
-    if (message.header === "init") {
+    if (message.header === "init") 
+    {
         uiState.set(message.body as Init);
-    } else {
+    } 
+    else
+    {
         const body = message.body as Update;
         uiState.update(body);
 
-        const isFocused = getWindowFocus();
-        const isOpen = getWindowOpen();
-        // show notification if window is not active
-        if (!isFocused || !isOpen) {
-            if (body.name === "add") {
-                const data = body.data as Message;
-                showNotification({
+        if (body.name === "add") 
+        {
+            const data = body.data as Message;
+
+            // Show a notification if new content and window is not focused.
+            if (!getWindowFocus())
+            {
+                new Notification({
                     title: 'New Message',
                     subtitle: data.context.text as string,
                     body: data.content[0].text,
-                });
+                })
+                .show();
             }
 
+            // Playback audio if message has audio.
+            // TODO: Should we worry about playback and record at the same time?
+            // if (data.content.audio)
+            // {
+            //     playback(data.content.audio);
+            // }
         }
     }
-
 }
 
 
 const connectionStatusCallback = (status: string) => {
-    // update icon
     // backgroundMitt.emit("update-icon-status", status);
     ipcEmit('set-connection-status', status);
 }
@@ -89,6 +83,14 @@ export const updateAppUI = (): void => {
     uiState.emit();
 }
 
+// Listen for audio recordings and send them to backend.
+backgroundMitt.on("audio", (audio: Int16Array) => {
+    sendMessage({
+        text: false,
+        audio: audio
+    } as Raw);
+});
+
 export function sendMessage(message: Raw | Request): void {
     send(message);
 }
@@ -99,13 +101,14 @@ export function launchSession(platformKey: string) {
     /* connect to the platform */
     connect(config.PLATFORM_URL, platformKey);
 
-    //  initialize the audio streams
-    // audio.record();
+    initAudio();
 
 }
 
 export function endSession() {
 
+    terminateAudio();
+
     close(1000, 'session-logout');
 
     uiState.reset();

src/tray.ts

@@ -4,8 +4,8 @@
 import fs from 'fs'
 import path from 'path'
 import { Tray } from 'electron';
-const nativeImage = require('electron').nativeImage
-import {NativeImage} from 'electron'
+const nativeImage = require('electron').nativeImage;
+import { NativeImage } from 'electron'
 import { backgroundMitt } from "@/composables/useEmitter";
 import { getIconState, setIconState } from "@/store";
 import { CONNECTION_STATUS } from '@/session';

src/types.ts

@@ -58,7 +58,7 @@ interface Annotation {
  */
 interface Raw {
     text: string | boolean;
-    audio: string | boolean;
+    audio: Int16Array | boolean;
 }
 
 /**