successfully imports nodeaudio, add yarnrc.yml for registry config

.yarnrc.yml

@@ -0,0 +1,4 @@
+npmScopes:
+  crimata:
+    npmRegistryServer: "https://gitlab.example.com/api/v4/projects/28849281/packages/npm/"
+    npmAuthToken: ${NPM_TOKEN}

package.json

@@ -1,5 +1,6 @@
 {
   "name": "Crimata",
+  "productName": "crimata-messenger",
   "version": "1.0.0-beta.0",
   "private": true,
   "description": "Cross-platform messenger application built with electron, vue3, and TS.",
@@ -14,7 +15,7 @@
   },
   "main": "background.js",
   "dependencies": {
-    "@google-cloud/speech": "^4.2.0",
+    "@crimata/nodeaudio": "^0.0.0",
     "@types/animejs": "^3.1.2",
     "@types/bindings": "^1.3.0",
     "@types/dom-mediacapture-record": "^1.0.7",
@@ -27,12 +28,9 @@
     "electron-is-dev": "^2.0.0",
     "electron-store": "^8.0.0",
     "electron-updater": "^4.3.8",
+    "iohook": "^0.9.3",
     "mitt": "^2.1.0",
-    "naudiodon": "^2.3.2",
-    "node-abi": "^2.30.0",
-    "node-record-lpcm16": "^1.0.1",
     "update-electron-app": "^2.0.1",
-    "uuid": "^8.3.2",
     "vue": "^3.0.0-0",
     "vue-router": "^4.0.0-0",
     "vuex": "^4.0.0-0",
@@ -67,7 +65,22 @@
     "spectron": "11.0.0",
     "typescript": "~3.9.3",
     "vue-cli-plugin-electron-builder": "~2.0.0-rc.6",
-    "vue-jest": "^5.0.0-0"
+    "vue-jest": "^5.0.0-0",
+    "webpack-node-externals": "^3.0.0"
+  },
+  "gitHooks": {
+    "pre-commit": "lint-staged"
+  },
+  "lint-staged": {
+    "*.{js,jsx,vue,ts,tsx}": [
+      "vue-cli-service lint",
+      "git add"
+    ]
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://gitlab.com/crimata/electron.git",
+    "release": "latest"
   },
   "vue": {
     "lintOnSave": false,
@@ -86,35 +99,5 @@
         }
       }
     }
-  },
-  "gitHooks": {
-    "pre-commit": "lint-staged"
-  },
-  "lint-staged": {
-    "*.{js,jsx,vue,ts,tsx}": [
-      "vue-cli-service lint",
-      "git add"
-    ]
-  },
-  "productName": "crimata-messenger",
-  "repository": {
-    "type": "git",
-    "url": "https://gitlab.com/crimata/electron-app.git",
-    "release": "latest"
-  },
-  "iohook": {
-    "targets": [
-      "node-83",
-      "electron-85"
-    ],
-    "platforms": [
-      "win32",
-      "darwin",
-      "linux"
-    ],
-    "arches": [
-      "x64",
-      "ia32"
-    ]
   }
 }

src/audio.ts

@@ -0,0 +1,74 @@
+const nodeAudio = require('@crimata/nodeaudio');
+import { globalShortcut } from "electron";
+import { backgroundMitt, ipcEmit } from '@/composables/useEmitter';
+
+let inputDevice: number;
+let outputDevice: number;
+
+let setStreamsId: ReturnType<typeof setTimeout>;
+
+let recording = false;
+const chunks: Int16Array[] = [];
+
+// backgroundMitt.on("data", (int16Arr: Int16Array) => {
+//     if (recording) {
+//         chunks.push(int16Arr);
+//         ipcEmit("recording", int16Arr); // for recording animation
+//     }
+// });
+
+// const setStreams = () => {
+
+//     const defaultInput = nodeAudio.GetDefaultInputDevice();
+//     const defaultOutput = nodeAudio.GetDefaultOutputDevice();
+
+//     if (inputDevice !== defaultInput) {
+//         inputDevice = defaultInput;
+//         nodeAudio.CloseInputStream(inputDevice);
+//         nodeAudio.OpenInputStream(inputDevice);
+//     }
+
+//     if (outputDevice !== defaultOutput) {
+//         outputDevice = defaultOutput;
+//         nodeAudio.CloseOutputStream(outputDevice);
+//         nodeAudio.OpenOutputStream(outputDevice);
+//     }
+// }
+
+// const startRecording = () => {
+//     recording = true;
+// }
+
+// const stopRecording = () => {
+//     recording = false;
+//     const audio: Int16Array = nodeAudio.MergeChunks(chunks);
+//     backgroundMitt.emit("audio", audio); // emit audio event
+//     chunks.length = 0;
+// }
+
+export const initAudio = () => {
+    // nodeAudio.Initialize(backgroundMitt.emit.bind(backgroundMitt));
+    // setStreamsId = setInterval(setStreams, 2000); // sense device 
+
+    const ret = globalShortcut.register('CommandOrControl+R', () => {
+        console.log('Toggle record...');
+    });
+
+    if (!ret) {
+        console.log('registration failed');
+    }
+
+    // Check whether a shortcut is registered.
+    console.log(globalShortcut.isRegistered('CommandOrControl+R'));
+
+}
+
+// export const playback = (audio: Int16Array) => {
+//     console.log("Playing audio");
+//     // nodeAudio.WriteToOutputStream(audio);
+// }
+
+// export const terminateAudio = () => {
+//     console.log("Terminating audio");
+//     // clearInterval(setStreamsId);
+// }

src/init.ts

@@ -5,11 +5,11 @@
 
 "use strict";
 
-import { app, protocol } from "electron";
+import { app, protocol, globalShortcut } from "electron";
 import createWindow from "./window";
 import main from "./main";
 import { backgroundMitt } from '@/composables/useEmitter';
-import {setWindowOpen, getWindowOpen } from './store';
+import { setWindowOpen, getWindowOpen } from './store';
 
 console.log('Starting Crimata electron app.');
 
@@ -30,6 +30,10 @@ app.on("ready", async () => {
     await main();
 });
 
+app.on("will-quit", () => {
+    globalShortcut.unregisterAll();
+});
+
 // Must keep to ensure app doesn't quit on close.
 app.on("before-quit", async () => {
 });

src/render/components/login.vue

@@ -122,7 +122,7 @@ export default defineComponent({
 .submit {
   font-family: "SF Compact Display";
   position: fixed;
-  margin-top: 141px;
+  margin-top: 260px;
   background-color: #58C4FD;
   color: white;
   padding: 10px 30px;

src/session.ts

@@ -6,6 +6,7 @@ import useWebsockets from "./composables/useWebsockets";
 import { backgroundMitt, ipcEmit } from "@/composables/useEmitter";
 import { Notification } from 'electron';
 import { getWindowFocus, getWindowOpen } from './store';
+import { initAudio, terminateAudio, playback } from "./audio";
 
 export const CONNECTION_STATUS = {
     connected: 'Connected',
@@ -14,24 +15,8 @@ export const CONNECTION_STATUS = {
     lost: 'Connection Lost',
 };
 
-function showNotification (options: {
-    title: string;
-    subtitle?: string;
-    body: string;
-}) {
-    new Notification(
-        {
-            title: options.title,
-            subtitle: options.subtitle,
-            body: options.body,
-        }).show()
-}
-
-// UI state
 const uiState = new UIState();
 
-// let audio = new Audio();
-
 const onMessageCallback = (payload: string) => {
 
     if (payload === "CLOSE_AUTH_FAIL") {
@@ -41,33 +26,42 @@ const onMessageCallback = (payload: string) => {
 
     const message = JSON.parse(payload) as ClientProtocol;
 
-    if (message.header === "init") {
+    if (message.header === "init") 
+    {
         uiState.set(message.body as Init);
-    } else {
+    } 
+    else
+    {
         const body = message.body as Update;
         uiState.update(body);
 
-        const isFocused = getWindowFocus();
-        const isOpen = getWindowOpen();
-        // show notification if window is not active
-        if (!isFocused || !isOpen) {
-            if (body.name === "add") {
-                const data = body.data as Message;
-                showNotification({
+        if (body.name === "add") 
+        {
+            const data = body.data as Message;
+
+            // Show a notification if new content and window is not focused.
+            if (!getWindowFocus())
+            {
+                new Notification({
                     title: 'New Message',
                     subtitle: data.context.text as string,
                     body: data.content[0].text,
-                });
+                })
+                .show();
             }
 
+            // Playback audio if message has audio.
+            // TODO: Should we worry about playback and record at the same time?
+            // if (data.content.audio)
+            // {
+            //     playback(data.content.audio);
+            // }
         }
     }
-
 }
 
 
 const connectionStatusCallback = (status: string) => {
-    // update icon
     // backgroundMitt.emit("update-icon-status", status);
     ipcEmit('set-connection-status', status);
 }
@@ -89,6 +83,14 @@ export const updateAppUI = (): void => {
     uiState.emit();
 }
 
+// Listen for audio recordings and send them to backend.
+backgroundMitt.on("audio", (audio: Int16Array) => {
+    sendMessage({
+        text: false,
+        audio: audio
+    } as Raw);
+});
+
 export function sendMessage(message: Raw | Request): void {
     send(message);
 }
@@ -99,13 +101,14 @@ export function launchSession(platformKey: string) {
     /* connect to the platform */
     connect(config.PLATFORM_URL, platformKey);
 
-    //  initialize the audio streams
-    // audio.record();
+    initAudio();
 
 }
 
 export function endSession() {
 
+    terminateAudio();
+
     close(1000, 'session-logout');
 
     uiState.reset();

src/tray.ts

@@ -4,8 +4,8 @@
 import fs from 'fs'
 import path from 'path'
 import { Tray } from 'electron';
-const nativeImage = require('electron').nativeImage
-import {NativeImage} from 'electron'
+const nativeImage = require('electron').nativeImage;
+import { NativeImage } from 'electron'
 import { backgroundMitt } from "@/composables/useEmitter";
 import { getIconState, setIconState } from "@/store";
 import { CONNECTION_STATUS } from '@/session';

src/types.ts

@@ -58,7 +58,7 @@ interface Annotation {
  */
 interface Raw {
     text: string | boolean;
-    audio: string | boolean;
+    audio: Int16Array | boolean;
 }
 
 /**