add audio worklet

package.json

@@ -66,25 +66,9 @@
     "spectron": "11.0.0",
     "typescript": "~3.9.3",
     "vue-cli-plugin-electron-builder": "~2.0.0-rc.6",
-    "vue-jest": "^5.0.0-0"
-  },
-  "vue": {
-    "lintOnSave": false,
-    "pluginOptions": {
-      "electronBuilder": {
-        "mainProcessFile": "./src/init.ts",
-        "rendererProcessFile": "./src/render/main.ts",
-        "preload": "./src/render/preload.ts",
-        "builderOptions": {
-          "appId": "com.crimata.ElectronUpdaterApp",
-          "artifactName": "${productName}-${version}.${ext}",
-          "publish": {
-            "provider": "generic",
-            "url": "https://gitlab.com/api/v4/projects/25637892/jobs/artifacts/main/raw/dist_electron?job=build"
-          }
-        }
-      }
-    }
+    "vue-jest": "^5.0.0-0",
+    "worker-loader": "^3.0.8",
+    "worklet-loader": "^1.0.0"
   },
   "gitHooks": {
     "pre-commit": "lint-staged"

public/processor.js

@@ -0,0 +1,12 @@
+
+class AudioProcessor extends AudioWorkletProcessor {
+  process (inputs, outputs, parameters) {
+    console.log(inputs);
+    console.log(outputs);
+    console.log(parameters);
+    return true
+  }
+}
+
+registerProcessor('processor', AudioProcessor)
+

public/worklet/audioProcessor.ts

@@ -0,0 +1,33 @@
+interface AudioWorkletProcessor {
+  readonly port: MessagePort;
+  process(
+    inputs: Float32Array[][],
+    outputs: Float32Array[][],
+    parameters: Record<string, Float32Array>
+  ): boolean;
+}
+
+declare let AudioWorkletProcessor: {
+  prototype: AudioWorkletProcessor;
+  new (options?: AudioWorkletNodeOptions): AudioWorkletProcessor;
+};
+
+declare function registerProcessor(
+  name: string,
+  processorCtor: (new (
+    options?: AudioWorkletNodeOptions
+  ) => AudioWorkletProcessor) & {
+    parameterDescriptors?: AudioParamDescriptor[];
+  }
+): undefined;
+
+class AudioProcessor extends AudioWorkletProcessor {
+  process (inputs: Float32Array[][], outputs: Float32Array[][], parameters: Record<string, Float32Array>) {
+    console.log(inputs);
+    console.log(outputs);
+    console.log(parameters);
+    return true
+  }
+}
+
+registerProcessor('audio-processor', AudioProcessor)

src/ipc/handlers.ts

@@ -1,7 +1,7 @@
 
 "use strict";
 
-import { accountLogin, accountLogout, accountProfile } from "@/account";
+import { accountLogin, accountLogout } from "@/account";
 import { IpcHandler } from "@/composables/useIpcMain";
 import { flush } from "@/audio";
 

src/render/audio/audio.worklet.ts

@@ -0,0 +1,33 @@
+interface AudioWorkletProcessor {
+  readonly port: MessagePort;
+  process(
+    inputs: Float32Array[][],
+    outputs: Float32Array[][],
+    parameters: Record<string, Float32Array>
+  ): boolean;
+}
+
+declare let AudioWorkletProcessor: {
+  prototype: AudioWorkletProcessor;
+  new (options?: AudioWorkletNodeOptions): AudioWorkletProcessor;
+};
+
+declare function registerProcessor(
+  name: string,
+  processorCtor: (new (
+    options?: AudioWorkletNodeOptions
+  ) => AudioWorkletProcessor) & {
+    parameterDescriptors?: AudioParamDescriptor[];
+  }
+): undefined;
+
+class AudioProcessor extends AudioWorkletProcessor {
+  process (inputs: Float32Array[][], outputs: Float32Array[][], parameters: Record<string, Float32Array>) {
+    console.log(inputs);
+    console.log(outputs);
+    console.log(parameters);
+    return true
+  }
+}
+
+registerProcessor('audio-processor', AudioProcessor)

src/render/audio/audioNode.ts

@@ -0,0 +1,9 @@
+
+// MyWorkletNode.js
+export default class MyWorkletNode extends AudioWorkletNode {
+   constructor(context) {
+    super(context, '-processor')
+    console.log(this.channelCount)
+  }
+}
+

src/render/audio/setupAudio.ts

@@ -0,0 +1,74 @@
+import AudioProcessor from "./audio.worklet.ts";
+console.log('YAYAYAYYA', AudioProcessor)
+//TODO: need to set export directory for audio worklet
+// webpack!!!
+
+ const constraints = {
+      audio: {
+        echoCancellation: true,
+        autoGainControl: true,
+        noiseSuppression: true,
+        channelCount: 1
+      },
+      video: false
+    }
+async function getWebAudioMediaStream() {
+  if (!window.navigator.mediaDevices) {
+    throw new Error(
+      "This browser does not support web audio or it is not enabled."
+    );
+  }
+
+  try {
+    const result = await window.navigator.mediaDevices.getUserMedia(constraints);
+
+    return result;
+  } catch (e) {
+    switch (e.name) {
+      case "NotAllowedError":
+        throw new Error(
+          "A recording device was found but has been disallowed for this application. Enable the device in the browser settings."
+        );
+
+      case "NotFoundError":
+        throw new Error(
+          "No recording device was found. Please attach a microphone and click Retry."
+        );
+
+      default:
+        throw e;
+    }
+  }
+}
+
+export async function setupAudio() {
+  // Get the browser audio. Awaits user "allowing" it for the current tab.
+  const mediaStream = await getWebAudioMediaStream();
+
+  const context = new window.AudioContext();
+  const audioSource = context.createMediaStreamSource(mediaStream);
+
+  let node;
+
+    // Add our audio processor worklet to the context.
+
+    try {
+        await context.audioWorklet.addModule("audio.worklet.js");
+    } catch (e) {
+      throw new Error(
+        `Failed to load audio analyzer worklet. Further info: ${e.message}`
+      );
+    }
+
+    node = new AudioWorkletNode(context, 'audio-processor')
+
+    // Connect the audio source (microphone output) to our analysis node.
+    audioSource.connect(node);
+
+    // Connect our analysis node to the output. Required even though we do not
+    // output any audio. Allows further downstream audio processing or output to
+    // occur.
+    node.connect(context.destination);
+
+  return { context, node };
+}

src/render/main.ts

@@ -3,6 +3,7 @@
 
 import App from "./App.vue";
 
+import { setupAudio } from "./audio/setupAudio";
 import mitt from "mitt";
 import { createApp } from "vue";
 
@@ -12,6 +13,8 @@ import { initIpcRendererListeners } from "./ipc"
 // Handle ipcMain events.
 initIpcRendererListeners();
 
+setupAudio();
+
 // Handle events.
 const emitter = mitt();
 

src/render/shims-vue.d.ts

@@ -11,3 +11,8 @@ declare module "anime-js" {
     export = anime;
   }
 }
+
+declare module "*.worklet.ts" {
+  const exportString: string;
+  export default exportString;
+}

tests/audio.js

@@ -1,84 +1,38 @@
 
-const speech = require('@google-cloud/speech');
+// const speech = require('@google-cloud/speech');
 const portAudio = require('naudiodon');
 // const rs = fs.createReadStream('rawAudio.wav');
 
-// Creates a client
-const client = new speech.SpeechClient();
-
-const encoding = 'LINEAR16';
-const sampleRateHertz = 16000;
-const languageCode = 'en-US';
-
 const audioContainer = {
     input: '',
     buffers: []
 }
 
-const config = {
-  encoding: encoding,
-  sampleRateHertz: sampleRateHertz,
-  languageCode: languageCode,
-};
-
-/**
- * Note that transcription is limited to 60 seconds audio.
- * Use a GCS file for audio longer than 1 minute.
- */
-async function transcribeSpeech (audio) {
-  const request = {
-    config: config,
-    audio: audio,
-  };
-
-  // Detects speech in the audio file. This creates a recognition job that you
-  // can wait for now, or get its result later.
-  const [operation] = await client.longRunningRecognize(request);
-
-  // Get a Promise representation of the final result of the job
-  const [response] = await operation.promise();
-
-  const transcription = response.results
-    .map(result => result.alternatives[0].transcript)
-    .join('\n');
-  console.log(`Transcription: ${transcription}`);
-}
 
 let record = true;
 
 // Create an instance of AudioIO with inOptions (defaults are as below), which will return a ReadableStream
-const ia = new portAudio.AudioIO({
+const aio = new portAudio.AudioIO({
   inOptions: {
-    channelCount: 1,
-    sampleFormat: 16,
-    sampleRate: 16000,
-    deviceId: -1,
-    closeOnError: false,
+    channelCount: 2,
+    sampleFormat: portAudio.SampleFormat16Bit,
+    sampleRate: 44100,
+    deviceId: -1 // Use -1 or omit the deviceId to select the default device
   },
-});
-ia.setEncoding('base64');
-ia.start();
-ia.on('data', (chunk) => {
-    if (record) {
-        console.log('recording data');
-        audioContainer.input += chunk;
-    } else {
-        if (audioContainer.input.length) audioContainer.input = "";
-    }
+  outOptions: {
+    channelCount: 2,
+    sampleFormat: portAudio.SampleFormat16Bit,
+    sampleRate: 44100,
+    deviceId: -1 // Use -1 or omit the deviceId to select the default device
+  }
 });
 
-const ao = new portAudio.AudioIO({
-    outOptions: {
-        sampleFormat: 16,
-        channelCount: 1,
-        sampleRate: 16000,
-        deviceId: -1,
-        closeOnError: false,
-    }
-});
-ao.start();
 
-let counter = 0;
+aio.start()
+aio.read()
+aio.on('data', buf => console.log(buf.timestamp));
+
+const counter = 0;
 const tests = [];
 
 function testCallback() {
@@ -143,13 +97,6 @@ function bufSplit(input){
 }
 
 async function test() {
-    transcribeSpeech({
-        content: Buffer.from(audioContainer.input, 'base64')
-    });
-    tests.push(audioContainer.input)
-    counter++;
-    console.log('audio string length:', audioContainer.input.length)
-    console.log('buffers written: ', audioContainer.buffers.length)
 }
 
 setTimeout(async () => {
@@ -157,26 +104,6 @@ setTimeout(async () => {
     test()
 }, 4000);
 
-// setTimeout(() => {
-//     record = true;
-// }, 6000)
-//
-// setTimeout(async () => {
-//     record = false;
-//     test();
-// }, 9000);
-//
-// setTimeout(() => {
-//     record = true;
-// }, 11000)
-//
-// setTimeout(async () => {
-//     record = false;
-//     test();
-// }, 14000);
-//
 setTimeout(async () => {
-    ia.quit();
-    testCallback()
     return;
 }, 6000);

tests/transcribe.js

@@ -0,0 +1,38 @@
+
+const speech = require('@google-cloud/speech');
+
+// Creates a client
+const client = new speech.SpeechClient();
+
+const encoding = 'LINEAR16';
+const sampleRateHertz = 16000;
+const languageCode = 'en-US';
+
+const config = {
+  encoding: encoding,
+  sampleRateHertz: sampleRateHertz,
+  languageCode: languageCode,
+};
+
+/**
+ * Note that transcription is limited to 60 seconds audio.
+ * Use a GCS file for audio longer than 1 minute.
+ */
+async function transcribeSpeech (audio) {
+  const request = {
+    config: config,
+    audio: audio,
+  };
+
+  // Detects speech in the audio file. This creates a recognition job that you
+  // can wait for now, or get its result later.
+  const [operation] = await client.longRunningRecognize(request);
+
+  // Get a Promise representation of the final result of the job
+  const [response] = await operation.promise();
+
+  const transcription = response.results
+    .map(result => result.alternatives[0].transcript)
+    .join('\n');
+  console.log(`Transcription: ${transcription}`);
+}

tsconfig.worklet.json

@@ -0,0 +1,23 @@
+{
+    "compilerOptions": {
+        "target": "ES2018",
+        "module": "esnext",
+        "strict": true,
+        "importHelpers": true,
+        "moduleResolution": "node",
+        "resolveJsonModule": true,
+        "experimentalDecorators": true,
+        "skipLibCheck": true,
+        "esModuleInterop": true,
+        "allowSyntheticDefaultImports": true,
+        "sourceMap": true,
+        "baseUrl": ".",
+        "types": ["webpack-env"],
+        "paths": {
+            "@/*": ["src/*"]
+        },
+        "lib": ["esnext", "scripthost"]
+    },
+    "include": ["src/**/*.worklet.ts"],
+    "exclude": ["node_modules"]
+}

vue.config.js

@@ -0,0 +1,50 @@
+
+const path = require("path");
+
+module.exports = {
+  assetsDir: "../../static/SPA"
+}
+module.exports = {
+    lintOnSave: false,
+    pluginOptions: {
+        electronBuilder: {
+          mainProcessFile: "./src/init.ts",
+          rendererProcessFile: "./src/render/main.ts",
+          preload: "./src/render/preload.ts",
+
+
+            builderOptions: {
+              "appId": "com.crimata.ElectronUpdaterApp",
+              "artifactName": "${productName}-${version}.${ext}",
+              "publish": {
+                "provider": "generic",
+                "url": "https://gitlab.com/api/v4/projects/25637892/jobs/artifacts/main/raw/dist_electron?job=build"
+              }
+            },
+
+
+          chainWebpackRendererProcess: (config) => {
+            // Chain webpack config for electron renderer process only (won't be applied to web builds)
+            config.outputDir = path.resolve(__dirnamei, "js");
+            config.module
+              .rule('worklet')
+              .test(/\.worklet\.ts$/)
+              .use('worklet-loader')
+                .loader('worklet-loader')
+                .tap(options => {
+                    options.name = "js/[hash].worklet.js";
+                    return options
+                })
+                .end()
+              // Add another loader
+              .use('ts-loader')
+                .loader('ts-loader')
+                .tap(options => {
+                    options.configFile = "tsconfig.worklet.json";
+                        return options
+                    })
+                    .end()
+          },
+        }
+      }
+};

yarn.lock

@@ -1456,6 +1456,11 @@
   resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.6.tgz#f4c7ec43e81b319a9815115031709f26987891f0"
   integrity sha512-3c+yGKvVP5Y9TYBEibGNR+kLtijnj7mYrXRg+WpFb2X9xm04g/DXYkfg4hmzJQosc9snFNUPkbYIhu+KAm6jJw==
 
+"@types/json-schema@^7.0.6":
+  version "7.0.7"
+  resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.7.tgz#98a993516c859eb0d5c4c8f098317a9ea68db9ad"
+  integrity sha512-cxWFQVseBm6O9Gbw1IWb8r6OS4OhSt3hPZLkFApLjM8TEXROBuQGLAH2i2gZpcXdLBIrpXuTDhH7Vbm1iXmNGA==
+
 "@types/long@^4.0.0", "@types/long@^4.0.1":
   version "4.0.1"
   resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.1.tgz#459c65fa1867dafe6a8f322c4c51695663cc55e9"
@@ -2367,7 +2372,7 @@ ajv-keywords@^3.1.0, ajv-keywords@^3.4.1, ajv-keywords@^3.5.2:
   resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-3.5.2.tgz#31f29da5ab6e00d1c2d329acf7b5929614d5014d"
   integrity sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==
 
-ajv@^6.1.0, ajv@^6.10.0, ajv@^6.10.2, ajv@^6.12.0, ajv@^6.12.2, ajv@^6.12.3, ajv@^6.12.4:
+ajv@^6.1.0, ajv@^6.10.0, ajv@^6.10.2, ajv@^6.12.0, ajv@^6.12.2, ajv@^6.12.3, ajv@^6.12.4, ajv@^6.12.5:
   version "6.12.6"
   resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4"
   integrity sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==
@@ -4898,11 +4903,6 @@ dotenv-expand@^5.1.0:
   resolved "https://registry.yarnpkg.com/dotenv-expand/-/dotenv-expand-5.1.0.tgz#3fbaf020bfd794884072ea26b1e9791d45a629f0"
   integrity sha512-YXQl1DSa4/PQyRfgrv6aoNjhasp/p4qs9FjJ4q4cQk+8m4r6k4ZSiEyytKG8f8W9gi8WsQtIObNmKd+tMzNTmA==
 
-dotenv@^10.0.0:
-  version "10.0.0"
-  resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-10.0.0.tgz#3d4227b8fb95f81096cdd2b66653fb2c7085ba81"
-  integrity sha512-rlBi9d8jpv9Sf1klPjNfFAuWDjKLwTIJJ/VxtoTwIR6hnZxcEOQCZg2oIL3MWBYw5GpUDKOEnND7LXTbIpQ03Q==
-
 dotenv@^8.2.0:
   version "8.2.0"
   resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-8.2.0.tgz#97e619259ada750eea3e4ea3e26bceea5424b16a"
@@ -6649,6 +6649,11 @@ hmac-drbg@^1.0.0:
     minimalistic-assert "^1.0.0"
     minimalistic-crypto-utils "^1.0.1"
 
+hoek@^4.2.1:
+  version "4.2.1"
+  resolved "https://registry.yarnpkg.com/hoek/-/hoek-4.2.1.tgz#9634502aa12c445dd5a7c5734b572bb8738aacbb"
+  integrity sha512-QLg82fGkfnJ/4iy1xZ81/9SIJiq1NGFUMGs6ParyjBZr6jW2Ufj/snDqTHixNlHdPNwN2RLVD0Pi3igeK9+JfA==
+
 hoopy@^0.1.4:
   version "0.1.4"
   resolved "https://registry.yarnpkg.com/hoopy/-/hoopy-0.1.4.tgz#609207d661100033a9a9402ad3dea677381c1b1d"
@@ -8447,7 +8452,7 @@ loader-utils@^0.2.16:
     json5 "^0.5.0"
     object-assign "^4.0.1"
 
-loader-utils@^1.0.2, loader-utils@^1.1.0, loader-utils@^1.2.3, loader-utils@^1.4.0:
+loader-utils@^1.0.0, loader-utils@^1.0.2, loader-utils@^1.1.0, loader-utils@^1.2.3, loader-utils@^1.4.0:
   version "1.4.0"
   resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-1.4.0.tgz#c579b5e34cb34b1a74edc6c1fb36bfa371d5a613"
   integrity sha512-qH0WSMBtn/oHuwjy/NucEgbx5dbxxnxup9s4PVXJUDHZBQY+s0NWA9rJf53RBnQZxfch7euUui7hpoAPvALZdA==
@@ -11154,6 +11159,14 @@ schema-utils@2.7.0:
     ajv "^6.12.2"
     ajv-keywords "^3.4.1"
 
+schema-utils@^0.4.0:
+  version "0.4.7"
+  resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-0.4.7.tgz#ba74f597d2be2ea880131746ee17d0a093c68187"
+  integrity sha512-v/iwU6wvwGK8HbU9yi3/nhGzP0yGSuhQMzL6ySiec1FSrZZDkhm4noOSWzrNFo/jEc+SJY6jRTwuwbSXJPDUnQ==
+  dependencies:
+    ajv "^6.1.0"
+    ajv-keywords "^3.1.0"
+
 schema-utils@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-1.0.0.tgz#0b79a93204d7b600d4b2850d1f66c2a34951c770"
@@ -11172,6 +11185,15 @@ schema-utils@^2.0.0, schema-utils@^2.5.0, schema-utils@^2.6.1, schema-utils@^2.6
     ajv "^6.12.4"
     ajv-keywords "^3.5.2"
 
+schema-utils@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-3.0.0.tgz#67502f6aa2b66a2d4032b4279a2944978a0913ef"
+  integrity sha512-6D82/xSzO094ajanoOSbe4YvXWMfn2A//8Y1+MUqFAJul5Bs+yn36xbK9OtNDcRVSBJ9jjeoXftM6CfztsjOAA==
+  dependencies:
+    "@types/json-schema" "^7.0.6"
+    ajv "^6.12.5"
+    ajv-keywords "^3.5.2"
+
 scss-tokenizer@^0.2.3:
   version "0.2.3"
   resolved "https://registry.yarnpkg.com/scss-tokenizer/-/scss-tokenizer-0.2.3.tgz#8eb06db9a9723333824d3f5530641149847ce5d1"
@@ -13300,6 +13322,14 @@ worker-farm@^1.7.0:
   dependencies:
     errno "~0.1.7"
 
+worker-loader@^3.0.8:
+  version "3.0.8"
+  resolved "https://registry.yarnpkg.com/worker-loader/-/worker-loader-3.0.8.tgz#5fc5cda4a3d3163d9c274a4e3a811ce8b60dbb37"
+  integrity sha512-XQyQkIFeRVC7f7uRhFdNMe/iJOdO6zxAaR3EWbDp45v3mDhrTi+++oswKNxShUNjPC/1xUp5DB29YKLhFo129g==
+  dependencies:
+    loader-utils "^2.0.0"
+    schema-utils "^3.0.0"
+
 worker-rpc@^0.1.0:
   version "0.1.1"
   resolved "https://registry.yarnpkg.com/worker-rpc/-/worker-rpc-0.1.1.tgz#cb565bd6d7071a8f16660686051e969ad32f54d5"
@@ -13307,6 +13337,15 @@ worker-rpc@^0.1.0:
   dependencies:
     microevent.ts "~0.1.1"
 
+worklet-loader@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/worklet-loader/-/worklet-loader-1.0.0.tgz#17e2eef75981de469c1e1e200ad1ffb54efe7a29"
+  integrity sha512-4yFqiGDwICoJB4ZbWHzCzyTyDrRnCU1XfvSJtjiBBDreuWDYpA6wf8yqQjNckcjL1jm/sT9ocSvP5tJnmsMOLA==
+  dependencies:
+    hoek "^4.2.1"
+    loader-utils "^1.0.0"
+    schema-utils "^0.4.0"
+
 wrap-ansi@^3.0.1:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-3.0.1.tgz#288a04d87eda5c286e060dfe8f135ce8d007f8ba"