From acbf85744a329dadb1b48b5b8167d46b4a6f9eaf Mon Sep 17 00:00:00 2001 From: riqo Date: Wed, 3 Feb 2021 09:27:18 -0600 Subject: [PATCH 1/3] remove google cloud client from background --- src/background.ts | 55 ++------------------------------- src/main.ts | 2 -- testAudio.js | 78 +++++++++++++++-------------------------------- 3 files changed, 28 insertions(+), 107 deletions(-) diff --git a/src/background.ts b/src/background.ts index b8f649d..de64c26 100644 --- a/src/background.ts +++ b/src/background.ts @@ -8,39 +8,6 @@ import * as path from "path"; const fs = require('fs'); import WebSocket from "ws"; const portAudio = require('naudiodon'); -const speech = require('@google-cloud/speech'); - -const client = new speech.SpeechClient(); - -const encoding = 'LINEAR16'; -const sampleRateHertz = 16000; -const languageCode = 'en-US'; - -const config = { - encoding: encoding, - sampleRateHertz: sampleRateHertz, - languageCode: languageCode, -}; -const request = { - config, - interimResults: true, -}; - -const speechCallback = (d: any) => { - console.log(d) -} - -const recognizeStream = client -.streamingRecognize(request) -.on('error', err => { - if (err.code === 11) { - // restartStream(); - } else { - console.error('API request error ' + err); - } -}) -.on('data', speechCallback); - // Create an instance of AudioIO with inOptions (defaults are as below), which will return a ReadableStream const audioOptions = { @@ -54,15 +21,9 @@ const audioOptions = { const ai = new portAudio.AudioIO({ inOptions: audioOptions }); -ai.pipe(recognizeStream); +//ai.pipe(); ai.start(); - -// const filename = "rawAudio.wav"; -// Create a write stream to write out to a raw audio file -// const writeStream = fs.createWriteStream(filename, { encoding: 'binary'}); -// ai.pipe(writeStream); - // Keep a global reference of the window object, if you don't, the window will // be closed automatically when the JavaScript object is garbage collected. let win: BrowserWindow | null; @@ -125,19 +86,9 @@ function createWindow() { ipcMain.on('update-recorder', (Event: any, record: boolean) => { if (record) { - // ai.start(); + // start recording } else { - // ai.quit(); - - // const audio = { - // content: fs.readFileSync(filename).toString('base64'), - // } - // const message = { - // text: "", - // audio: audioData - // } - // ws.send(JSON.stringify(message)) - // audioData = ""; + // stop recording } }) }) diff --git a/src/main.ts b/src/main.ts index bc2fa9d..531bb90 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,7 +1,6 @@ import { createApp } from "vue"; import App from "./App.vue"; import router from "./router"; -import store from "./store"; // animation library import anime from "animejs"; @@ -11,7 +10,6 @@ import mitt from "mitt"; const emitter = mitt(); createApp(App) - .use(store) .use(router) .provide("animejs", anime) .provide("mitt", emitter) diff --git a/testAudio.js b/testAudio.js index 6a92222..3312d27 100644 --- a/testAudio.js +++ b/testAudio.js @@ -1,71 +1,40 @@ const fs = require('fs'); -// const portAudio = require('naudiodon'); -// Imports the Google Cloud client library const speech = require('@google-cloud/speech'); const portAudio = require('naudiodon'); - -// Create a stream to pipe into the AudioOutput -// Note that this does not strip the WAV header so a click will be heard at the beginning const rs = fs.createReadStream('rawAudio.wav'); const WebSocket = require("ws") -// Start piping data and start streaming const {Writable} = require('stream'); -const ws = new WebSocket("ws://localhost:8080"); -//const duplex = WebSocket.createWebSocketStream(ws, { encoding: 'binary' }); - -const audioChunks = []; -const audioInputStreamTransform = new Writable({ - write(chunk, encoding, next) { - console.log(chunk) - // ws send chunk - // audioChunks.push(chunk); - // ws.send(chunk) - next(); - }, - - final() { - console.log(audioChunks) - // ws.send("stop") - // stop ws transer - }, -}); - // Creates a client const client = new speech.SpeechClient(); -/** - * TODO(developer): Uncomment the following lines before running the sample. - */ -const filename = './rawAudio.wav'; +// connect to test server +const ws = new WebSocket("ws://localhost:8080"); + +// google cloud speech to text settings const encoding = 'LINEAR16'; const sampleRateHertz = 44100; const languageCode = 'en-US'; +const audioChunks = []; + +const audioInputStreamTransform = new Writable({ + write(chunk, encoding, next) { + console.log(chunk); + audioChunks.push(chunk); + next(); + }, + + final() { + console.log(audioChunks); + }, +}); + const config = { encoding: encoding, sampleRateHertz: sampleRateHertz, languageCode: languageCode, }; -const audio = { - content: fs.readFileSync(filename).toString('base64'), -}; - -// const request = { -// config: config, -// audio: audio, -// }; - -// Detects speech in the audio file -async function test() { - const [response] = await client.recognize(request); - const transcription = response.results - .map(result => result.alternatives[0].transcript) - .join('\n'); - console.log('Transcription: ', transcription); -} - -// test(); const request = { config, @@ -75,7 +44,7 @@ const request = { const speechCallback = (data) => { console.log( `Transcription: ${data.results[0].alternatives[0].transcript}` - ) + ); } const recognizeStream = client @@ -90,7 +59,7 @@ const recognizeStream = client .on('data', speechCallback); // Create an instance of AudioIO with inOptions (defaults are as below), which will return a ReadableStream -const audioOptions = { +const recorderOptions = { channelCount: 1, sampleFormat: portAudio.sampleFormat16Bit, sampleRate: 16000, @@ -98,13 +67,16 @@ const audioOptions = { closeOnError: false // Close the stream if an audio error is detected, if set false then just log the error } -const audioData = []; const ai = new portAudio.AudioIO({ - inOptions: audioOptions + inOptions: recorderOptions }); + +// manipulate individual chunks as they're available +// const audioData = []; // ai.on('data', (d) => { // audioData.push(d.toString('binary')) // }) + ai.pipe(audioInputStreamTransform) ai.start(); setTimeout(() => { From 1a328b41b94ffc30cf3c012bb90de1e8442ca05e Mon Sep 17 00:00:00 2001 From: riqo Date: Wed, 3 Feb 2021 18:01:48 -0600 Subject: [PATCH 2/3] add recorder file --- src/background.ts | 17 ++------ src/background/recorder.ts | 79 ++++++++++++++++++++++++++++++++++++++ testAudio.js | 1 + 3 files changed, 84 insertions(+), 13 deletions(-) create mode 100644 src/background/recorder.ts diff --git a/src/background.ts b/src/background.ts index de64c26..e53ab80 100644 --- a/src/background.ts +++ b/src/background.ts @@ -9,20 +9,11 @@ const fs = require('fs'); import WebSocket from "ws"; const portAudio = require('naudiodon'); -// Create an instance of AudioIO with inOptions (defaults are as below), which will return a ReadableStream -const audioOptions = { - channelCount: 2, - sampleFormat: portAudio.sampleFormat16Bit, - sampleRate: 16000, - deviceId: -1, // Use -1 or omit the deviceId to select the default device - closeOnError: false // Close the stream if an audio error is detected, if set false then just log the error -} - -const ai = new portAudio.AudioIO({ - inOptions: audioOptions -}); +// const ai = new portAudio.AudioIO({ +// inOptions: audioOptions +// }); //ai.pipe(); -ai.start(); +// ai.start(); // Keep a global reference of the window object, if you don't, the window will // be closed automatically when the JavaScript object is garbage collected. diff --git a/src/background/recorder.ts b/src/background/recorder.ts new file mode 100644 index 0000000..f47a228 --- /dev/null +++ b/src/background/recorder.ts @@ -0,0 +1,79 @@ +// eslint-disable-next-line @typescript-eslint/no-var-requires +const portAudio = require('naudiodon'); +// eslint-disable-next-line @typescript-eslint/no-var-requires +const { Writable } = require('stream'); +// eslint-disable-next-line @typescript-eslint/no-var-requires +const { StringDecoder } = require('string_decoder'); + +interface RecorderI { + start(): void; + stop(): string; +} + +class StringWritable extends Writable { + + constructor(options: { + defaultEncoding: string; + }) { + super(options); + this._decoder = new StringDecoder(options && options.defaultEncoding); + this.data = ''; + } + + _write(chunk: Buffer, encoding: string, callback: (error? : Error) => void) { + if (encoding === 'buffer') { + chunk = this._decoder.write(chunk); + } + this.data += chunk; + callback(); + } + + _final(callback: (error? : Error) => void) { + this.data += this._decoder.end(); + callback(); + } + +} + +export class Recorder implements RecorderI { + + private ia: any; + private micWritable: StringWritable; + + constructor(options: { + + }) { + this.ia = new portAudio.AudioIO({ + inOptions: options + }); + this.micWritable = new StringWritable({ + defaultEncoding: 'binary' + }); + this.start(); + } + + start(): void { + console.log('inititate recording'); + this.ia.pipe(this.micWritable); + this.ia.start(); + this.ia.pause(); + } + + record(): void { + this.ia.resume(); + } + + pause(): void { + this.ia.pause(); + } + + getData(): string { + return this.micWritable.data; + } + + stop(): string { + this.ia.quit(); + return this.micWritable.data; + } + +} diff --git a/testAudio.js b/testAudio.js index 3312d27..c2c291b 100644 --- a/testAudio.js +++ b/testAudio.js @@ -1,3 +1,4 @@ + const fs = require('fs'); const speech = require('@google-cloud/speech'); const portAudio = require('naudiodon'); From 5c55910e26ffcb214650f1bbecab4d11bd2c1d13 Mon Sep 17 00:00:00 2001 From: Enrique Hernandez Date: Thu, 4 Feb 2021 10:12:21 -0600 Subject: [PATCH 3/3] refactor background.ts --- src/background.ts | 144 +++++-------------------------------- src/background/initApp.ts | 100 ++++++++++++++++++++++++++ src/background/recorder.ts | 78 ++++++++++++++------ 3 files changed, 173 insertions(+), 149 deletions(-) create mode 100644 src/background/initApp.ts diff --git a/src/background.ts b/src/background.ts index e53ab80..d3cecd9 100644 --- a/src/background.ts +++ b/src/background.ts @@ -1,137 +1,27 @@ "use strict"; +import {initApp} from './background/initApp'; +import {Recorder} from './background/recorder'; +var portAudio = require('naudiodon'); -import { app, protocol, BrowserWindow, ipcMain } from "electron"; -import { createProtocol } from "vue-cli-plugin-electron-builder/lib"; -import installExtension, { VUEJS_DEVTOOLS } from "electron-devtools-installer"; -const isDevelopment = process.env.NODE_ENV !== "production"; -import * as path from "path"; -const fs = require('fs'); -import WebSocket from "ws"; -const portAudio = require('naudiodon'); -// const ai = new portAudio.AudioIO({ -// inOptions: audioOptions -// }); -//ai.pipe(); -// ai.start(); +// NOTE Program Begins Here +(async () => { -// Keep a global reference of the window object, if you don't, the window will -// be closed automatically when the JavaScript object is garbage collected. -let win: BrowserWindow | null; + const isDevelopment = process.env.NODE_ENV !== "production"; -// Scheme must be registered before the app is ready -protocol.registerSchemesAsPrivileged([ - { scheme: "app", privileges: { secure: true, standard: true } } -]); + const recorder = new Recorder({ + channelCount: 1, + sampleFormat: portAudio.SampleFormat16Bit, + sampleRate: 16000, + deviceId: -1, // Use -1 or omit the deviceId to select the default device + closeOnError: true + }, 'binary'); -function createWindow() { - // Create the browser window. - win = new BrowserWindow({ - width: 450, - height: 1025, - resizable: true, - webPreferences: { - // Use pluginOptions.nodeIntegration, leave this alone - // See nklayman.github.io/vue-cli-plugin-electron-builder/guide/security.html#node-integration for more info - nodeIntegration: (process.env - .ELECTRON_NODE_INTEGRATION as unknown) as boolean, - preload: path.join(__dirname, "preload.js") - } - }); - - if (process.env.WEBPACK_DEV_SERVER_URL) { - // Load the url of the dev server if in development mode - win.loadURL(process.env.WEBPACK_DEV_SERVER_URL as string); - // NOTE uncomment for dev tools on app launch - if (!process.env.IS_TEST) win.webContents.openDevTools(); - } else { - createProtocol("app"); - // Load the index.html when not in development - win.loadURL("app://./index.html"); - } - - win.webContents.on('did-finish-load', () => { - - const ws = new WebSocket("ws://localhost:8080"); - ws.on("open", function open() { - // ws.send("hernandeze2@xavier.edu"); - }); - ws.on("message", (message: any) => { - const parsed = JSON.parse(message); - console.log('new message', message) - if (win) { - win.webContents.send('render-message', { - message: parsed - }) - } - }); - - ipcMain.on('update-menu-bar', (Event: any, payload: any) => { - // win.setTitle("Listening..."); - }) - - ipcMain.on('send-message', (Event: any, payload: any) => { - ws.send(JSON.stringify(payload)); - // win.setTitle("Listening..."); - }) - - ipcMain.on('update-recorder', (Event: any, record: boolean) => { - if (record) { - // start recording - } else { - // stop recording - } - }) - }) - - win.on("closed", () => { - win = null; - }); -} - -// Quit when all windows are closed. -app.on("window-all-closed", () => { - // On macOS it is common for applications and their menu bar - // to stay active until the user quits explicitly with Cmd + Q - if (process.platform !== "darwin") { - app.quit(); - } -}); - -app.on("activate", () => { - // On macOS it's common to re-create a window in the app when the - // dock icon is clicked and there are no other windows open. - if (win === null) { - createWindow(); - } -}); - -// This method will be called when Electron has finished -// initialization and is ready to create browser windows. -// Some APIs can only be used after this event occurs. -app.on("ready", async () => { - if (isDevelopment && !process.env.IS_TEST) { - // Install Vue Devtools try { - await installExtension(VUEJS_DEVTOOLS); + // await authenticate method + initApp(isDevelopment); } catch (e) { - console.error("Vue Devtools failed to install:", e.toString()); + console.log("Error", e); } - } - createWindow(); -}); -// Exit cleanly on request from parent process in development mode. -if (isDevelopment) { - if (process.platform === "win32") { - process.on("message", data => { - if (data === "graceful-exit") { - app.quit(); - } - }); - } else { - process.on("SIGTERM", () => { - app.quit(); - }); - } -} \ No newline at end of file +})(); diff --git a/src/background/initApp.ts b/src/background/initApp.ts new file mode 100644 index 0000000..e0ea203 --- /dev/null +++ b/src/background/initApp.ts @@ -0,0 +1,100 @@ +"use strict"; + +import { app, protocol, BrowserWindow, ipcMain } from "electron"; +import { createProtocol } from "vue-cli-plugin-electron-builder/lib"; +import installExtension, { VUEJS_DEVTOOLS } from "electron-devtools-installer"; +import { createProtocol } from "vue-cli-plugin-electron-builder/lib"; +import * as path from "path"; + +var win: BrowserWindow | null; + +function createWindow() { + // Create the browser window. + win = new BrowserWindow({ + width: 450, + height: 1025, + resizable: true, + webPreferences: { + // Use pluginOptions.nodeIntegration, leave this alone + // See nklayman.github.io/vue-cli-plugin-electron-builder/guide/security.html#node-integration for more info + nodeIntegration: (process.env + .ELECTRON_NODE_INTEGRATION as unknown) as boolean, + preload: path.join(__dirname, "preload.js") + } + }); + + if (process.env.WEBPACK_DEV_SERVER_URL) { + // Load the url of the dev server if in development mode + win.loadURL(process.env.WEBPACK_DEV_SERVER_URL as string); + // NOTE uncomment for dev tools on app launch + if (!process.env.IS_TEST) win.webContents.openDevTools(); + } else { + createProtocol("app"); + // Load the index.html when not in development + win.loadURL("app://./index.html"); + } + + win.webContents.on('did-finish-load', () => { + // do stuff once window has finished loading + }) + + win.on("closed", () => { + win = null; + }); +} + +export function initApp(isDev: boolean){ + + // Scheme must be registered before the app is ready + protocol.registerSchemesAsPrivileged([ + { scheme: "app", privileges: { secure: true, standard: true } } + ]); + + // This method will be called when Electron has finished + // initialization and is ready to create browser windows. + // Some APIs can only be used after this event occurs. + app.on("ready", async () => { + if (isDev && !process.env.IS_TEST) { + // Install Vue Devtools + try { + await installExtension(VUEJS_DEVTOOLS); + } catch (e) { + console.error("Vue Devtools failed to install:", e.toString()); + } + } + createWindow(); + }); + + // Quit when all windows are closed. + app.on("window-all-closed", () => { + // On macOS it is common for applications and their menu bar + // to stay active until the user quits explicitly with Cmd + Q + if (process.platform !== "darwin") { + app.quit(); + } + }); + + app.on("activate", () => { + // On macOS it's common to re-create a window in the app when the + // dock icon is clicked and there are no other windows open. + if (win === null) { + createWindow(); + } + }); + + // Exit cleanly on request from parent process in development mode. + if (isDev) { + if (process.platform === "win32") { + process.on("message", data => { + if (data === "graceful-exit") { + app.quit(); + } + }); + } else { + process.on("SIGTERM", () => { + app.quit(); + }); + } + } +} + diff --git a/src/background/recorder.ts b/src/background/recorder.ts index f47a228..36f93ba 100644 --- a/src/background/recorder.ts +++ b/src/background/recorder.ts @@ -4,10 +4,11 @@ const portAudio = require('naudiodon'); const { Writable } = require('stream'); // eslint-disable-next-line @typescript-eslint/no-var-requires const { StringDecoder } = require('string_decoder'); +import { ipcMain } from "electron"; interface RecorderI { - start(): void; stop(): string; + pause(): void; } class StringWritable extends Writable { @@ -25,55 +26,88 @@ class StringWritable extends Writable { chunk = this._decoder.write(chunk); } this.data += chunk; + console.log('writing'); callback(); } _final(callback: (error? : Error) => void) { + console.log('yayayay'); this.data += this._decoder.end(); callback(); } +} +type inputOptions = {clear + + channelCount: number, + sampleFormat: string, + sampleRate: number, + deviceId: number, + closeOnError: boolean } export class Recorder implements RecorderI { - + private ia: any; private micWritable: StringWritable; + private recorderOptions: inputOptions; - constructor(options: { - - }) { - this.ia = new portAudio.AudioIO({ - inOptions: options - }); + constructor(options: inputOptions, encoding: string) { + this.recorderOptions = options; this.micWritable = new StringWritable({ - defaultEncoding: 'binary' + defaultEncoding: encoding + }) + this.restart(); + ipcMain.on('update-recorder', this._update); + } + + restart() { + this.ia = new portAudio.AudioIO({ + inOptions: this.recorderOptions }); - this.start(); - } - - start(): void { - console.log('inititate recording'); - this.ia.pipe(this.micWritable); - this.ia.start(); - this.ia.pause(); - } - - record(): void { - this.ia.resume(); + this._start(); } pause(): void { this.ia.pause(); } + resume(){ + this.ia.resume(); + } + getData(): string { return this.micWritable.data; } stop(): string { this.ia.quit(); - return this.micWritable.data; + return this.getData(); } + private _start(): void { + this.ia.pipe(this.micWritable); + this.ia.start(); + this.pause(); + } + + private _update = (Event: any, record: boolean) => { + if (record) { + console.log('recording'); + this.micWritable.data = ''; + if(this.ia._readableState.paused){ + this.resume(); + return; + } + this.ia.pipe(this.micWritable); + + console.log(this.ia._readableState.paused); + } else { + console.log('stop recording'); + this.ia.unpipe(this.micWritable); + this.micWritable.on('finish', () => { + console.log('hello'); + }) + } + } }