add speech to text transcription
This commit is contained in:
riqo
parent
29dc7de3e4
commit
0ba02f2f09
10 changed files with 378 additions and 438 deletions
|
|
@ -1,92 +1,41 @@
|
|||
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
||||
const portAudio = require('naudiodon');
|
||||
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
||||
const { Writable } = require('stream');
|
||||
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
||||
const { StringDecoder } = require('string_decoder');
|
||||
|
||||
interface RecorderI {
|
||||
stop(): string;
|
||||
pause(): void;
|
||||
}
|
||||
let audio: string;
|
||||
let audioInput: string[] = [];
|
||||
|
||||
type inputOptions = {
|
||||
channelCount: number;
|
||||
sampleFormat: number;
|
||||
sampleRate: number;
|
||||
deviceId: number;
|
||||
closeOnError: boolean;
|
||||
}
|
||||
export function initRecorder() {
|
||||
|
||||
class StringWritable extends Writable {
|
||||
|
||||
constructor(options: {
|
||||
defaultEncoding: string;
|
||||
}) {
|
||||
super(options);
|
||||
this._decoder = new StringDecoder(options && options.defaultEncoding);
|
||||
this.data = '';
|
||||
}
|
||||
|
||||
_write(chunk: Buffer, encoding: string, callback: (error? : Error) => void) {
|
||||
if (encoding === 'buffer') {
|
||||
chunk = this._decoder.write(chunk);
|
||||
}
|
||||
this.data += chunk;
|
||||
console.log('writing');
|
||||
callback();
|
||||
}
|
||||
|
||||
_final(callback: (error? : Error) => void) {
|
||||
console.log('yayayay');
|
||||
this.data += this._decoder.end();
|
||||
callback();
|
||||
}
|
||||
}
|
||||
|
||||
export class Recorder implements RecorderI {
|
||||
|
||||
private ia: any;
|
||||
private micWritable: StringWritable;
|
||||
private recorderOptions: inputOptions;
|
||||
|
||||
constructor(options: inputOptions, encoding: string) {
|
||||
this.recorderOptions = options;
|
||||
this.micWritable = new StringWritable({
|
||||
defaultEncoding: encoding
|
||||
let ia = new portAudio.AudioIO({
|
||||
inOptions: {
|
||||
channelCount: 1,
|
||||
sampleFormat: 16,
|
||||
sampleRate: 16000,
|
||||
deviceId: -1,
|
||||
closeOnError: false,
|
||||
}
|
||||
});
|
||||
try {
|
||||
this.start();
|
||||
} catch(e) {
|
||||
console.log('RECORDER ERROR!', e);
|
||||
}
|
||||
}
|
||||
|
||||
private start(): void {
|
||||
this.ia = new portAudio.AudioIO({
|
||||
inOptions: this.recorderOptions
|
||||
ia.setEncoding('base64');
|
||||
ia.start();
|
||||
ia.pause();
|
||||
ia.on('error', (e: Error) => {
|
||||
console.log('error recording audio', + e);
|
||||
});
|
||||
ia.on('data', (chunk: string) => {
|
||||
audioInput.push(chunk);
|
||||
console.log('Got %d characters of string data:', chunk.length);
|
||||
});
|
||||
this.ia.pipe(this.micWritable);
|
||||
this.ia.start();
|
||||
this.pause();
|
||||
}
|
||||
|
||||
pause(): void {
|
||||
this.ia.pause();
|
||||
}
|
||||
|
||||
resume(){
|
||||
this.ia.resume();
|
||||
}
|
||||
|
||||
getData(): string {
|
||||
const data = this.micWritable.data;
|
||||
this.micWritable.data = '';
|
||||
return data;
|
||||
}
|
||||
|
||||
stop(): string {
|
||||
this.ia.quit();
|
||||
return this.getData();
|
||||
}
|
||||
}
|
||||
|
||||
async function processData(chunks: string[]): Buffer | Error {
|
||||
audio = '';
|
||||
chunks.forEach(s => {
|
||||
audio += s;
|
||||
})
|
||||
const buffer = Buffer.from(audio, "base64");
|
||||
audioInput = [];
|
||||
return buffer;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,40 +1,34 @@
|
|||
import { createWindow } from './createWindow';
|
||||
import WebSocket from 'ws';
|
||||
import { Recorder } from './recorder';
|
||||
import { ipcMain, BrowserWindow } from "electron";
|
||||
import WebSocket from 'ws';
|
||||
import { windowEmitter } from './windowEmitter';
|
||||
const portAudio = require('naudiodon');
|
||||
|
||||
/*
|
||||
* The main function will be run after electron app is ready.
|
||||
*/
|
||||
export function main(): void {
|
||||
|
||||
let audioInput: string[] = [];
|
||||
|
||||
// create main window.
|
||||
let win: BrowserWindow | null;
|
||||
win = createWindow({ width: 350, height: 525, resizable: false });
|
||||
const win = createWindow({ width: 350, height: 525, resizable: false });
|
||||
|
||||
// Instantiate ws session with crimata-platorm.
|
||||
const ws = new WebSocket('ws://localhost:8080');
|
||||
|
||||
// Instantiate portAudio recorder.
|
||||
const recorder = new Recorder({
|
||||
channelCount: 1,
|
||||
sampleFormat: 16,
|
||||
sampleRate: 16000,
|
||||
deviceId: -1,
|
||||
closeOnError: false,
|
||||
}, 'binary');
|
||||
// Instantiate socket session with crimata-platorm.
|
||||
const socket = new WebSocket('socket://localhost:8080');
|
||||
socket.binaryType = 'arraybuffer';
|
||||
|
||||
const initSession = (): void => {
|
||||
ws.send('[email protected]');
|
||||
socket.send('[email protected]');
|
||||
}
|
||||
|
||||
const sendMessage = (Event: any, payload: any): void => {
|
||||
ws.send(JSON.stringify(payload));
|
||||
socket.send(JSON.stringify(payload));
|
||||
}
|
||||
|
||||
const sendAudio = (data: string): void => {
|
||||
ws.send(data);
|
||||
const sendAudio = (data: Buffer): void => {
|
||||
socket.send(data);
|
||||
}
|
||||
|
||||
const receiveMessage = (message: string): void => {
|
||||
|
|
@ -45,39 +39,65 @@ export function main(): void {
|
|||
});
|
||||
}
|
||||
|
||||
const updateRecorder = (Event: any, record: boolean): void => {
|
||||
const updateRecorder = async (Event: any, record: boolean): void => {
|
||||
if (record) {
|
||||
recorder.resume();
|
||||
ia.resume();
|
||||
} else {
|
||||
recorder.pause();
|
||||
const data = recorder.getData();
|
||||
sendAudio(data);
|
||||
ia.pause();
|
||||
const audio = processAudio();
|
||||
sendAudio(audio);
|
||||
audioInput = [];
|
||||
}
|
||||
}
|
||||
|
||||
function processAudio(): Buffer {
|
||||
let audio = '';
|
||||
audioInput.forEach(s => {
|
||||
audio += s;
|
||||
});
|
||||
const buffer = Buffer.from(audio, "base64");
|
||||
return buffer;
|
||||
}
|
||||
|
||||
const windowMount = (): void => {
|
||||
console.log('testing mount');
|
||||
// ipc event handlers
|
||||
windowEmitter.emit('window-active', true);
|
||||
// ipc event handlers
|
||||
ipcMain.on('update-recorder', updateRecorder);
|
||||
ipcMain.on('send-message', sendMessage);
|
||||
}
|
||||
|
||||
const windowDismount = (): void => {
|
||||
console.log('testing dismount');
|
||||
// TODO: Gracefully close ws connection
|
||||
win = null;
|
||||
windowEmitter.emit('window-active', false);
|
||||
// TODO: Gracefully close socket connection
|
||||
ipcMain.removeListener('update-recorder', updateRecorder);
|
||||
ipcMain.removeListener('send-message', sendMessage);
|
||||
}
|
||||
|
||||
// ws handlers
|
||||
ws.on('connect', initSession);
|
||||
ws.on("message", receiveMessage);
|
||||
// socket handlers
|
||||
socket.on('connect', initSession);
|
||||
socket.on("message", receiveMessage);
|
||||
|
||||
// Handle window mount and dismount.
|
||||
win.webContents.on('did-finish-load', windowMount);
|
||||
win.on("closed", windowDismount);
|
||||
|
||||
let ia = new portAudio.AudioIO({
|
||||
inOptions: {
|
||||
channelCount: 1,
|
||||
sampleFormat: 16,
|
||||
sampleRate: 16000,
|
||||
deviceId: -1,
|
||||
closeOnError: false,
|
||||
}
|
||||
});
|
||||
ia.setEncoding('base64');
|
||||
ia.start();
|
||||
ia.on('error', (e: Error) => {
|
||||
console.log('error recording audio', + e);
|
||||
});
|
||||
ia.pause();
|
||||
ia.on('data', (chunk: string) => {
|
||||
audioInput.push(chunk);
|
||||
console.log('Got %d characters of string data:', chunk.length);
|
||||
});
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue