add speech to text transcription
This commit is contained in:
parent
29dc7de3e4
commit
0ba02f2f09
10 changed files with 378 additions and 438 deletions
|
|
@ -24,6 +24,7 @@
|
|||
"@types/animejs": "^3.1.2",
|
||||
"@types/bindings": "^1.3.0",
|
||||
"@types/dom-mediacapture-record": "^1.0.7",
|
||||
"@types/node": "^14.14.25",
|
||||
"@types/uuid": "^8.3.0",
|
||||
"@types/ws": "^7.2.7",
|
||||
"animejs": "^3.2.0",
|
||||
|
|
|
|||
202
src/audio.js
Normal file
202
src/audio.js
Normal file
|
|
@ -0,0 +1,202 @@
|
|||
function main(
|
||||
encoding = 'LINEAR16',
|
||||
sampleRateHertz = 16000,
|
||||
languageCode = 'en-US',
|
||||
streamingLimit = 290000
|
||||
) {
|
||||
// [START speech_transcribe_infinite_streaming]
|
||||
|
||||
// const encoding = 'LINEAR16';
|
||||
// const sampleRateHertz = 16000;
|
||||
// const languageCode = 'en-US';
|
||||
// const streamingLimit = 10000; // ms - set to low number for demo purposes
|
||||
|
||||
const chalk = require('chalk');
|
||||
const {Writable} = require('stream');
|
||||
const recorder = require('node-record-lpcm16');
|
||||
const fs = require('fs');
|
||||
const writeStream = fs.createWriteStream('test.wav', { encoding: 'binary'});
|
||||
|
||||
// // Imports the Google Cloud client library
|
||||
// // Currently, only v1p1beta1 contains result-end-time
|
||||
const speech = require('@google-cloud/speech').v1p1beta1;
|
||||
|
||||
const client = new speech.SpeechClient();
|
||||
|
||||
const config = {
|
||||
encoding: encoding,
|
||||
sampleRateHertz: sampleRateHertz,
|
||||
languageCode: languageCode,
|
||||
};
|
||||
|
||||
const request = {
|
||||
config,
|
||||
interimResults: true,
|
||||
};
|
||||
|
||||
let recognizeStream = null;
|
||||
let restartCounter = 0;
|
||||
let audioInput = [];
|
||||
let lastAudioInput = [];
|
||||
let resultEndTime = 0;
|
||||
let isFinalEndTime = 0;
|
||||
let finalRequestEndTime = 0;
|
||||
let newStream = true;
|
||||
let bridgingOffset = 0;
|
||||
let lastTranscriptWasFinal = false;
|
||||
|
||||
function startStream() {
|
||||
// Clear current audioInput
|
||||
audioInput = [];
|
||||
// Initiate (Reinitiate) a recognize stream
|
||||
recognizeStream = client
|
||||
.streamingRecognize(request)
|
||||
.on('error', err => {
|
||||
if (err.code === 11) {
|
||||
// restartStream();
|
||||
} else {
|
||||
console.error('API request error ' + err);
|
||||
}
|
||||
})
|
||||
.on('data', speechCallback);
|
||||
|
||||
// Restart stream when streamingLimit expires
|
||||
setTimeout(restartStream, streamingLimit);
|
||||
}
|
||||
|
||||
const speechCallback = stream => {
|
||||
// Convert API result end time from seconds + nanoseconds to milliseconds
|
||||
resultEndTime =
|
||||
stream.results[0].resultEndTime.seconds * 1000 +
|
||||
Math.round(stream.results[0].resultEndTime.nanos / 1000000);
|
||||
|
||||
// Calculate correct time based on offset from audio sent twice
|
||||
const correctedTime =
|
||||
resultEndTime - bridgingOffset + streamingLimit * restartCounter;
|
||||
|
||||
process.stdout.clearLine();
|
||||
process.stdout.cursorTo(0);
|
||||
let stdoutText = '';
|
||||
if (stream.results[0] && stream.results[0].alternatives[0]) {
|
||||
stdoutText =
|
||||
correctedTime + ': ' + stream.results[0].alternatives[0].transcript;
|
||||
}
|
||||
|
||||
if (stream.results[0].isFinal) {
|
||||
process.stdout.write(chalk.green(`${stdoutText}\n`));
|
||||
|
||||
isFinalEndTime = resultEndTime;
|
||||
lastTranscriptWasFinal = true;
|
||||
} else {
|
||||
// Make sure transcript does not exceed console character length
|
||||
if (stdoutText.length > process.stdout.columns) {
|
||||
stdoutText =
|
||||
stdoutText.substring(0, process.stdout.columns - 4) + '...';
|
||||
}
|
||||
process.stdout.write(chalk.red(`${stdoutText}`));
|
||||
|
||||
lastTranscriptWasFinal = false;
|
||||
}
|
||||
};
|
||||
|
||||
const audioInputStreamTransform = new Writable({
|
||||
write(chunk, encoding, next) {
|
||||
if (newStream && lastAudioInput.length !== 0) {
|
||||
// Approximate math to calculate time of chunks
|
||||
const chunkTime = streamingLimit / lastAudioInput.length;
|
||||
if (chunkTime !== 0) {
|
||||
if (bridgingOffset < 0) {
|
||||
bridgingOffset = 0;
|
||||
}
|
||||
if (bridgingOffset > finalRequestEndTime) {
|
||||
bridgingOffset = finalRequestEndTime;
|
||||
}
|
||||
const chunksFromMS = Math.floor(
|
||||
(finalRequestEndTime - bridgingOffset) / chunkTime
|
||||
);
|
||||
bridgingOffset = Math.floor(
|
||||
(lastAudioInput.length - chunksFromMS) * chunkTime
|
||||
);
|
||||
|
||||
for (let i = chunksFromMS; i < lastAudioInput.length; i++) {
|
||||
recognizeStream.write(lastAudioInput[i]);
|
||||
}
|
||||
}
|
||||
newStream = false;
|
||||
}
|
||||
|
||||
audioInput.push(chunk);
|
||||
|
||||
if (recognizeStream) {
|
||||
recognizeStream.write(chunk);
|
||||
}
|
||||
|
||||
next();
|
||||
},
|
||||
|
||||
final() {
|
||||
if (recognizeStream) {
|
||||
// recognize event is a stream readable object
|
||||
recognizeStream.end();
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
function restartStream() {
|
||||
if (recognizeStream) {
|
||||
recognizeStream.end();
|
||||
recognizeStream.removeListener('data', speechCallback);
|
||||
recognizeStream = null;
|
||||
}
|
||||
if (resultEndTime > 0) {
|
||||
finalRequestEndTime = isFinalEndTime;
|
||||
}
|
||||
resultEndTime = 0;
|
||||
|
||||
lastAudioInput = [];
|
||||
lastAudioInput = audioInput;
|
||||
|
||||
restartCounter++;
|
||||
|
||||
if (!lastTranscriptWasFinal) {
|
||||
process.stdout.write('\n');
|
||||
}
|
||||
process.stdout.write(
|
||||
chalk.yellow(`${streamingLimit * restartCounter}: RESTARTING REQUEST\n`)
|
||||
);
|
||||
|
||||
newStream = true;
|
||||
|
||||
startStream();
|
||||
}
|
||||
// Start recording and send the microphone input to the Speech API
|
||||
recorder
|
||||
.record({
|
||||
sampleRateHertz: sampleRateHertz,
|
||||
threshold: 0, // Silence threshold
|
||||
silence: 1000,
|
||||
keepSilence: true,
|
||||
recordProgram: 'rec', // Try also "arecord" or "sox"
|
||||
})
|
||||
.stream()
|
||||
.on('error', err => {
|
||||
console.error('Audio recording error ' + err);
|
||||
})
|
||||
.pipe(audioInputStreamTransform);
|
||||
|
||||
console.log('');
|
||||
console.log('Listening, press Ctrl+C to stop.');
|
||||
console.log('');
|
||||
console.log('End (ms) Transcript Results/Status');
|
||||
console.log('=========================================================');
|
||||
|
||||
// startStream();
|
||||
// [END speech_transcribe_infinite_streaming]
|
||||
}
|
||||
|
||||
// process.on('unhandledRejection', err => {
|
||||
// console.error(err.message);
|
||||
// process.exitCode = 1;
|
||||
// });
|
||||
|
||||
main(...process.argv.slice(2));
|
||||
201
src/audio.ts
201
src/audio.ts
|
|
@ -1,201 +0,0 @@
|
|||
function main(
|
||||
encoding = 'LINEAR16',
|
||||
sampleRateHertz = 16000,
|
||||
languageCode = 'en-US',
|
||||
streamingLimit = 290000
|
||||
) {
|
||||
// [START speech_transcribe_infinite_streaming]
|
||||
|
||||
// const encoding = 'LINEAR16';
|
||||
// const sampleRateHertz = 16000;
|
||||
// const languageCode = 'en-US';
|
||||
// const streamingLimit = 10000; // ms - set to low number for demo purposes
|
||||
|
||||
const chalk = require('chalk');
|
||||
const {Writable} = require('stream');
|
||||
const recorder = require('node-record-lpcm16');
|
||||
const fs = require('fs');
|
||||
const writeStream = fs.createWriteStream('test.wav', { encoding: 'binary'});
|
||||
|
||||
// // Imports the Google Cloud client library
|
||||
// // Currently, only v1p1beta1 contains result-end-time
|
||||
// const speech = require('@google-cloud/speech').v1p1beta1;
|
||||
|
||||
// const client = new speech.SpeechClient();
|
||||
|
||||
// const config = {
|
||||
// encoding: encoding,
|
||||
// sampleRateHertz: sampleRateHertz,
|
||||
// languageCode: languageCode,
|
||||
// };
|
||||
|
||||
// const request = {
|
||||
// config,
|
||||
// interimResults: true,
|
||||
// };
|
||||
|
||||
// let recognizeStream = null;
|
||||
// let restartCounter = 0;
|
||||
// let audioInput = [];
|
||||
// let lastAudioInput = [];
|
||||
// let resultEndTime = 0;
|
||||
// let isFinalEndTime = 0;
|
||||
// let finalRequestEndTime = 0;
|
||||
// let newStream = true;
|
||||
// let bridgingOffset = 0;
|
||||
// let lastTranscriptWasFinal = false;
|
||||
|
||||
// function startStream() {
|
||||
// // Clear current audioInput
|
||||
// audioInput = [];
|
||||
// // Initiate (Reinitiate) a recognize stream
|
||||
// recognizeStream = client
|
||||
// .streamingRecognize(request)
|
||||
// .on('error', err => {
|
||||
// if (err.code === 11) {
|
||||
// // restartStream();
|
||||
// } else {
|
||||
// console.error('API request error ' + err);
|
||||
// }
|
||||
// })
|
||||
// .on('data', speechCallback);
|
||||
|
||||
// // Restart stream when streamingLimit expires
|
||||
// setTimeout(restartStream, streamingLimit);
|
||||
// }
|
||||
|
||||
// const speechCallback = stream => {
|
||||
// // Convert API result end time from seconds + nanoseconds to milliseconds
|
||||
// resultEndTime =
|
||||
// stream.results[0].resultEndTime.seconds * 1000 +
|
||||
// Math.round(stream.results[0].resultEndTime.nanos / 1000000);
|
||||
|
||||
// // Calculate correct time based on offset from audio sent twice
|
||||
// const correctedTime =
|
||||
// resultEndTime - bridgingOffset + streamingLimit * restartCounter;
|
||||
|
||||
// process.stdout.clearLine();
|
||||
// process.stdout.cursorTo(0);
|
||||
// let stdoutText = '';
|
||||
// if (stream.results[0] && stream.results[0].alternatives[0]) {
|
||||
// stdoutText =
|
||||
// correctedTime + ': ' + stream.results[0].alternatives[0].transcript;
|
||||
// }
|
||||
|
||||
// if (stream.results[0].isFinal) {
|
||||
// process.stdout.write(chalk.green(`${stdoutText}\n`));
|
||||
|
||||
// isFinalEndTime = resultEndTime;
|
||||
// lastTranscriptWasFinal = true;
|
||||
// } else {
|
||||
// // Make sure transcript does not exceed console character length
|
||||
// if (stdoutText.length > process.stdout.columns) {
|
||||
// stdoutText =
|
||||
// stdoutText.substring(0, process.stdout.columns - 4) + '...';
|
||||
// }
|
||||
// process.stdout.write(chalk.red(`${stdoutText}`));
|
||||
|
||||
// lastTranscriptWasFinal = false;
|
||||
// }
|
||||
// };
|
||||
|
||||
// const audioInputStreamTransform = new Writable({
|
||||
// write(chunk, encoding, next) {
|
||||
// if (newStream && lastAudioInput.length !== 0) {
|
||||
// // Approximate math to calculate time of chunks
|
||||
// const chunkTime = streamingLimit / lastAudioInput.length;
|
||||
// if (chunkTime !== 0) {
|
||||
// if (bridgingOffset < 0) {
|
||||
// bridgingOffset = 0;
|
||||
// }
|
||||
// if (bridgingOffset > finalRequestEndTime) {
|
||||
// bridgingOffset = finalRequestEndTime;
|
||||
// }
|
||||
// const chunksFromMS = Math.floor(
|
||||
// (finalRequestEndTime - bridgingOffset) / chunkTime
|
||||
// );
|
||||
// bridgingOffset = Math.floor(
|
||||
// (lastAudioInput.length - chunksFromMS) * chunkTime
|
||||
// );
|
||||
|
||||
// for (let i = chunksFromMS; i < lastAudioInput.length; i++) {
|
||||
// recognizeStream.write(lastAudioInput[i]);
|
||||
// }
|
||||
// }
|
||||
// newStream = false;
|
||||
// }
|
||||
|
||||
// audioInput.push(chunk);
|
||||
|
||||
// if (recognizeStream) {
|
||||
// recognizeStream.write(chunk);
|
||||
// }
|
||||
|
||||
// next();
|
||||
// },
|
||||
|
||||
// final() {
|
||||
// if (recognizeStream) {
|
||||
// recognizeStream.end();
|
||||
// }
|
||||
// },
|
||||
// });
|
||||
|
||||
// function restartStream() {
|
||||
// if (recognizeStream) {
|
||||
// recognizeStream.end();
|
||||
// recognizeStream.removeListener('data', speechCallback);
|
||||
// recognizeStream = null;
|
||||
// }
|
||||
// if (resultEndTime > 0) {
|
||||
// finalRequestEndTime = isFinalEndTime;
|
||||
// }
|
||||
// resultEndTime = 0;
|
||||
|
||||
// lastAudioInput = [];
|
||||
// lastAudioInput = audioInput;
|
||||
|
||||
// restartCounter++;
|
||||
|
||||
// if (!lastTranscriptWasFinal) {
|
||||
// process.stdout.write('\n');
|
||||
// }
|
||||
// process.stdout.write(
|
||||
// chalk.yellow(`${streamingLimit * restartCounter}: RESTARTING REQUEST\n`)
|
||||
// );
|
||||
|
||||
// newStream = true;
|
||||
|
||||
// startStream();
|
||||
// }
|
||||
// Start recording and send the microphone input to the Speech API
|
||||
recorder
|
||||
.record({
|
||||
sampleRateHertz: sampleRateHertz,
|
||||
threshold: 0, // Silence threshold
|
||||
silence: 1000,
|
||||
keepSilence: true,
|
||||
recordProgram: 'rec', // Try also "arecord" or "sox"
|
||||
})
|
||||
.stream()
|
||||
.on('error', err => {
|
||||
console.error('Audio recording error ' + err);
|
||||
})
|
||||
.pipe(audioInputStreamTransform);
|
||||
|
||||
console.log('');
|
||||
console.log('Listening, press Ctrl+C to stop.');
|
||||
console.log('');
|
||||
console.log('End (ms) Transcript Results/Status');
|
||||
console.log('=========================================================');
|
||||
|
||||
// startStream();
|
||||
// [END speech_transcribe_infinite_streaming]
|
||||
}
|
||||
|
||||
// process.on('unhandledRejection', err => {
|
||||
// console.error(err.message);
|
||||
// process.exitCode = 1;
|
||||
// });
|
||||
|
||||
main(...process.argv.slice(2));
|
||||
|
|
@ -1,92 +1,41 @@
|
|||
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
||||
const portAudio = require('naudiodon');
|
||||
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
||||
const { Writable } = require('stream');
|
||||
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
||||
const { StringDecoder } = require('string_decoder');
|
||||
|
||||
interface RecorderI {
|
||||
stop(): string;
|
||||
pause(): void;
|
||||
}
|
||||
let audio: string;
|
||||
let audioInput: string[] = [];
|
||||
|
||||
type inputOptions = {
|
||||
channelCount: number;
|
||||
sampleFormat: number;
|
||||
sampleRate: number;
|
||||
deviceId: number;
|
||||
closeOnError: boolean;
|
||||
}
|
||||
export function initRecorder() {
|
||||
|
||||
class StringWritable extends Writable {
|
||||
|
||||
constructor(options: {
|
||||
defaultEncoding: string;
|
||||
}) {
|
||||
super(options);
|
||||
this._decoder = new StringDecoder(options && options.defaultEncoding);
|
||||
this.data = '';
|
||||
}
|
||||
|
||||
_write(chunk: Buffer, encoding: string, callback: (error? : Error) => void) {
|
||||
if (encoding === 'buffer') {
|
||||
chunk = this._decoder.write(chunk);
|
||||
}
|
||||
this.data += chunk;
|
||||
console.log('writing');
|
||||
callback();
|
||||
}
|
||||
|
||||
_final(callback: (error? : Error) => void) {
|
||||
console.log('yayayay');
|
||||
this.data += this._decoder.end();
|
||||
callback();
|
||||
}
|
||||
}
|
||||
|
||||
export class Recorder implements RecorderI {
|
||||
|
||||
private ia: any;
|
||||
private micWritable: StringWritable;
|
||||
private recorderOptions: inputOptions;
|
||||
|
||||
constructor(options: inputOptions, encoding: string) {
|
||||
this.recorderOptions = options;
|
||||
this.micWritable = new StringWritable({
|
||||
defaultEncoding: encoding
|
||||
let ia = new portAudio.AudioIO({
|
||||
inOptions: {
|
||||
channelCount: 1,
|
||||
sampleFormat: 16,
|
||||
sampleRate: 16000,
|
||||
deviceId: -1,
|
||||
closeOnError: false,
|
||||
}
|
||||
});
|
||||
try {
|
||||
this.start();
|
||||
} catch(e) {
|
||||
console.log('RECORDER ERROR!', e);
|
||||
}
|
||||
}
|
||||
|
||||
private start(): void {
|
||||
this.ia = new portAudio.AudioIO({
|
||||
inOptions: this.recorderOptions
|
||||
ia.setEncoding('base64');
|
||||
ia.start();
|
||||
ia.pause();
|
||||
ia.on('error', (e: Error) => {
|
||||
console.log('error recording audio', + e);
|
||||
});
|
||||
ia.on('data', (chunk: string) => {
|
||||
audioInput.push(chunk);
|
||||
console.log('Got %d characters of string data:', chunk.length);
|
||||
});
|
||||
this.ia.pipe(this.micWritable);
|
||||
this.ia.start();
|
||||
this.pause();
|
||||
}
|
||||
|
||||
pause(): void {
|
||||
this.ia.pause();
|
||||
}
|
||||
|
||||
resume(){
|
||||
this.ia.resume();
|
||||
}
|
||||
|
||||
getData(): string {
|
||||
const data = this.micWritable.data;
|
||||
this.micWritable.data = '';
|
||||
return data;
|
||||
}
|
||||
|
||||
stop(): string {
|
||||
this.ia.quit();
|
||||
return this.getData();
|
||||
}
|
||||
}
|
||||
|
||||
async function processData(chunks: string[]): Buffer | Error {
|
||||
audio = '';
|
||||
chunks.forEach(s => {
|
||||
audio += s;
|
||||
})
|
||||
const buffer = Buffer.from(audio, "base64");
|
||||
audioInput = [];
|
||||
return buffer;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,40 +1,34 @@
|
|||
import { createWindow } from './createWindow';
|
||||
import WebSocket from 'ws';
|
||||
import { Recorder } from './recorder';
|
||||
import { ipcMain, BrowserWindow } from "electron";
|
||||
import WebSocket from 'ws';
|
||||
import { windowEmitter } from './windowEmitter';
|
||||
const portAudio = require('naudiodon');
|
||||
|
||||
/*
|
||||
* The main function will be run after electron app is ready.
|
||||
*/
|
||||
export function main(): void {
|
||||
|
||||
let audioInput: string[] = [];
|
||||
|
||||
// create main window.
|
||||
let win: BrowserWindow | null;
|
||||
win = createWindow({ width: 350, height: 525, resizable: false });
|
||||
const win = createWindow({ width: 350, height: 525, resizable: false });
|
||||
|
||||
// Instantiate ws session with crimata-platorm.
|
||||
const ws = new WebSocket('ws://localhost:8080');
|
||||
|
||||
// Instantiate portAudio recorder.
|
||||
const recorder = new Recorder({
|
||||
channelCount: 1,
|
||||
sampleFormat: 16,
|
||||
sampleRate: 16000,
|
||||
deviceId: -1,
|
||||
closeOnError: false,
|
||||
}, 'binary');
|
||||
// Instantiate socket session with crimata-platorm.
|
||||
const socket = new WebSocket('socket://localhost:8080');
|
||||
socket.binaryType = 'arraybuffer';
|
||||
|
||||
const initSession = (): void => {
|
||||
ws.send('hernandeze2@xavier.edu');
|
||||
socket.send('hernandeze2@xavier.edu');
|
||||
}
|
||||
|
||||
const sendMessage = (Event: any, payload: any): void => {
|
||||
ws.send(JSON.stringify(payload));
|
||||
socket.send(JSON.stringify(payload));
|
||||
}
|
||||
|
||||
const sendAudio = (data: string): void => {
|
||||
ws.send(data);
|
||||
const sendAudio = (data: Buffer): void => {
|
||||
socket.send(data);
|
||||
}
|
||||
|
||||
const receiveMessage = (message: string): void => {
|
||||
|
|
@ -45,39 +39,65 @@ export function main(): void {
|
|||
});
|
||||
}
|
||||
|
||||
const updateRecorder = (Event: any, record: boolean): void => {
|
||||
const updateRecorder = async (Event: any, record: boolean): void => {
|
||||
if (record) {
|
||||
recorder.resume();
|
||||
ia.resume();
|
||||
} else {
|
||||
recorder.pause();
|
||||
const data = recorder.getData();
|
||||
sendAudio(data);
|
||||
ia.pause();
|
||||
const audio = processAudio();
|
||||
sendAudio(audio);
|
||||
audioInput = [];
|
||||
}
|
||||
}
|
||||
|
||||
function processAudio(): Buffer {
|
||||
let audio = '';
|
||||
audioInput.forEach(s => {
|
||||
audio += s;
|
||||
});
|
||||
const buffer = Buffer.from(audio, "base64");
|
||||
return buffer;
|
||||
}
|
||||
|
||||
const windowMount = (): void => {
|
||||
console.log('testing mount');
|
||||
// ipc event handlers
|
||||
windowEmitter.emit('window-active', true);
|
||||
// ipc event handlers
|
||||
ipcMain.on('update-recorder', updateRecorder);
|
||||
ipcMain.on('send-message', sendMessage);
|
||||
}
|
||||
|
||||
const windowDismount = (): void => {
|
||||
console.log('testing dismount');
|
||||
// TODO: Gracefully close ws connection
|
||||
win = null;
|
||||
windowEmitter.emit('window-active', false);
|
||||
// TODO: Gracefully close socket connection
|
||||
ipcMain.removeListener('update-recorder', updateRecorder);
|
||||
ipcMain.removeListener('send-message', sendMessage);
|
||||
}
|
||||
|
||||
// ws handlers
|
||||
ws.on('connect', initSession);
|
||||
ws.on("message", receiveMessage);
|
||||
// socket handlers
|
||||
socket.on('connect', initSession);
|
||||
socket.on("message", receiveMessage);
|
||||
|
||||
// Handle window mount and dismount.
|
||||
win.webContents.on('did-finish-load', windowMount);
|
||||
win.on("closed", windowDismount);
|
||||
|
||||
let ia = new portAudio.AudioIO({
|
||||
inOptions: {
|
||||
channelCount: 1,
|
||||
sampleFormat: 16,
|
||||
sampleRate: 16000,
|
||||
deviceId: -1,
|
||||
closeOnError: false,
|
||||
}
|
||||
});
|
||||
ia.setEncoding('base64');
|
||||
ia.start();
|
||||
ia.on('error', (e: Error) => {
|
||||
console.log('error recording audio', + e);
|
||||
});
|
||||
ia.pause();
|
||||
ia.on('data', (chunk: string) => {
|
||||
audioInput.push(chunk);
|
||||
console.log('Got %d characters of string data:', chunk.length);
|
||||
});
|
||||
}
|
||||
|
|
|
|||
127
testAudio.js
127
testAudio.js
|
|
@ -2,86 +2,101 @@
|
|||
const fs = require('fs');
|
||||
const speech = require('@google-cloud/speech');
|
||||
const portAudio = require('naudiodon');
|
||||
const rs = fs.createReadStream('rawAudio.wav');
|
||||
const WebSocket = require("ws")
|
||||
const {Writable} = require('stream');
|
||||
|
||||
// Creates a client
|
||||
const client = new speech.SpeechClient();
|
||||
|
||||
// connect to test server
|
||||
const ws = new WebSocket("ws://localhost:8080");
|
||||
// const ws = new WebSocket("ws://localhost:8080");
|
||||
// Creates a client const client = new speech.SpeechClient();
|
||||
|
||||
// google cloud speech to text settings
|
||||
const encoding = 'LINEAR16';
|
||||
const sampleRateHertz = 44100;
|
||||
const sampleRateHertz = 16000;
|
||||
const languageCode = 'en-US';
|
||||
|
||||
const audioChunks = [];
|
||||
|
||||
const audioInputStreamTransform = new Writable({
|
||||
write(chunk, encoding, next) {
|
||||
console.log(chunk);
|
||||
audioChunks.push(chunk);
|
||||
next();
|
||||
},
|
||||
|
||||
final() {
|
||||
console.log(audioChunks);
|
||||
},
|
||||
});
|
||||
|
||||
const config = {
|
||||
encoding: encoding,
|
||||
sampleRateHertz: sampleRateHertz,
|
||||
languageCode: languageCode,
|
||||
};
|
||||
|
||||
const request = {
|
||||
config,
|
||||
interimResults: true,
|
||||
};
|
||||
/**
|
||||
* Note that transcription is limited to 60 seconds audio.
|
||||
* Use a GCS file for audio longer than 1 minute.
|
||||
*/
|
||||
let audio;
|
||||
|
||||
const speechCallback = (data) => {
|
||||
console.log(
|
||||
`Transcription: ${data.results[0].alternatives[0].transcript}`
|
||||
);
|
||||
async function transcribeSpeech (audio) {
|
||||
const request = {
|
||||
config: config,
|
||||
audio: audio,
|
||||
};
|
||||
|
||||
// Detects speech in the audio file. This creates a recognition job that you
|
||||
// can wait for now, or get its result later.
|
||||
const [operation] = await client.longRunningRecognize(request);
|
||||
|
||||
// Get a Promise representation of the final result of the job
|
||||
const [response] = await operation.promise();
|
||||
|
||||
const transcription = response.results
|
||||
.map(result => result.alternatives[0].transcript)
|
||||
.join('\n');
|
||||
console.log(`Transcription: ${transcription}`);
|
||||
}
|
||||
|
||||
const recognizeStream = client
|
||||
.streamingRecognize(request)
|
||||
.on('error', err => {
|
||||
if (err.code === 11) {
|
||||
// restartStream();
|
||||
} else {
|
||||
console.error('API request error ' + err);
|
||||
}
|
||||
})
|
||||
.on('data', speechCallback);
|
||||
const audioChunks = [];
|
||||
|
||||
let audioInput = [];
|
||||
// Create an instance of AudioIO with inOptions (defaults are as below), which will return a ReadableStream
|
||||
const recorderOptions = {
|
||||
channelCount: 1,
|
||||
sampleFormat: portAudio.sampleFormat16Bit,
|
||||
sampleRate: 16000,
|
||||
deviceId: -1, // Use -1 or omit the deviceId to select the default device
|
||||
closeOnError: false // Close the stream if an audio error is detected, if set false then just log the error
|
||||
}
|
||||
|
||||
const ai = new portAudio.AudioIO({
|
||||
inOptions: recorderOptions
|
||||
let ia = new portAudio.AudioIO({
|
||||
inOptions: {
|
||||
channelCount: 1,
|
||||
sampleFormat: 16,
|
||||
sampleRate: 16000,
|
||||
deviceId: -1,
|
||||
closeOnError: false,
|
||||
}
|
||||
});
|
||||
ia.setEncoding('base64');
|
||||
ia.start();
|
||||
ia.on('error', (e) => {
|
||||
console.log('error recording audio', + e);
|
||||
});
|
||||
ia.on('data', (chunk) => {
|
||||
audioInput.push(chunk);
|
||||
console.log('Got %d characters of string data:', chunk.length);
|
||||
});
|
||||
|
||||
// manipulate individual chunks as they're available
|
||||
// const audioData = [];
|
||||
// ai.on('data', (d) => {
|
||||
// audioData.push(d.toString('binary'))
|
||||
// })
|
||||
async function processAudio() {
|
||||
audio = '';
|
||||
audioInput.forEach(s => {
|
||||
audio += s;
|
||||
})
|
||||
const buffer = Buffer.from(audio, 'base64');
|
||||
audioInput = [];
|
||||
await transcribeSpeech({
|
||||
content: buffer
|
||||
});
|
||||
}
|
||||
|
||||
setTimeout(async () => {
|
||||
ia.pause();
|
||||
await processAudio();
|
||||
// ia.resume();
|
||||
}, 3000);
|
||||
|
||||
ai.pipe(audioInputStreamTransform)
|
||||
ai.start();
|
||||
setTimeout(() => {
|
||||
ai.quit();
|
||||
}, 2000)
|
||||
ia.resume();
|
||||
}, 4000)
|
||||
|
||||
setTimeout(async () => {
|
||||
ia.pause();
|
||||
await processAudio();
|
||||
// ia.resume();
|
||||
}, 8000);
|
||||
|
||||
setTimeout(() => {
|
||||
ia.pause();
|
||||
}, 9000)
|
||||
|
|
|
|||
|
|
@ -10,25 +10,6 @@ const testMessage = {
|
|||
}
|
||||
const wss = new WebSocket.Server({
|
||||
port: 8080
|
||||
// perMessageDeflate: {
|
||||
// zlibDeflateOptions: {
|
||||
// // See zlib defaults.
|
||||
// chunkSize: 1024,
|
||||
// memLevel: 7,
|
||||
// level: 3
|
||||
// },
|
||||
// zlibInflateOptions: {
|
||||
// chunkSize: 10 * 1024
|
||||
// },
|
||||
// // Other options settable:
|
||||
// clientNoContextTakeover: true, // Defaults to negotiated value.
|
||||
// serverNoContextTakeover: true, // Defaults to negotiated value.
|
||||
// serverMaxWindowBits: 10, // Defaults to negotiated value.
|
||||
// // Below options specified as default values.
|
||||
// concurrencyLimit: 10, // Limits zlib concurrency for perf.
|
||||
// threshold: 1024 // Size (in bytes) below which messages
|
||||
// // should not be compressed.
|
||||
// }
|
||||
});
|
||||
|
||||
wss.on("connection", function connection(ws, req) {
|
||||
|
|
|
|||
|
|
@ -16,7 +16,8 @@
|
|||
"webpack-env",
|
||||
"jest",
|
||||
"animejs",
|
||||
"dom-mediacapture-record"
|
||||
"dom-mediacapture-record",
|
||||
"node"
|
||||
],
|
||||
"paths": {
|
||||
"@/*": [
|
||||
|
|
|
|||
63
ws.py
63
ws.py
|
|
@ -11,59 +11,23 @@ from google.cloud import speech
|
|||
# Instantiates a client
|
||||
client = speech.SpeechClient()
|
||||
|
||||
|
||||
# config = speech.RecognitionConfig(
|
||||
# encoding=speech.RecognitionConfig.AudioEncoding.LINEAR16,
|
||||
# sample_rate_hertz=16000,
|
||||
# language_code="en-US",
|
||||
# )
|
||||
|
||||
# streaming_config = speech.StreamingRecognitionConfig(config=config)
|
||||
|
||||
# def transcribe(stream):
|
||||
|
||||
# requests = (
|
||||
# speech.StreamingRecognizeRequest(audio_content=chunk) for chunk in stream
|
||||
# )
|
||||
|
||||
# # Detects speech in the audio file
|
||||
# responses = client.streaming_recognize(
|
||||
# config=config,
|
||||
# requests=requests
|
||||
# )
|
||||
|
||||
# for response in responses:
|
||||
# # Once the transcription has settled, the first result will contain the
|
||||
# # is_final result. The other results will be for subsequent portions of
|
||||
# # the audio.
|
||||
# for result in response.results:
|
||||
# print("Finished: {}".format(result.is_final))
|
||||
# print("Stability: {}".format(result.stability))
|
||||
# alternatives = result.alternatives
|
||||
# # The alternatives are ordered from most likely to least.
|
||||
# for alternative in alternatives:
|
||||
# print("Confidence: {}".format(alternative.confidence))
|
||||
# print(u"Transcript: {}".format(alternative.transcript))
|
||||
|
||||
|
||||
speechtotext_client = speech.SpeechClient()
|
||||
|
||||
config = speech.RecognitionConfig(
|
||||
audio_channel_count=1,
|
||||
encoding=speech.RecognitionConfig.AudioEncoding.LINEAR16,
|
||||
sample_rate_hertz=44100,
|
||||
sample_rate_hertz=16000,
|
||||
language_code="en-US",
|
||||
)
|
||||
|
||||
# Transcribe an audio file.
|
||||
def transcribe(content):
|
||||
input(content)
|
||||
input(type(content))
|
||||
|
||||
content = speech.RecognitionAudio(content=content)
|
||||
print(len(content))
|
||||
audio = speech.RecognitionAudio(content=content)
|
||||
|
||||
# Detects speech in the audio file
|
||||
response = speechtotext_client.recognize(config=config, audio=content)
|
||||
response = speechtotext_client.recognize(config=config, audio=audio)
|
||||
input(f"Response: {response}")
|
||||
|
||||
for result in response.results:
|
||||
|
|
@ -73,18 +37,21 @@ def transcribe(content):
|
|||
async def session(websocket, path):
|
||||
buf = []
|
||||
|
||||
while True:
|
||||
chunk = await websocket.recv()
|
||||
chunk = await websocket.recv()
|
||||
input(type(chunk))
|
||||
transcribe(chunk)
|
||||
# while True:
|
||||
# chunk = await websocket.recv_frame()
|
||||
|
||||
if chunk == "stop":
|
||||
break
|
||||
# if chunk == "stop":
|
||||
# break
|
||||
|
||||
buf.append(chunk)
|
||||
# buf.append(chunk)
|
||||
|
||||
input(f"Buffer sample: {buf[0]}")
|
||||
audio = b"".join(buf)
|
||||
# input(f"Buffer sample: {buf[0]}")
|
||||
# audio = b"".join(buf)
|
||||
|
||||
transcribe(audio)
|
||||
# transcribe(audio)
|
||||
|
||||
|
||||
start_server = websockets.serve(session, "localhost", 8080)
|
||||
|
|
|
|||
|
|
@ -1556,6 +1556,11 @@
|
|||
resolved "https://registry.yarnpkg.com/@types/node/-/node-13.13.40.tgz#f655ef327362cc83912f2e69336ddc62a24a9f88"
|
||||
integrity sha512-eKaRo87lu1yAXrzEJl0zcJxfUMDT5/mZalFyOkT44rnQps41eS2pfWzbaulSPpQLFNy29bFqn+Y5lOTL8ATlEQ==
|
||||
|
||||
"@types/node@^14.14.25":
|
||||
version "14.14.25"
|
||||
resolved "https://registry.yarnpkg.com/@types/node/-/node-14.14.25.tgz#15967a7b577ff81383f9b888aa6705d43fbbae93"
|
||||
integrity sha512-EPpXLOVqDvisVxtlbvzfyqSsFeQxltFbluZNRndIb8tr9KiBnYNLzrc1N3pyKUCww2RNrfHDViqDWWE1LCJQtQ==
|
||||
|
||||
"@types/normalize-package-data@^2.4.0":
|
||||
version "2.4.0"
|
||||
resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz#e486d0d97396d79beedd0a6e33f4534ff6b4973e"
|
||||
|
|
|
|||
Loading…
Reference in a new issue