cleaning some things up
This commit is contained in:
Andrew Gundersen
parent
7ea09b99f0
commit
9bf01c0a33
12 changed files with 48 additions and 694 deletions
|
|
@ -5,11 +5,12 @@
|
|||
</template>
|
||||
|
||||
<style lang="scss">
|
||||
html,
|
||||
body {
|
||||
|
||||
html, body {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
#app {
|
||||
font-family: Avenir, Helvetica, Arial, sans-serif;
|
||||
-webkit-font-smoothing: antialiased;
|
||||
|
|
|
|||
Binary file not shown.
|
Before Width: | Height: | Size: 6.7 KiB |
201
src/audio.ts
201
src/audio.ts
|
|
@ -1,201 +0,0 @@
|
|||
function main(
|
||||
encoding = 'LINEAR16',
|
||||
sampleRateHertz = 16000,
|
||||
languageCode = 'en-US',
|
||||
streamingLimit = 290000
|
||||
) {
|
||||
// [START speech_transcribe_infinite_streaming]
|
||||
|
||||
// const encoding = 'LINEAR16';
|
||||
// const sampleRateHertz = 16000;
|
||||
// const languageCode = 'en-US';
|
||||
// const streamingLimit = 10000; // ms - set to low number for demo purposes
|
||||
|
||||
const chalk = require('chalk');
|
||||
const {Writable} = require('stream');
|
||||
const recorder = require('node-record-lpcm16');
|
||||
const fs = require('fs');
|
||||
const writeStream = fs.createWriteStream('test.wav', { encoding: 'binary'});
|
||||
|
||||
// // Imports the Google Cloud client library
|
||||
// // Currently, only v1p1beta1 contains result-end-time
|
||||
// const speech = require('@google-cloud/speech').v1p1beta1;
|
||||
|
||||
// const client = new speech.SpeechClient();
|
||||
|
||||
// const config = {
|
||||
// encoding: encoding,
|
||||
// sampleRateHertz: sampleRateHertz,
|
||||
// languageCode: languageCode,
|
||||
// };
|
||||
|
||||
// const request = {
|
||||
// config,
|
||||
// interimResults: true,
|
||||
// };
|
||||
|
||||
// let recognizeStream = null;
|
||||
// let restartCounter = 0;
|
||||
// let audioInput = [];
|
||||
// let lastAudioInput = [];
|
||||
// let resultEndTime = 0;
|
||||
// let isFinalEndTime = 0;
|
||||
// let finalRequestEndTime = 0;
|
||||
// let newStream = true;
|
||||
// let bridgingOffset = 0;
|
||||
// let lastTranscriptWasFinal = false;
|
||||
|
||||
// function startStream() {
|
||||
// // Clear current audioInput
|
||||
// audioInput = [];
|
||||
// // Initiate (Reinitiate) a recognize stream
|
||||
// recognizeStream = client
|
||||
// .streamingRecognize(request)
|
||||
// .on('error', err => {
|
||||
// if (err.code === 11) {
|
||||
// // restartStream();
|
||||
// } else {
|
||||
// console.error('API request error ' + err);
|
||||
// }
|
||||
// })
|
||||
// .on('data', speechCallback);
|
||||
|
||||
// // Restart stream when streamingLimit expires
|
||||
// setTimeout(restartStream, streamingLimit);
|
||||
// }
|
||||
|
||||
// const speechCallback = stream => {
|
||||
// // Convert API result end time from seconds + nanoseconds to milliseconds
|
||||
// resultEndTime =
|
||||
// stream.results[0].resultEndTime.seconds * 1000 +
|
||||
// Math.round(stream.results[0].resultEndTime.nanos / 1000000);
|
||||
|
||||
// // Calculate correct time based on offset from audio sent twice
|
||||
// const correctedTime =
|
||||
// resultEndTime - bridgingOffset + streamingLimit * restartCounter;
|
||||
|
||||
// process.stdout.clearLine();
|
||||
// process.stdout.cursorTo(0);
|
||||
// let stdoutText = '';
|
||||
// if (stream.results[0] && stream.results[0].alternatives[0]) {
|
||||
// stdoutText =
|
||||
// correctedTime + ': ' + stream.results[0].alternatives[0].transcript;
|
||||
// }
|
||||
|
||||
// if (stream.results[0].isFinal) {
|
||||
// process.stdout.write(chalk.green(`${stdoutText}\n`));
|
||||
|
||||
// isFinalEndTime = resultEndTime;
|
||||
// lastTranscriptWasFinal = true;
|
||||
// } else {
|
||||
// // Make sure transcript does not exceed console character length
|
||||
// if (stdoutText.length > process.stdout.columns) {
|
||||
// stdoutText =
|
||||
// stdoutText.substring(0, process.stdout.columns - 4) + '...';
|
||||
// }
|
||||
// process.stdout.write(chalk.red(`${stdoutText}`));
|
||||
|
||||
// lastTranscriptWasFinal = false;
|
||||
// }
|
||||
// };
|
||||
|
||||
// const audioInputStreamTransform = new Writable({
|
||||
// write(chunk, encoding, next) {
|
||||
// if (newStream && lastAudioInput.length !== 0) {
|
||||
// // Approximate math to calculate time of chunks
|
||||
// const chunkTime = streamingLimit / lastAudioInput.length;
|
||||
// if (chunkTime !== 0) {
|
||||
// if (bridgingOffset < 0) {
|
||||
// bridgingOffset = 0;
|
||||
// }
|
||||
// if (bridgingOffset > finalRequestEndTime) {
|
||||
// bridgingOffset = finalRequestEndTime;
|
||||
// }
|
||||
// const chunksFromMS = Math.floor(
|
||||
// (finalRequestEndTime - bridgingOffset) / chunkTime
|
||||
// );
|
||||
// bridgingOffset = Math.floor(
|
||||
// (lastAudioInput.length - chunksFromMS) * chunkTime
|
||||
// );
|
||||
|
||||
// for (let i = chunksFromMS; i < lastAudioInput.length; i++) {
|
||||
// recognizeStream.write(lastAudioInput[i]);
|
||||
// }
|
||||
// }
|
||||
// newStream = false;
|
||||
// }
|
||||
|
||||
// audioInput.push(chunk);
|
||||
|
||||
// if (recognizeStream) {
|
||||
// recognizeStream.write(chunk);
|
||||
// }
|
||||
|
||||
// next();
|
||||
// },
|
||||
|
||||
// final() {
|
||||
// if (recognizeStream) {
|
||||
// recognizeStream.end();
|
||||
// }
|
||||
// },
|
||||
// });
|
||||
|
||||
// function restartStream() {
|
||||
// if (recognizeStream) {
|
||||
// recognizeStream.end();
|
||||
// recognizeStream.removeListener('data', speechCallback);
|
||||
// recognizeStream = null;
|
||||
// }
|
||||
// if (resultEndTime > 0) {
|
||||
// finalRequestEndTime = isFinalEndTime;
|
||||
// }
|
||||
// resultEndTime = 0;
|
||||
|
||||
// lastAudioInput = [];
|
||||
// lastAudioInput = audioInput;
|
||||
|
||||
// restartCounter++;
|
||||
|
||||
// if (!lastTranscriptWasFinal) {
|
||||
// process.stdout.write('\n');
|
||||
// }
|
||||
// process.stdout.write(
|
||||
// chalk.yellow(`${streamingLimit * restartCounter}: RESTARTING REQUEST\n`)
|
||||
// );
|
||||
|
||||
// newStream = true;
|
||||
|
||||
// startStream();
|
||||
// }
|
||||
// Start recording and send the microphone input to the Speech API
|
||||
recorder
|
||||
.record({
|
||||
sampleRateHertz: sampleRateHertz,
|
||||
threshold: 0, // Silence threshold
|
||||
silence: 1000,
|
||||
keepSilence: true,
|
||||
recordProgram: 'rec', // Try also "arecord" or "sox"
|
||||
})
|
||||
.stream()
|
||||
.on('error', err => {
|
||||
console.error('Audio recording error ' + err);
|
||||
})
|
||||
.pipe(audioInputStreamTransform);
|
||||
|
||||
console.log('');
|
||||
console.log('Listening, press Ctrl+C to stop.');
|
||||
console.log('');
|
||||
console.log('End (ms) Transcript Results/Status');
|
||||
console.log('=========================================================');
|
||||
|
||||
// startStream();
|
||||
// [END speech_transcribe_infinite_streaming]
|
||||
}
|
||||
|
||||
// process.on('unhandledRejection', err => {
|
||||
// console.error(err.message);
|
||||
// process.exitCode = 1;
|
||||
// });
|
||||
|
||||
main(...process.argv.slice(2));
|
||||
|
|
@ -1,67 +1,11 @@
|
|||
"use strict";
|
||||
|
||||
import { app, protocol, BrowserWindow, ipcMain } from "electron";
|
||||
import { createProtocol } from "vue-cli-plugin-electron-builder/lib";
|
||||
import installExtension, { VUEJS_DEVTOOLS } from "electron-devtools-installer";
|
||||
const isDevelopment = process.env.NODE_ENV !== "production";
|
||||
import * as path from "path";
|
||||
const fs = require('fs');
|
||||
import WebSocket from "ws";
|
||||
const portAudio = require('naudiodon');
|
||||
const speech = require('@google-cloud/speech');
|
||||
|
||||
const client = new speech.SpeechClient();
|
||||
|
||||
const encoding = 'LINEAR16';
|
||||
const sampleRateHertz = 16000;
|
||||
const languageCode = 'en-US';
|
||||
|
||||
const config = {
|
||||
encoding: encoding,
|
||||
sampleRateHertz: sampleRateHertz,
|
||||
languageCode: languageCode,
|
||||
};
|
||||
const request = {
|
||||
config,
|
||||
interimResults: true,
|
||||
};
|
||||
|
||||
const speechCallback = (d: any) => {
|
||||
console.log(d)
|
||||
}
|
||||
|
||||
const recognizeStream = client
|
||||
.streamingRecognize(request)
|
||||
.on('error', err => {
|
||||
if (err.code === 11) {
|
||||
// restartStream();
|
||||
} else {
|
||||
console.error('API request error ' + err);
|
||||
}
|
||||
})
|
||||
.on('data', speechCallback);
|
||||
|
||||
|
||||
// Create an instance of AudioIO with inOptions (defaults are as below), which will return a ReadableStream
|
||||
const audioOptions = {
|
||||
channelCount: 2,
|
||||
sampleFormat: portAudio.sampleFormat16Bit,
|
||||
sampleRate: 16000,
|
||||
deviceId: -1, // Use -1 or omit the deviceId to select the default device
|
||||
closeOnError: false // Close the stream if an audio error is detected, if set false then just log the error
|
||||
}
|
||||
|
||||
const ai = new portAudio.AudioIO({
|
||||
inOptions: audioOptions
|
||||
});
|
||||
ai.pipe(recognizeStream);
|
||||
ai.start();
|
||||
|
||||
|
||||
// const filename = "rawAudio.wav";
|
||||
// Create a write stream to write out to a raw audio file
|
||||
// const writeStream = fs.createWriteStream(filename, { encoding: 'binary'});
|
||||
// ai.pipe(writeStream);
|
||||
const isDevelopment = process.env.NODE_ENV !== "production";
|
||||
|
||||
// Keep a global reference of the window object, if you don't, the window will
|
||||
// be closed automatically when the JavaScript object is garbage collected.
|
||||
|
|
@ -73,18 +17,28 @@ protocol.registerSchemesAsPrivileged([
|
|||
]);
|
||||
|
||||
function createWindow() {
|
||||
|
||||
// Create the browser window.
|
||||
win = new BrowserWindow({
|
||||
width: 450,
|
||||
height: 1025,
|
||||
resizable: true,
|
||||
width: 350,
|
||||
height: 520,
|
||||
resizable: false,
|
||||
maximizable: false,
|
||||
|
||||
// Postition at launch
|
||||
x: 100,
|
||||
y: 100,
|
||||
|
||||
webPreferences: {
|
||||
// Use pluginOptions.nodeIntegration, leave this alone
|
||||
// See nklayman.github.io/vue-cli-plugin-electron-builder/guide/security.html#node-integration for more info
|
||||
nodeIntegration: (process.env
|
||||
.ELECTRON_NODE_INTEGRATION as unknown) as boolean,
|
||||
devTools: false,
|
||||
|
||||
nodeIntegration:
|
||||
(process.env.ELECTRON_NODE_INTEGRATION as unknown) as boolean,
|
||||
|
||||
preload: path.join(__dirname, "preload.js")
|
||||
|
||||
}
|
||||
|
||||
});
|
||||
|
||||
if (process.env.WEBPACK_DEV_SERVER_URL) {
|
||||
|
|
@ -98,12 +52,17 @@ function createWindow() {
|
|||
win.loadURL("app://./index.html");
|
||||
}
|
||||
|
||||
|
||||
win.webContents.on('did-finish-load', () => {
|
||||
|
||||
// Define the websocket
|
||||
const ws = new WebSocket("ws://localhost:8080");
|
||||
|
||||
// Send ID on open.
|
||||
ws.on("open", function open() {
|
||||
// ws.send("hernandeze2@xavier.edu");
|
||||
ws.send("gundersena@crimata.com");
|
||||
});
|
||||
|
||||
ws.on("message", (message: any) => {
|
||||
const parsed = JSON.parse(message);
|
||||
console.log('new message', message)
|
||||
|
|
@ -124,21 +83,7 @@ function createWindow() {
|
|||
})
|
||||
|
||||
ipcMain.on('update-recorder', (Event: any, record: boolean) => {
|
||||
if (record) {
|
||||
// ai.start();
|
||||
} else {
|
||||
// ai.quit();
|
||||
|
||||
// const audio = {
|
||||
// content: fs.readFileSync(filename).toString('base64'),
|
||||
// }
|
||||
// const message = {
|
||||
// text: "",
|
||||
// audio: audioData
|
||||
// }
|
||||
// ws.send(JSON.stringify(message))
|
||||
// audioData = "";
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
|
|
|
|||
|
|
@ -12,8 +12,6 @@
|
|||
</template>
|
||||
|
||||
<script lang="ts">
|
||||
// mock messages
|
||||
import Messages from "./mockMessages";
|
||||
|
||||
// child components
|
||||
import MessageVisualizer from "./messageVisualizer/index.vue";
|
||||
|
|
@ -23,44 +21,40 @@ import TextAudioInput from "@/components/taInput/index.vue";
|
|||
import { Mitt } from "@/types/mitt/index";
|
||||
|
||||
import { defineComponent, ref, inject } from "vue";
|
||||
|
||||
export default defineComponent({
|
||||
|
||||
name: "Messenger",
|
||||
components: { MessageVisualizer, TextAudioInput },
|
||||
|
||||
setup() {
|
||||
const messages = Messages;
|
||||
const count = ref(0);
|
||||
|
||||
// imports mitt Emitter safely
|
||||
let emitter: Mitt;
|
||||
const emitterInject: Mitt | undefined = inject("mitt");
|
||||
if (emitterInject) emitter = emitterInject;
|
||||
|
||||
function emittMessage() {
|
||||
if (count.value < messages.length) {
|
||||
emitter.emit("renderMessage", messages[count.value]);
|
||||
count.value++;
|
||||
return;
|
||||
}
|
||||
if (emitterInject) {
|
||||
emitter = emitterInject;
|
||||
}
|
||||
|
||||
// Render message when render-message event called.
|
||||
window.ipcRenderer.on("render-message", (event, payload) => {
|
||||
console.log(payload.message);
|
||||
emitter.emit("renderMessage", payload.message);
|
||||
});
|
||||
|
||||
return {
|
||||
emittMessage,
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
</script>
|
||||
|
||||
<style lang="scss" scoped>
|
||||
#messenger {
|
||||
width: 350px;
|
||||
height: 500px;
|
||||
background-color: #e6e6e6;
|
||||
box-shadow: 0px 3px 10px 0px rgba(0, 0, 0, 0.3);
|
||||
// border-radius: 20px;
|
||||
}
|
||||
#messenger {
|
||||
width: 350px;
|
||||
height: 500px;
|
||||
background-color: #e6e6e6;
|
||||
box-shadow: 0px 3px 10px 0px rgba(0, 0, 0, 0.3);
|
||||
// border-radius: 20px;
|
||||
}
|
||||
</style>
|
||||
|
|
|
|||
|
|
@ -1,127 +0,0 @@
|
|||
<template>
|
||||
<defs>
|
||||
<radialGradient
|
||||
id="radial-gradient"
|
||||
cx="0.5"
|
||||
cy="0.7"
|
||||
:r="radius"
|
||||
gradientTransform="translate(-1.489 1.5) rotate(-90) scale(1 1.989)"
|
||||
gradientUnits="objectBoundingBox"
|
||||
>
|
||||
<stop offset="0" stop-color="#fff" />
|
||||
<stop id="sound" offset="1" stop-color="rgb(230, 230, 230)" />
|
||||
</radialGradient>
|
||||
<filter id="Rectangle_410" x="0" y="0" width="380" height="530" filterUnits="userSpaceOnUse">
|
||||
<feOffset dy="3" input="SourceAlpha" />
|
||||
<feGaussianBlur stdDeviation="5" result="blur" />
|
||||
<feFlood flood-opacity="0.129" />
|
||||
<feComposite operator="in" in2="blur" />
|
||||
<feComposite in="SourceGraphic" />
|
||||
</filter>
|
||||
</defs>
|
||||
|
||||
<g transform="matrix(1, 0, 0, 1, 0, 0)" filter="url(#Rectangle_410)">
|
||||
<rect
|
||||
id="Rectangle_410-2"
|
||||
data-name="Rectangle 410"
|
||||
width="350"
|
||||
height="500"
|
||||
fill="url(#radial-gradient)"
|
||||
/>
|
||||
</g>
|
||||
</template>
|
||||
|
||||
<script>
|
||||
import { defineComponent, onMounted, ref, inject, watch } from "vue";
|
||||
export default defineComponent({
|
||||
name: "voiceBackground",
|
||||
setup() {
|
||||
const audioCtx = new (window.AudioContext || window.webkitAudioContext)();
|
||||
const analyzer = audioCtx.createAnalyser();
|
||||
analyzer.fftSize = 2048;
|
||||
const xInitial = 24;
|
||||
const xFinal = 640;
|
||||
// analyzer.minDecibels = -90;
|
||||
analyzer.smoothingTimeConstant = 0.85;
|
||||
const dataArray = new Uint8Array(analyzer.frequencyBinCount);
|
||||
const sampleRate = 16000;
|
||||
let gradient;
|
||||
const radius = ref(0);
|
||||
const think = ref(false);
|
||||
const aiOffset = ref(0.7);
|
||||
|
||||
const anime = inject("animejs");
|
||||
let thinkAnimation;
|
||||
|
||||
onMounted(() => {
|
||||
gradient = document.getElementById("radial-gradient");
|
||||
|
||||
thinkAnimation = anime({
|
||||
targets: gradient,
|
||||
loop: true,
|
||||
cy: "1.3",
|
||||
direction: "alternate",
|
||||
easing: "easeInOutCirc",
|
||||
});
|
||||
});
|
||||
|
||||
function convertBlock(buffer) {
|
||||
// incoming data is an ArrayBuffer
|
||||
const incomingData = new Uint8Array(buffer); // create a uint8 view on the ArrayBuffer
|
||||
const l = incomingData.length; // length, we need this for the loop
|
||||
const outputData = new Float32Array(incomingData.length); // create the Float32Array for output
|
||||
for (let i = 0; i < l; i++) {
|
||||
outputData[i] = (incomingData[i] - 128) / 128.0; // convert audio to float
|
||||
}
|
||||
return outputData; // return the Float32Array
|
||||
}
|
||||
|
||||
window.ipcRenderer.on("render-audio", (event, payload) => {
|
||||
const floatArray = convertBlock(payload.audio.buffer);
|
||||
const buffer = audioCtx.createBuffer(1, floatArray.length, sampleRate);
|
||||
buffer.copyToChannel(floatArray, 0, 0);
|
||||
const source = audioCtx.createBufferSource();
|
||||
source.buffer = buffer;
|
||||
source.connect(analyzer);
|
||||
source.start();
|
||||
analyzer.getByteFrequencyData(dataArray);
|
||||
const dataView = dataArray.slice(xInitial, xFinal);
|
||||
const sum = dataView.reduce((a, b) => a + b);
|
||||
const average = sum / dataView.length;
|
||||
const scaled = average / 255;
|
||||
const zeroed = scaled - 0.6;
|
||||
if (zeroed > 0) {
|
||||
radius.value = zeroed;
|
||||
}
|
||||
if (payload.speech === think.value) {
|
||||
return;
|
||||
} else {
|
||||
think.value = payload.speech;
|
||||
}
|
||||
});
|
||||
|
||||
// watch(think, async (think, prevThink) => {
|
||||
// if (think) {
|
||||
// anime({
|
||||
// targets: gradient,
|
||||
// cy: "0.7",
|
||||
// });
|
||||
// thinkAnimation.play();
|
||||
// } else {
|
||||
// thinkAnimation.pause();
|
||||
// // thinkAnimation.reset();
|
||||
// anime({
|
||||
// targets: gradient,
|
||||
// cy: "1",
|
||||
// });
|
||||
//
|
||||
// // thinkAnimation.reset();
|
||||
// }
|
||||
// });
|
||||
return {
|
||||
radius,
|
||||
aiOffset,
|
||||
};
|
||||
},
|
||||
});
|
||||
</script>
|
||||
|
|
@ -1,7 +1,6 @@
|
|||
import { createApp } from "vue";
|
||||
import App from "./App.vue";
|
||||
import router from "./router";
|
||||
import store from "./store";
|
||||
|
||||
// animation library
|
||||
import anime from "animejs";
|
||||
|
|
@ -11,7 +10,6 @@ import mitt from "mitt";
|
|||
const emitter = mitt();
|
||||
|
||||
createApp(App)
|
||||
.use(store)
|
||||
.use(router)
|
||||
.provide("animejs", anime)
|
||||
.provide("mitt", emitter)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,10 @@
|
|||
import { createRouter, createWebHistory, createWebHashHistory, RouteRecordRaw } from "vue-router";
|
||||
import {
|
||||
createRouter,
|
||||
createWebHistory,
|
||||
createWebHashHistory,
|
||||
RouteRecordRaw
|
||||
} from "vue-router";
|
||||
|
||||
import HomeIndex from "../views/home/index.vue";
|
||||
|
||||
const routes: Array<RouteRecordRaw> = [
|
||||
|
|
|
|||
Loading…
Reference in a new issue