me/mime-chat
Watch
1
0
Fork
You've already forked mime-chat
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

more audio changes

Browse source
This commit is contained in:
Andrew Gundersen 2021-05-29 10:36:43 -05:00
commit ee63bb0ac1
11 changed files with 222 additions and 335 deletions
Download patch file Download diff file Expand all files Collapse all files

37
src/background/audio.ts 14-00-16-259.ts
View file

@ -1,37 +0,0 @@
import { ipcMain } from "electron";
import electronAudio from "@/modules/electron-audio";
export default function initAudioIO () {
console.log('[AUDIO]initializing audio');
// initialize the audio interface
const {
read,
stopRead,
write,
shutdown
} = electronAudio();
// event driven controls
// begin capuring audio from input stream
ipcMain.on("start-recording", () => {
console.log("[AUDIO]Reading...");
read();
});
// write a string of audio to the output stream
ipcMain.on("write-audio", (_e: any, audio: string) => write(audio));
// close the audio streams
ipcMain.on("shutdown-audio", shutdown);
// stop capturing audio and return it to caller
ipcMain.handle("stop-recording", () => {
console.log("[AUDIO]Stopping read...");
const audio = stopRead();
return audio;
});
}

9
src/background/init.ts
View file

@ -4,7 +4,6 @@
import { app, dialog } from "electron";
import { createWindow } from './window';
import { initSession } from './session';
import initAudioIO from './audio';
import { backgroundMitt } from '@/modules/emitter';
let win: boolean;
@ -43,13 +42,10 @@ async function main(dev: boolean) {
console.log("MAIN:Initializing Electron App.")
// Must wait til window is created.
// await createWindow();
// Begin audio stream.
initAudioIO();
await createWindow();
// Instantiate socket session with crimata-platorm.
// initSession(dev);
initSession(dev);
}
@ -64,7 +60,6 @@ export function initApp(dev: boolean): void {
// Must keep to ensure app doesn't quit on close.
app.on("before-quit", async () => {
await stopStream();
});
// Must keep to ensure app doesn't quit on close.

7
src/background/session.ts
View file

@ -14,7 +14,6 @@ import { ipcMain, IpcMainEvent, IpcMainInvokeEvent } from "electron";
import useWebSockets from "@/modules/websockets";
import { play } from "./audio";
import { renderMessage } from "@/modules/message";
import { AuthProtocol, SessionState, Profile } from "@/types";
@ -88,9 +87,9 @@ const onMessage = (data: string) => {
if (win) {
console.log("SESS:Emitting standard message.")
if (message.audio) {
play(message.audio)
}
// if (message.audio) {
// play(message.audio)
// }
ipcEmit("render-message", message)
}

111
src/components/inputitem/controllers/audio-control.ts
View file

@ -1,111 +0,0 @@
import anime from "animejs";
import useMitt from "@/modules/mitt";
import { useIpc } from '@/modules/ipc';
import { onMounted, onUnmounted, ref, Ref } from "vue";
import keyboardNameMap from "../keyBoardMaps/keyboardNameMap";
import { renderMessage, clientMessage } from '@/modules/message';
function showRecIcon () {
anime({
targets: '#recIcon',
opacity: [0, 0.75],
scale: [0.0, 1],
duration: 250,
easing: 'linear',
})
}
function hideRecIcon () {
anime({
targets: '#recIcon',
opacity: [0.75, 0],
scale: [1, 0],
duration: 250,
easing: 'linear',
})
}
export default function useAudioInputController (typing: Ref) {
// For sending messages.
const { post, invoke } = useIpc();
const { emitter } = useMitt();
// Keepp track of when we are recording.
const recording = ref(false);
//---Callbacks-----------------------------------------------
const onKeyDown = (e: KeyboardEvent) => {
const cmd = keyboardNameMap[e.keyCode];
// console.log(cmd)
// Start recording on space bar.
if (cmd == "SPACE" && !recording.value && !typing.value) {
console.log("INPT:Starting record.")
post("start-recording", "");
showRecIcon()
recording.value = true;
}
}
const onKeyUp = async (e: KeyboardEvent) => {
const cmd = keyboardNameMap[e.keyCode];
// Stop recording on space up.
if (cmd == "SPACE" && recording.value) {
// Create a message.
const message = renderMessage(
"",
"",
"",
"sf"
)
// Render it immediately.
emitter.emit("self-message", message);
// Stop recording and get audio from recorder.
console.log("INPT:Stopping record.")
const audio = await invoke("stop-recording", "");
// Send message to the backend for processing.
const clientM = clientMessage("", audio, message.uid);
post('client-message', clientM);
hideRecIcon()
recording.value = false;
}
}
//-----------------------------------------------------------
onMounted(() => {
window.addEventListener("keydown", onKeyDown);
window.addEventListener("keyup", onKeyUp);
})
onUnmounted(() => {
window.removeEventListener("keydown", onKeyDown);
window.removeEventListener("keyup", onKeyUp);
});
return {
recording
}
}

26
src/components/inputitem/helpers/animations.ts Normal file
View file

@ -0,0 +1,26 @@
import anime from "animejs";
export function showRecIcon () {
anime({
targets: '#recIcon',
opacity: [0, 0.75],
scale: [0.0, 1],
duration: 250,
easing: 'linear',
})
}
export function hideRecIcon () {
anime({
targets: '#recIcon',
opacity: [0.75, 0],
scale: [1, 0],
duration: 250,
easing: 'linear',
})
}

113
src/components/inputitem/helpers/audiocontrol.ts Normal file
View file

@ -0,0 +1,113 @@
import useMitt from "@/modules/mitt";
import { useIpc } from '@/modules/ipc';
import { showRecIcon, hideRecIcon } from "./animations";
// import useMediaRecorder from "./mediarecorder";
import { onMounted, onUnmounted, ref, Ref } from "vue";
import keyboardNameMap from "../keyBoardMaps/keyboardNameMap";
import { renderMessage, clientMessage } from '@/modules/message';
import { AudioData } from "@/types";
export default function useAudioInputController (typing: Ref) {
let mediaRecorder: any;
let buffer = '';
// For sending messages.
const { post, invoke } = useIpc();
const { emitter } = useMitt();
const recording = ref(false);
// initiate media recorder
if (navigator.mediaDevices) {
console.log("Initializing media recorder.");
const usrOptions: any = {
audio: true,
video: false
}
navigator.mediaDevices.getUserMedia(usrOptions).then((stream: any) => {
const context = new AudioContext();
const source = context.createMediaStreamSource(stream);
const processor = context.createScriptProcessor(1024, 1, 1);
source.connect(processor);
processor.connect(context.destination);
processor.onaudioprocess = (e: AudioProcessingEvent) => {
if (recording.value) {
const arr = new Uint16Array(e.inputBuffer);
buffer += String.fromCharCode.apply(null, new Float32Array(e.inputBuffer));
} else buffer = '';
};
});
}
//---Callbacks-----------------------------------------------
const onKeyDown = (e: KeyboardEvent) => {
const cmd = keyboardNameMap[e.keyCode];
// Start recording on space bar.
if (cmd == "SPACE" && !typing.value && !recording.value) {
console.log("INPT:Starting record, filling buffer.")
recording.value = true;
showRecIcon();
}
}
const onKeyUp = async (e: KeyboardEvent) => {
const cmd = keyboardNameMap[e.keyCode];
// Stop recording on space up.
if (cmd == "SPACE" && recording.value) {
// get audio from buffer then stop recording
console.log("INPT:Stopping record.")
const audio = buffer;
recording.value = false;
hideRecIcon()
// render a place holder message
const message = renderMessage(
"",
"",
"",
"sf"
);
emitter.emit("self-message", message);
console.log(`Posting audio: ${audio}`);
console.log(typeof audio);
}
}
//-----------------------------------------------------------
onMounted(() => {
window.addEventListener("keydown", onKeyDown);
window.addEventListener("keyup", onKeyUp);
})
onUnmounted(() => {
window.removeEventListener("keydown", onKeyDown);
window.removeEventListener("keyup", onKeyUp);
});
return {
recording
}
}

74
src/components/inputitem/helpers/mediarecorder.ts Normal file
View file

@ -0,0 +1,74 @@
if (navigator.mediaDevices) {
console.log('getUserMedia supported.');
var constraints = { audio: true };
var chunks = [];
navigator.mediaDevices.getUserMedia(constraints)
.then(function(stream) {
var mediaRecorder = new MediaRecorder(stream);
visualize(stream);
record.onclick = function() {
mediaRecorder.start();
console.log(mediaRecorder.state);
console.log("recorder started");
record.style.background = "red";
record.style.color = "black";
}
stop.onclick = function() {
mediaRecorder.stop();
console.log(mediaRecorder.state);
console.log("recorder stopped");
record.style.background = "";
record.style.color = "";
}
mediaRecorder.onstop = function(e) {
console.log("data available after MediaRecorder.stop() called.");
var clipName = prompt('Enter a name for your sound clip');
var clipContainer = document.createElement('article');
var clipLabel = document.createElement('p');
var audio = document.createElement('audio');
var deleteButton = document.createElement('button');
clipContainer.classList.add('clip');
audio.setAttribute('controls', '');
deleteButton.innerHTML = "Delete";
clipLabel.innerHTML = clipName;
clipContainer.appendChild(audio);
clipContainer.appendChild(clipLabel);
clipContainer.appendChild(deleteButton);
soundClips.appendChild(clipContainer);
audio.controls = true;
var blob = new Blob(chunks, { 'type' : 'audio/ogg; codecs=opus' });
chunks = [];
var audioURL = URL.createObjectURL(blob);
audio.src = audioURL;
console.log("recorder stopped");
deleteButton.onclick = function(e) {
evtTgt = e.target;
evtTgt.parentNode.parentNode.removeChild(evtTgt.parentNode);
}
}
mediaRecorder.ondataavailable = function(e) {
chunks.push(e.data);
}
})
.catch(function(err) {
console.log('The following error occurred: ' + err);
})
}

0
src/components/inputitem/controllers/text-control.ts → src/components/inputitem/helpers/textcontrol.ts
View file

8
src/components/inputitem/inputItem.vue
View file

@ -26,12 +26,10 @@
import { defineComponent } from "vue";
import draggify from "@/modules/draggify";
import TextInput from "@/components/textInput.vue";
import TextInput from "./textinput.vue";
import useTextInputController from
"@/components/controllers/text-control";
import useAudioInputController from
"@/components/controllers/audio-control";
import useTextInputController from "./helpers/textcontrol";
import useAudioInputController from "./helpers/audiocontrol";
export default defineComponent({
name: "InputItem",

2
src/components/messenger.vue
View file

@ -18,7 +18,7 @@
<script lang="ts">
import { defineComponent, onMounted, onUnmounted } from "vue";
import Message from "@/components/message.vue";
import InputItem from "@/components/inputItem.vue";
import InputItem from "@/components/inputitem/inputitem.vue";
import Settings from "@/components/settings.vue";
import useMitt from "@/modules/mitt";
import useMessages from "@/modules/messages";

170
src/modules/electron-audio.ts
View file

@ -1,170 +0,0 @@
const portAudio = require('naudiodon');
const stream = require('stream');
const fs = require('fs');
import { AudioData } from "@/types";
// split Buffer into an array of len-sized Buffers
const bufSplit = (buf: Buffer, len: number) => {
const chunks = [];
let i = 0;
let L = len;
while(i < buf.byteLength) {
chunks.push(buf.slice(i, L));
i = L;
L += len;
}
return chunks;
}
export default function electronAudio () {
// current config
let currentHost: number;
let currentInputDevice: number;
let currentOutputDevice: number;
// audio streams
let ai: any;
let ao: any;
// write audio here
let record = false;
let readAudio = '';
console.log(portAudio.getHostAPIs());
// get info for given host api
const getHostInfo = (id: number) => {
return portAudio.getHostAPIs().HostAPIs.filter((host: any) =>
host.id === id)[0];
}
// get info for given device
const getDeviceInfo = (id: number) => {
return portAudio.getDevices().filter((device: any) =>
device.id === id)[0];
};
// constructor to start audio streams
const setup = () => {
console.log("Checking audio devices.");
currentHost = portAudio.getHostAPIs().defaultHostAPI;
const hostInfo = getHostInfo(currentHost);
if (currentInputDevice !== hostInfo.defaultInput) {
currentInputDevice = hostInfo.defaultInput;
ai = setInputStream();
ai.start();
}
if (currentOutputDevice !== hostInfo.defaultOutput) {
currentOutputDevice = hostInfo.defaultOutput;
ao = setOutputStream()
ao.start();
}
};
// init and start input stream
const setInputStream = () => {
const inputDeviceInfo = getDeviceInfo(currentInputDevice);
console.log(inputDeviceInfo);
return new portAudio.AudioIO({
inOptions: {
channelCount: inputDeviceInfo.maxInputChannels,
sampleFormat: portAudio.SampleFormat16Bit,
sampleRate: inputDeviceInfo.defaultSampleRate,
deviceId: inputDeviceInfo.id,
closeOnError: false
}
})
.setEncoding("hex")
.on('error', () => console.log("input stream error"))
.on('data', (chunk: Buffer) => {
console.log("data")
if (record) readAudio += chunk.toString();
});
}
// init and start output stream
const setOutputStream = () => {
const outputDeviceInfo = getDeviceInfo(currentOutputDevice);
console.log(outputDeviceInfo);
return new portAudio.AudioIO({
outOptions: {
channelCount: outputDeviceInfo.maxOutputChannels,
sampleFormat: portAudio.SampleFormat16Bit,
sampleRate: outputDeviceInfo.defaultSampleRate,
deviceId: outputDeviceInfo.id,
closeOnError: false
}
})
.on('error', () => console.log("output stream error"))
}
// pipe ai to writable
const read = () => {
record = true;
};
// remove pipe and return audio contents
const stopRead = (): AudioData => {
const deviceInfo = getDeviceInfo(currentInputDevice);
const audio: AudioData = {
hex: readAudio,
channels: deviceInfo.maxInputChannels,
fs: deviceInfo.defaultSampleRate
}
record = false;
readAudio = '';
return audio;
};
// write a string of audio to stream
const write = (audio: string) => {
// convert audio into chunks of buffers
const chunks = bufSplit(
Buffer.from(audio, 'hex'),
8192 // chunk size
);
// write the chunks
const readable = new stream.Readable()
readable.pipe(ao);
chunks.forEach(chunk => readable.push(chunk))
};
// quit both streams
const shutdown = () => {
if (ao) ao.quit();
if (ai) ai.quit();
};
// run
setInterval(setup, 2000);
return {
read,
stopRead,
write,
shutdown
};
}