me/mime-chat
Watch
1
0
Fork
You've already forked mime-chat
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

prototype browser audio code

Browse source
This commit is contained in:
Andrew Gundersen 2021-06-02 19:14:48 -05:00
commit fd7235ae5b
8 changed files with 268 additions and 224 deletions
Download patch file Download diff file Expand all files Collapse all files

0
src/components/inputitem/helpers/animations.ts → src/components/inputitem/animations.ts
View file

261
src/components/inputitem/audiocontrol.ts Normal file
View file

@ -0,0 +1,261 @@
import useMitt from "@/modules/mitt";
import { useIpc } from '@/modules/ipc';
import { showRecIcon, hideRecIcon } from "./animations";
import { onMounted, onUnmounted, ref, Ref } from "vue";
import keyboardNameMap from "./keyBoardMaps/keyboardNameMap";
import { renderMessage, clientMessage } from '@/modules/message';
const { post, invoke } = useIpc();
function floatTo16bPCM(output: DataView, offset: number, input: Float32Array) {
for (var i = 0; i < input.length; i++, offset += 2) {
var s = Math.max(-1, Math.min(1, input[i]));
output.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7FFF, true);
}
}
function writeFloat32 (output: DataView, offset: number, input: Float32Array) {
for (var i = 0; i < input.length; i++, offset += 4) {
output.setFloat32(offset, input[i], true)
}
}
function writeString(view: DataView, offset: number, string: string) {
for (var i = 0; i < string.length; i++) {
view.setUint8(offset + i, string.charCodeAt(i));
}
}
function encodeWAV (samples: Float32Array, format: number, sampleRate: number, numChannels: number, bitDepth: number) {
var bytesPerSample = bitDepth / 8
var blockAlign = numChannels * bytesPerSample
var buffer = new ArrayBuffer(44 + samples.length * bytesPerSample)
var view = new DataView(buffer)
/* RIFF identifier */
writeString(view, 0, 'RIFF')
/* RIFF chunk length */
view.setUint32(4, 36 + samples.length * bytesPerSample, true)
/* RIFF type */
writeString(view, 8, 'WAVE')
/* format chunk identifier */
writeString(view, 12, 'fmt ')
/* format chunk length */
view.setUint32(16, 16, true)
/* sample format (raw) */
view.setUint16(20, format, true)
/* channel count */
view.setUint16(22, numChannels, true)
/* sample rate */
view.setUint32(24, sampleRate, true)
/* byte rate (sample rate * block align) */
view.setUint32(28, sampleRate * blockAlign, true)
/* block align (channel count * bytes per sample) */
view.setUint16(32, blockAlign, true)
/* bits per sample */
view.setUint16(34, bitDepth, true)
/* data chunk identifier */
writeString(view, 36, 'data')
/* data chunk length */
view.setUint32(40, samples.length * bytesPerSample, true)
/* write data */
if (format === 1) {
floatTo16bPCM(view, 44, samples)
} else {
writeFloat32(view, 44, samples)
}
return buffer
}
function mergeBuffers(bufferArray: Float32Array[], recLength: number) {
// initialize array to hold all samples
var result = new Float32Array(recLength);
// for each array in bufferArray, append values to result
var offset = 0;
for (var i = 0; i < bufferArray.length; i++) {
result.set(bufferArray[i], offset);
offset += bufferArray[i].length;
}
return result;
}
function exportBuffer(recBuffer: Float32Array[], recLength: number, exportSampleRate: number) {
var mergedBuffers = mergeBuffers(recBuffer, recLength);
var encodedWav = encodeWAV(mergedBuffers, 1, exportSampleRate, 1, 16);
var audioBlob = new Blob([encodedWav], {type: 'audio/wav'});
return audioBlob;
}
function postAudioBlob(blob: Blob, uid: string) {
const reader = new FileReader();
reader.readAsDataURL(blob);
reader.onload = (e: ProgressEvent<FileReader>) => {
post("client-message", clientMessage("", reader.result as string, uid))
}
}
export default function useAudioInputController (typing: Ref) {
let audioContext: AudioContext;
let buffer: Float32Array[] = [];
let bufferLen = 0;
let sampleRate = 16000;
let numChannels = 1;
// For sending messages.
const { emitter } = useMitt();
const recording = ref(false);
// initiate media recorder
if (navigator.mediaDevices) {
console.log("Initializing media recorder.");
const usrOptions: any = {
audio: true,
video: false
}
navigator.mediaDevices.getUserMedia(usrOptions).then((stream: any) => {
audioContext = new AudioContext();
const source = audioContext.createMediaStreamSource(stream);
const processor = audioContext.createScriptProcessor(4096, 1, 1);
sampleRate = audioContext.sampleRate;
source.connect(processor);
processor.connect(audioContext.destination);
processor.onaudioprocess = (e: AudioProcessingEvent) => {
if (recording.value) {
const data: Float32Array = e.inputBuffer.getChannelData(0);
buffer.push(data);
bufferLen += data.length;
} else {
buffer = [];
bufferLen = 0;
}
};
});
}
const play = (dataUrl: string) => {
const snd = new Audio(dataUrl);
snd.addEventListener("canplaythrough", (event: any) => {
console.log("Playing");
snd.play();
});
}
// var play = function (blob: Blob) {
// // We'll use a FileReader to create and ArrayBuffer out of the audio response.
// var fileReader = new FileReader();
// fileReader.onload = function() {
// // Once we have an ArrayBuffer we can create our BufferSource and decode the result as an AudioBuffer.
// const playbackSource = audioContext.createBufferSource();
// audioContext.decodeAudioData(fileReader.result as Buffer, function(audioBuffer) {
// console.log(audioBuffer.length);
// console.log(audioBuffer.sampleRate);
// console.log(audioBuffer.numberOfChannels);
// console.log(audioBuffer.duration);
// // Set the source buffer as our new AudioBuffer.
// playbackSource.buffer = audioBuffer;
// // Set the destination (the actual audio-rendering device--your device's speakers).
// playbackSource.connect(audioContext.destination);
// // Add an "on ended" callback.
// playbackSource.onended = function(event) {
// console.log("Playback ended");
// };
// // Start the playback.
// playbackSource.start(0);
// });
// };
// fileReader.readAsArrayBuffer(blob);
// };
//---Callbacks-----------------------------------------------
const onKeyDown = (e: KeyboardEvent) => {
const cmd = keyboardNameMap[e.keyCode];
// Start recording on space bar.
if (cmd == "SPACE" && !typing.value && !recording.value) {
console.log("INPT:Starting record, filling buffer.")
recording.value = true;
showRecIcon();
}
}
const onKeyUp = async (e: KeyboardEvent) => {
const cmd = keyboardNameMap[e.keyCode];
// Stop recording on space up.
if (cmd == "SPACE" && recording.value) {
// get audio from buffer then stop recording
console.log("INPT:Stopping record.")
hideRecIcon()
// render a place holder message
const message = renderMessage(
"",
"",
"",
"sf"
);
emitter.emit("self-message", message);
const blob: Blob = exportBuffer(buffer, bufferLen, sampleRate);
postAudioBlob(blob, message.uid);
recording.value = false;
// const reader = new FileReader();
// reader.readAsDataURL(blob);
// reader.onload = (e: ProgressEvent<FileReader>) => {
// play(reader.result as string);
// }
}
}
//-----------------------------------------------------------
onMounted(() => {
window.addEventListener("keydown", onKeyDown);
window.addEventListener("keyup", onKeyUp);
})
onUnmounted(() => {
window.removeEventListener("keydown", onKeyDown);
window.removeEventListener("keyup", onKeyUp);
});
return {
recording
}
}

138
src/components/inputitem/helpers/audiocontrol.ts
View file

@ -1,138 +0,0 @@
import useMitt from "@/modules/mitt";
import { useIpc } from '@/modules/ipc';
import { showRecIcon, hideRecIcon } from "./animations";
import { onMounted, onUnmounted, ref, Ref } from "vue";
import keyboardNameMap from "../keyBoardMaps/keyboardNameMap";
import { renderMessage, clientMessage } from '@/modules/message';
import { AudioData } from "@/types";
function arrayBufferToBase64 (buffer: any) {
var binary = '';
var bytes = new Uint8Array(buffer);
var len = bytes.byteLength;
for (var i = 0; i < len; i++) {
binary += String.fromCharCode( bytes[ i ] );
}
return window.btoa( binary );
}
export default function useAudioInputController (typing: Ref) {
let mediaRecorder: any;
let buffer = '';
let sampleRate = 44100; // default
let channels = 1;
// For sending messages.
const { post, invoke } = useIpc();
const { emitter } = useMitt();
const recording = ref(false);
// initiate media recorder
if (navigator.mediaDevices) {
console.log("Initializing media recorder.");
const usrOptions: any = {
audio: true,
video: false
}
navigator.mediaDevices.getUserMedia(usrOptions).then((stream: any) => {
const context = new AudioContext();
const source = context.createMediaStreamSource(stream);
const processor = context.createScriptProcessor(1024, 1, 1);
source.connect(processor);
processor.connect(context.destination);
processor.onaudioprocess = (e: AudioProcessingEvent) => {
if (recording.value) {
const data = e.inputBuffer.getChannelData(0).buffer;
buffer += arrayBufferToBase64(data);
sampleRate = e.inputBuffer.sampleRate;
} else buffer = '';
};
});
}
//---Callbacks-----------------------------------------------
const onKeyDown = (e: KeyboardEvent) => {
const cmd = keyboardNameMap[e.keyCode];
// Start recording on space bar.
if (cmd == "SPACE" && !typing.value && !recording.value) {
console.log("INPT:Starting record, filling buffer.")
recording.value = true;
showRecIcon();
}
}
const onKeyUp = async (e: KeyboardEvent) => {
const cmd = keyboardNameMap[e.keyCode];
// Stop recording on space up.
if (cmd == "SPACE" && recording.value) {
// get audio from buffer then stop recording
console.log("INPT:Stopping record.")
recording.value = false;
hideRecIcon()
// render a place holder message
const message = renderMessage(
"",
"",
"",
"sf"
);
emitter.emit("self-message", message);
// send audio to backend
const audio: AudioData = {
content: buffer,
channels: channels,
fs: sampleRate
}
console.log("INPT:Sending audio to backend");
post("client-message", clientMessage("", audio, message.uid));
}
}
//-----------------------------------------------------------
onMounted(() => {
window.addEventListener("keydown", onKeyDown);
window.addEventListener("keyup", onKeyUp);
})
onUnmounted(() => {
window.removeEventListener("keydown", onKeyDown);
window.removeEventListener("keyup", onKeyUp);
});
return {
recording
}
}

74
src/components/inputitem/helpers/mediarecorder.ts
View file

@ -1,74 +0,0 @@
if (navigator.mediaDevices) {
console.log('getUserMedia supported.');
var constraints = { audio: true };
var chunks = [];
navigator.mediaDevices.getUserMedia(constraints)
.then(function(stream) {
var mediaRecorder = new MediaRecorder(stream);
visualize(stream);
record.onclick = function() {
mediaRecorder.start();
console.log(mediaRecorder.state);
console.log("recorder started");
record.style.background = "red";
record.style.color = "black";
}
stop.onclick = function() {
mediaRecorder.stop();
console.log(mediaRecorder.state);
console.log("recorder stopped");
record.style.background = "";
record.style.color = "";
}
mediaRecorder.onstop = function(e) {
console.log("data available after MediaRecorder.stop() called.");
var clipName = prompt('Enter a name for your sound clip');
var clipContainer = document.createElement('article');
var clipLabel = document.createElement('p');
var audio = document.createElement('audio');
var deleteButton = document.createElement('button');
clipContainer.classList.add('clip');
audio.setAttribute('controls', '');
deleteButton.innerHTML = "Delete";
clipLabel.innerHTML = clipName;
clipContainer.appendChild(audio);
clipContainer.appendChild(clipLabel);
clipContainer.appendChild(deleteButton);
soundClips.appendChild(clipContainer);
audio.controls = true;
var blob = new Blob(chunks, { 'type' : 'audio/ogg; codecs=opus' });
chunks = [];
var audioURL = URL.createObjectURL(blob);
audio.src = audioURL;
console.log("recorder stopped");
deleteButton.onclick = function(e) {
evtTgt = e.target;
evtTgt.parentNode.parentNode.removeChild(evtTgt.parentNode);
}
}
mediaRecorder.ondataavailable = function(e) {
chunks.push(e.data);
}
})
.catch(function(err) {
console.log('The following error occurred: ' + err);
})
}

5
src/components/inputitem/inputItem.vue
View file

@ -11,6 +11,7 @@
<!-- Recording animation on space bar -->
<span v-if="recording" class="play"></span>
<span v-if="recording" class="pause"></span>
<span id="audio"></span>
<!-- Show text input on key-down -->
<TextInput />
@ -28,8 +29,8 @@ import draggify from "@/modules/draggify";
import TextInput from "./textinput.vue";
import useTextInputController from "./helpers/textcontrol";
import useAudioInputController from "./helpers/audiocontrol";
import useTextInputController from "./textcontrol";
import useAudioInputController from "./audiocontrol";
export default defineComponent({
name: "InputItem",

2
src/components/inputitem/helpers/textcontrol.ts → src/components/inputitem/textcontrol.ts
View file

@ -2,7 +2,7 @@ import anime from "animejs";
import useMitt from "@/modules/mitt";
import { useIpc } from '@/modules/ipc';
import { Ref, ref, watch, onMounted, onUnmounted } from "vue";
import keyboardNameMap from "../keyBoardMaps/keyboardNameMap";
import keyboardNameMap from "./keyBoardMaps/keyboardNameMap";
import { clientMessage, renderMessage } from '@/modules/message';
//---Animations-----------------------------------------------

5
src/modules/message.ts
View file

@ -3,8 +3,7 @@ import {
ClientMessage,
ClientRequest,
AuthRequest,
LogoutRequest,
AudioData
LogoutRequest
} from "@/types";
import { v4 as uuidv4 } from 'uuid';
@ -31,7 +30,7 @@ export const renderMessage = (text: boolean | string, audio: boolean | string, c
}
)
export const clientMessage = (text: string, audio: AudioData | null, uid: string): ClientMessage => (
export const clientMessage = (text: string, audio: string | null, uid: string): ClientMessage => (
{
audio,
text,

7
src/types.ts
View file

@ -12,15 +12,10 @@ export interface RenderMessage {
newMessage: boolean;
}
export interface AudioData {
content: string;
channels: number;
fs: number;
}
export interface ClientMessage {
text: string;
audio: AudioData | null;
audio: string | null;
uid: string;
}