diff --git a/package.json b/package.json index 04f3603..db34fe6 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "electron-is-dev": "^2.0.0", "electron-updater": "^4.3.8", "mitt": "^2.1.0", - "naudiodon": "^2.3.2", + "naudiodon": "^2.3.5", "node-record-lpcm16": "^1.0.1", "update-electron-app": "^2.0.1", "uuid": "^8.3.2", @@ -57,7 +57,7 @@ "eslint": "^6.7.2", "eslint-plugin-vue": "^7.0.0-0", "lint-staged": "^9.5.0", - "node-sass": "^4.12.0", + "node-sass": "4", "optimize-wasm-webpack-plugin": "^1.0.12", "sass-loader": "^8.0.2", "spectron": "11.0.0", diff --git a/src/background/audio.ts b/src/background/audio.ts deleted file mode 100644 index 0a98206..0000000 --- a/src/background/audio.ts +++ /dev/null @@ -1,188 +0,0 @@ -/* eslint @typescript-eslint/no-var-requires: "off" */ - -"use strict"; - -import { ipcMain } from "electron"; -import { backgroundMitt } from '@/modules/emitter'; - -const portAudio = require('naudiodon'); - -// Audio in and out stream objects. -let ai: typeof portAudio.AudioIO | boolean = false; -let ao: typeof portAudio.AudioIO | boolean = false; - -// Whether activly recording. -let record = false; - -const audioContainer = { - input: '', -} - -const audioOptions = { - channelCount: 1, - sampleFormat: 16, - sampleRate: 16000, - deviceId: -1, - closeOnError: false, -} - -// Toggles record to true to begin capturing chunks. -const onRecordingStart = (_event: any, _payload: any) => { - console.log("AUDIO: Beginning audio capture.") - record = true; -} - -// Returns recorded audio to frontend and sets record to false. -const onRecordingEnd = async (_event: any, payload: any) => { - console.log("AUDIO:Sending audio to browser.") - - return new Promise((resolve, reject) => { - - try { - resolve(audioContainer.input); - record = false; - } catch (e) { - reject() - } - - }); -}; - - -// Main audio function run by run.ts module. -export function initAudioIO(): void { - console.log("AUDIO:Starting io streams.") - - if (!ai) { - - // Initialize and start input stream. - ai = new portAudio.AudioIO({ inOptions: audioOptions }); - ai.setEncoding("hex"); - ai.start(); - - // On each data chunk... - ai.on('data', (chunk: string) => { - - // If recording, we capture the data. - if (record) { - console.log('AUDIO:Recording...') - audioContainer.input += chunk; - } - - // Else, we don't capture and also clear audioContainer. - else { - if (audioContainer.input.length) { - audioContainer.input = ""; - } - } - - }); - } - - if (!ao) { - - // Initialize and start input stream. - ao = new portAudio.AudioIO({ outOptions: audioOptions }); - ao.start(); - - } - - // Listen to record. - console.log("AUDIO:Adding recording listeners.") - - ipcMain.removeAllListeners("start-recording"); - ipcMain.on("start-recording", onRecordingStart); - - ipcMain.removeHandler("stop-recording"); - ipcMain.handle("stop-recording", onRecordingEnd); -} - - -// ---Audio playback-------------------------------------------- - -// Split Buffer into an array of len-sized Buffers. -function bufSplit(buf: Buffer, len: number): Array { - const chunks = []; - let i = 0; - let L = len; - - while(i < buf.byteLength) { - chunks.push(buf.slice(i, L)); - i = L; - L += len; - } - - return chunks; -} - -// Audio playback. -export function play(input: string): void { - - // Format the audio. - const audio = bufSplit( - Buffer.from(input as string, 'hex'), - 8192 - ); - - // Called on end of write. - const callback = () => { - - // We stop audio playback anim. - backgroundMitt.emit('ipc-renderer', { - endpoint: 'stop-playback-anim' - }); - - } - - write(); - - // Iterate through audio array and write buffers to portAudio writable. - function write() { - let chunk: Buffer; - let ok = true; - let i = 0; - - do { - chunk = audio[i]; - if (i === audio.length - 1) { - // write last chunk. - ao.write(chunk, null, callback); - } else { - // check for backpreassure. - ok = ao.write(chunk, null); - } - i++; - } while (i < audio.length && ok); - - if (i < audio.length) { - // Had to stop early! - // Write some more once it drains. - ao.once('drain', write); - } - } -} - -// ------------------------------------------------------------- - -// Get's called on window close. -export async function stopStream() { - console.log("AUDIO:Stopping audio stream.") - if (ai) { - try { - await ai.quit() - } catch(e){ - console.log('AUDIO: Failed to shutdown audio input.'); - throw e; - } - } - if (ao) { - try { - await ao.quit() - } catch(e){ - console.log('AUDIO: Failed to shutdown audio output.'); - throw e; - } - } -} - - diff --git a/src/background/init.ts b/src/background/init.ts index f251246..b3eff82 100644 --- a/src/background/init.ts +++ b/src/background/init.ts @@ -4,7 +4,6 @@ import { app, dialog } from "electron"; import { createWindow } from './window'; import { initSession } from './session'; -import { initAudioIO, stopStream } from './audio'; import { backgroundMitt } from '@/modules/emitter'; let win: boolean; @@ -48,9 +47,6 @@ async function main(dev: boolean) { // Instantiate socket session with crimata-platorm. initSession(dev); - // Begin audio stream. - initAudioIO(); - } // Root function of app. @@ -64,7 +60,6 @@ export function initApp(dev: boolean): void { // Must keep to ensure app doesn't quit on close. app.on("before-quit", async () => { - await stopStream(); }); // Must keep to ensure app doesn't quit on close. diff --git a/src/background/session.ts b/src/background/session.ts index 89a8802..82d1f4b 100644 --- a/src/background/session.ts +++ b/src/background/session.ts @@ -14,7 +14,6 @@ import { ipcMain, IpcMainEvent, IpcMainInvokeEvent } from "electron"; import useWebSockets from "@/modules/websockets"; -import { play } from "./audio"; import { renderMessage } from "@/modules/message"; import { AuthProtocol, SessionState, Profile } from "@/types"; @@ -88,9 +87,9 @@ const onMessage = (data: string) => { if (win) { console.log("SESS:Emitting standard message.") - if (message.audio) { - play(message.audio) - } + // if (message.audio) { + // play(message.audio) + // } ipcEmit("render-message", message) } @@ -126,12 +125,12 @@ const { createSocket, sendMessage } = useWebSockets(onMessage, onOpen); // Handle messages from window/client. const onClientMessage = (_event: IpcMainEvent, payload: any) => { - console.log("New client message") + console.log("New client message"); - const success = sendMessage(payload) + const success = sendMessage(payload); if (!success) { - console.log("Unable to send message: ", payload) + console.log("Unable to send message: ", payload); } } @@ -150,14 +149,14 @@ export const initSession = (dev: boolean) => { // Open socket connection. let url = "ws://crimata.com:8760"; if (dev) url = "ws://localhost:8760"; - createSocket(url) + createSocket(url); // Attack browser window init listener. - ipcMain.removeAllListeners("app-mounted") + ipcMain.removeAllListeners("app-mounted"); ipcMain.on("app-mounted", onNewBrowserWindow); // Attach listeners for frontend. - ipcMain.removeAllListeners("client-message") + ipcMain.removeAllListeners("client-message"); ipcMain.on("client-message", onClientMessage); // Keep win up-to-date. diff --git a/src/components/controllers/audioCtrl.ts b/src/components/controllers/audioCtrl.ts deleted file mode 100644 index c9efaa9..0000000 --- a/src/components/controllers/audioCtrl.ts +++ /dev/null @@ -1,111 +0,0 @@ -import anime from "animejs"; -import useMitt from "@/modules/mitt"; -import { useIpc } from '@/modules/ipc'; -import { onMounted, onUnmounted, ref, Ref } from "vue"; -import keyboardNameMap from "../keyBoardMaps/keyboardNameMap"; -import { renderMessage, clientMessage } from '@/modules/message'; - - -function showRecIcon () { - - anime({ - targets: '#recIcon', - opacity: [0, 0.75], - scale: [0.0, 1], - duration: 250, - easing: 'linear', - }) - -} - -function hideRecIcon () { - - anime({ - targets: '#recIcon', - opacity: [0.75, 0], - scale: [1, 0], - duration: 250, - easing: 'linear', - }) - -} - - -export default function useAudioInputController (typing: Ref) { - - // For sending messages. - const { post, invoke } = useIpc(); - const { emitter } = useMitt(); - - // Keepp track of when we are recording. - const recording = ref(false); - - //---Callbacks----------------------------------------------- - - const onKeyDown = (e: KeyboardEvent) => { - const cmd = keyboardNameMap[e.keyCode]; - // console.log(cmd) - - // Start recording on space bar. - if (cmd == "SPACE" && !recording.value && !typing.value) { - - console.log("INPT:Starting record.") - post("start-recording", ""); - - showRecIcon() - recording.value = true; - - } - - } - - const onKeyUp = async (e: KeyboardEvent) => { - const cmd = keyboardNameMap[e.keyCode]; - - // Stop recording on space up. - if (cmd == "SPACE" && recording.value) { - - // Create a message. - const message = renderMessage( - "", - "", - "", - "sf" - ) - - // Render it immediately. - emitter.emit("self-message", message); - - // Stop recording and get audio from recorder. - console.log("INPT:Stopping record.") - const audio = await invoke("stop-recording", ""); - - // Send message to the backend for processing. - const clientM = clientMessage("", audio, message.uid); - post('client-message', clientM); - - hideRecIcon() - recording.value = false; - - } - - } - - //----------------------------------------------------------- - - onMounted(() => { - window.addEventListener("keydown", onKeyDown); - window.addEventListener("keyup", onKeyUp); - }) - - onUnmounted(() => { - window.removeEventListener("keydown", onKeyDown); - window.removeEventListener("keyup", onKeyUp); - }); - - - return { - recording - } - -} diff --git a/src/components/inputitem/animations.ts b/src/components/inputitem/animations.ts new file mode 100644 index 0000000..9683acc --- /dev/null +++ b/src/components/inputitem/animations.ts @@ -0,0 +1,26 @@ +import anime from "animejs"; + + +export function showRecIcon () { + + anime({ + targets: '#recIcon', + opacity: [0, 0.75], + scale: [0.0, 1], + duration: 250, + easing: 'linear', + }) + +} + +export function hideRecIcon () { + + anime({ + targets: '#recIcon', + opacity: [0.75, 0], + scale: [1, 0], + duration: 250, + easing: 'linear', + }) + +} \ No newline at end of file diff --git a/src/components/inputitem/audiocontrol.ts b/src/components/inputitem/audiocontrol.ts new file mode 100644 index 0000000..1b46436 --- /dev/null +++ b/src/components/inputitem/audiocontrol.ts @@ -0,0 +1,261 @@ +import useMitt from "@/modules/mitt"; +import { useIpc } from '@/modules/ipc'; +import { showRecIcon, hideRecIcon } from "./animations"; +import { onMounted, onUnmounted, ref, Ref } from "vue"; +import keyboardNameMap from "./keyBoardMaps/keyboardNameMap"; +import { renderMessage, clientMessage } from '@/modules/message'; + +const { post, invoke } = useIpc(); + + +function floatTo16bPCM(output: DataView, offset: number, input: Float32Array) { + for (var i = 0; i < input.length; i++, offset += 2) { + var s = Math.max(-1, Math.min(1, input[i])); + output.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7FFF, true); + } +} + +function writeFloat32 (output: DataView, offset: number, input: Float32Array) { + for (var i = 0; i < input.length; i++, offset += 4) { + output.setFloat32(offset, input[i], true) + } +} + +function writeString(view: DataView, offset: number, string: string) { + for (var i = 0; i < string.length; i++) { + view.setUint8(offset + i, string.charCodeAt(i)); + } +} + +function encodeWAV (samples: Float32Array, format: number, sampleRate: number, numChannels: number, bitDepth: number) { + + var bytesPerSample = bitDepth / 8 + var blockAlign = numChannels * bytesPerSample + + var buffer = new ArrayBuffer(44 + samples.length * bytesPerSample) + var view = new DataView(buffer) + + /* RIFF identifier */ + writeString(view, 0, 'RIFF') + /* RIFF chunk length */ + view.setUint32(4, 36 + samples.length * bytesPerSample, true) + /* RIFF type */ + writeString(view, 8, 'WAVE') + /* format chunk identifier */ + writeString(view, 12, 'fmt ') + /* format chunk length */ + view.setUint32(16, 16, true) + /* sample format (raw) */ + view.setUint16(20, format, true) + /* channel count */ + view.setUint16(22, numChannels, true) + /* sample rate */ + view.setUint32(24, sampleRate, true) + /* byte rate (sample rate * block align) */ + view.setUint32(28, sampleRate * blockAlign, true) + /* block align (channel count * bytes per sample) */ + view.setUint16(32, blockAlign, true) + /* bits per sample */ + view.setUint16(34, bitDepth, true) + /* data chunk identifier */ + writeString(view, 36, 'data') + /* data chunk length */ + view.setUint32(40, samples.length * bytesPerSample, true) + /* write data */ + if (format === 1) { + floatTo16bPCM(view, 44, samples) + } else { + writeFloat32(view, 44, samples) + } + + return buffer +} + +function mergeBuffers(bufferArray: Float32Array[], recLength: number) { + + // initialize array to hold all samples + var result = new Float32Array(recLength); + + // for each array in bufferArray, append values to result + var offset = 0; + + for (var i = 0; i < bufferArray.length; i++) { + result.set(bufferArray[i], offset); + offset += bufferArray[i].length; + } + + return result; +} + + +function exportBuffer(recBuffer: Float32Array[], recLength: number, exportSampleRate: number) { + var mergedBuffers = mergeBuffers(recBuffer, recLength); + var encodedWav = encodeWAV(mergedBuffers, 1, exportSampleRate, 1, 16); + var audioBlob = new Blob([encodedWav], {type: 'audio/wav'}); + return audioBlob; +} + +function postAudioBlob(blob: Blob, uid: string) { + const reader = new FileReader(); + reader.readAsDataURL(blob); + reader.onload = (e: ProgressEvent) => { + post("client-message", clientMessage("", reader.result as string, uid)) + } +} + +export default function useAudioInputController (typing: Ref) { + + let audioContext: AudioContext; + + let buffer: Float32Array[] = []; + let bufferLen = 0; + let sampleRate = 16000; + let numChannels = 1; + + // For sending messages. + const { emitter } = useMitt(); + + const recording = ref(false); + + // initiate media recorder + if (navigator.mediaDevices) { + console.log("Initializing media recorder."); + + const usrOptions: any = { + audio: true, + video: false + } + + navigator.mediaDevices.getUserMedia(usrOptions).then((stream: any) => { + + audioContext = new AudioContext(); + const source = audioContext.createMediaStreamSource(stream); + const processor = audioContext.createScriptProcessor(4096, 1, 1); + + sampleRate = audioContext.sampleRate; + + source.connect(processor); + processor.connect(audioContext.destination); + + processor.onaudioprocess = (e: AudioProcessingEvent) => { + + if (recording.value) { + + const data: Float32Array = e.inputBuffer.getChannelData(0); + + buffer.push(data); + bufferLen += data.length; + + } else { + buffer = []; + bufferLen = 0; + } + + }; + + }); + + } + + const play = (dataUrl: string) => { + const snd = new Audio(dataUrl); + snd.addEventListener("canplaythrough", (event: any) => { + console.log("Playing"); + snd.play(); + }); + } + + // var play = function (blob: Blob) { + // // We'll use a FileReader to create and ArrayBuffer out of the audio response. + // var fileReader = new FileReader(); + // fileReader.onload = function() { + // // Once we have an ArrayBuffer we can create our BufferSource and decode the result as an AudioBuffer. + // const playbackSource = audioContext.createBufferSource(); + // audioContext.decodeAudioData(fileReader.result as Buffer, function(audioBuffer) { + // console.log(audioBuffer.length); + // console.log(audioBuffer.sampleRate); + // console.log(audioBuffer.numberOfChannels); + // console.log(audioBuffer.duration); + + // // Set the source buffer as our new AudioBuffer. + // playbackSource.buffer = audioBuffer; + // // Set the destination (the actual audio-rendering device--your device's speakers). + // playbackSource.connect(audioContext.destination); + // // Add an "on ended" callback. + // playbackSource.onended = function(event) { + // console.log("Playback ended"); + // }; + // // Start the playback. + // playbackSource.start(0); + // }); + // }; + // fileReader.readAsArrayBuffer(blob); + // }; + + //---Callbacks----------------------------------------------- + + const onKeyDown = (e: KeyboardEvent) => { + const cmd = keyboardNameMap[e.keyCode]; + + // Start recording on space bar. + if (cmd == "SPACE" && !typing.value && !recording.value) { + + console.log("INPT:Starting record, filling buffer.") + recording.value = true; + showRecIcon(); + + } + + } + + const onKeyUp = async (e: KeyboardEvent) => { + const cmd = keyboardNameMap[e.keyCode]; + + // Stop recording on space up. + if (cmd == "SPACE" && recording.value) { + + // get audio from buffer then stop recording + console.log("INPT:Stopping record.") + hideRecIcon() + + // render a place holder message + const message = renderMessage( + "", + "", + "", + "sf" + ); + + emitter.emit("self-message", message); + + const blob: Blob = exportBuffer(buffer, bufferLen, sampleRate); + postAudioBlob(blob, message.uid); + recording.value = false; + + // const reader = new FileReader(); + // reader.readAsDataURL(blob); + // reader.onload = (e: ProgressEvent) => { + // play(reader.result as string); + // } + + } + + } + + //----------------------------------------------------------- + + onMounted(() => { + window.addEventListener("keydown", onKeyDown); + window.addEventListener("keyup", onKeyUp); + }) + + onUnmounted(() => { + window.removeEventListener("keydown", onKeyDown); + window.removeEventListener("keyup", onKeyUp); + }); + + return { + recording + } + +} diff --git a/src/components/inputItem.vue b/src/components/inputitem/inputItem.vue similarity index 94% rename from src/components/inputItem.vue rename to src/components/inputitem/inputItem.vue index 861d7c1..9c05a0d 100644 --- a/src/components/inputItem.vue +++ b/src/components/inputitem/inputItem.vue @@ -11,6 +11,7 @@ + @@ -26,12 +27,10 @@ import { defineComponent } from "vue"; import draggify from "@/modules/draggify"; -import TextInput from "@/components/textInput.vue"; +import TextInput from "./textinput.vue"; -import useTextInputController from - "@/components/controllers/textCtrl"; -import useAudioInputController from - "@/components/controllers/audioCtrl"; +import useTextInputController from "./textcontrol"; +import useAudioInputController from "./audiocontrol"; export default defineComponent({ name: "InputItem", diff --git a/src/components/keyBoardMaps/keyboardCharMap.ts b/src/components/inputitem/keyBoardMaps/keyboardCharMap.ts similarity index 100% rename from src/components/keyBoardMaps/keyboardCharMap.ts rename to src/components/inputitem/keyBoardMaps/keyboardCharMap.ts diff --git a/src/components/keyBoardMaps/keyboardNameMap.ts b/src/components/inputitem/keyBoardMaps/keyboardNameMap.ts similarity index 100% rename from src/components/keyBoardMaps/keyboardNameMap.ts rename to src/components/inputitem/keyBoardMaps/keyboardNameMap.ts diff --git a/src/components/textInput.vue b/src/components/inputitem/textInput.vue similarity index 100% rename from src/components/textInput.vue rename to src/components/inputitem/textInput.vue diff --git a/src/components/controllers/textCtrl.ts b/src/components/inputitem/textcontrol.ts similarity index 96% rename from src/components/controllers/textCtrl.ts rename to src/components/inputitem/textcontrol.ts index 7a2e2e8..19a9414 100644 --- a/src/components/controllers/textCtrl.ts +++ b/src/components/inputitem/textcontrol.ts @@ -2,7 +2,7 @@ import anime from "animejs"; import useMitt from "@/modules/mitt"; import { useIpc } from '@/modules/ipc'; import { Ref, ref, watch, onMounted, onUnmounted } from "vue"; -import keyboardNameMap from "../keyBoardMaps/keyboardNameMap"; +import keyboardNameMap from "./keyBoardMaps/keyboardNameMap"; import { clientMessage, renderMessage } from '@/modules/message'; //---Animations----------------------------------------------- @@ -96,7 +96,7 @@ export default function useTextInputController(elementX: Ref) { emitter.emit("self-message", message); // Send it to the backend for processing. - const clientM = clientMessage(textInput.value, false, message.uid); + const clientM = clientMessage(textInput.value, null, message.uid); post('client-message', clientM); clearInput() diff --git a/src/components/message.vue b/src/components/message.vue index e69f29c..c701155 100644 --- a/src/components/message.vue +++ b/src/components/message.vue @@ -47,7 +47,7 @@
? diff --git a/src/components/messenger.vue b/src/components/messenger.vue index edc50a1..3719730 100644 --- a/src/components/messenger.vue +++ b/src/components/messenger.vue @@ -18,7 +18,7 @@