From d00fb0f3be95d4737871626c5aa41fe9065c6040 Mon Sep 17 00:00:00 2001 From: riqo Date: Wed, 23 Jun 2021 07:52:06 -0500 Subject: [PATCH 01/12] add navbar listener --- src/composables/useIpcMain.ts | 6 ++---- src/ipc/listeners.ts | 22 ++++++++++++++++++++++ src/render/App.vue | 23 +++++++++++++++++++++-- src/render/components/header.vue | 13 +++++++------ src/render/composables/useIpcRend.ts | 6 +++--- src/render/ipc.ts | 25 ++++++++++++++++++++----- src/session.ts | 1 - src/types.ts | 2 +- src/window.ts | 16 ++++++---------- 9 files changed, 82 insertions(+), 32 deletions(-) diff --git a/src/composables/useIpcMain.ts b/src/composables/useIpcMain.ts index ebfc4bc..4c3ba24 100644 --- a/src/composables/useIpcMain.ts +++ b/src/composables/useIpcMain.ts @@ -71,12 +71,10 @@ export class IpcListener implements IIpcListener { ipcMain.removeAllListeners(this.channel); } - private _onPost = (_e: IpcMainEvent, payload?: string | null): void => { + private _onPost = (_e: IpcMainEvent, payload: InputParam): void => { console.log(`[IPC] Post: ${this.channel}`); - - const params = payload ? JSON.parse(payload) : null; - this._listenerCallback(params); + this._listenerCallback(payload); } } diff --git a/src/ipc/listeners.ts b/src/ipc/listeners.ts index 8ef5d45..cb17f65 100644 --- a/src/ipc/listeners.ts +++ b/src/ipc/listeners.ts @@ -3,10 +3,13 @@ import { IpcListener } from "@/composables/useIpcMain" import { sendMessage } from '@/session'; import { collect } from "@/audio"; import { updateAppState } from "@/account"; +import { onNavBar } from "@/window"; const CLIENT_MESSAGE_CHANNEL = "post-session-send" const GET_AUDIO_CHANNEL = "post-audio-collect"; const APP_MOUNT_CHANNEL = "post-app-mount"; +const NAV_BAR_CHANNEL = "post-nav-bar"; +const WINDOW_BLUR_CHANNEL = "post-window-focus"; export const messageListener = new IpcListener({ channel: CLIENT_MESSAGE_CHANNEL, @@ -22,3 +25,22 @@ export const appMountListener = new IpcListener({ channel: APP_MOUNT_CHANNEL, listenerCallback: updateAppState }); + +export const navBarListener = new IpcListener({ + channel: NAV_BAR_CHANNEL, + listenerCallback: onNavBar +}); + + +const onWindowFocus: IpcListenerCallback<{ isFocused: boolean }> = (payload) => { + if (payload.isFocused) { + console.log('window is focused') + } else { + console.log('window is blurred') + } +} + +export const windowFocusListener = new IpcListener({ + channel: WINDOW_BLUR_CHANNEL, + listenerCallback: onWindowFocus +}); diff --git a/src/render/App.vue b/src/render/App.vue index 686591b..7337147 100644 --- a/src/render/App.vue +++ b/src/render/App.vue @@ -21,7 +21,7 @@ + diff --git a/src/render/components/header.vue b/src/render/components/header.vue index ebc47c6..a97f061 100644 --- a/src/render/components/header.vue +++ b/src/render/components/header.vue @@ -9,6 +9,7 @@ class="menuButton minimizeButton" @click.prevent="postNavBarMin" /> + @@ -18,8 +19,11 @@ import { defineComponent } from "vue"; import { postNavBar } from "@/render/ipc"; + import ConnectionStatus from "./connectionStatus.vue"; + export default defineComponent({ name: "Header", + components: { ConnectionStatus }, setup() { const postNavBarExit = () => postNavBar('close'); const postNavBarMin = () => postNavBar('min'); diff --git a/src/render/listeners.ts b/src/render/listeners.ts index 0a05208..3a7b547 100644 --- a/src/render/listeners.ts +++ b/src/render/listeners.ts @@ -12,6 +12,7 @@ const SET_PROFILE_CHANNEL = "set-profile"; const INIT_MESSAGES_CHANNEL = "init-messages"; const ADD_MESSAGE_CHANNEL = "add-message"; const UPDATE_MESSAGE_CHANNEL = "update-message"; +const CONNECTION_STATUS_CHANNEL = "set-connection-status"; export const setProfileListener = new IpcRendererListener({ channel: SET_PROFILE_CHANNEL, @@ -35,7 +36,7 @@ export const updateMessagesListener = new IpcRendererListener({ }); export const connectionStatusListener = new IpcRendererListener({ - channel: UPDATE_MESSAGE_CHANNEL, + channel: CONNECTION_STATUS_CHANNEL, listenerCallback: setConnectionStatus }); diff --git a/src/render/shared/connectionStatus.ts b/src/render/shared/connectionStatus.ts index 8e675dd..7fb15eb 100644 --- a/src/render/shared/connectionStatus.ts +++ b/src/render/shared/connectionStatus.ts @@ -2,9 +2,11 @@ import { ref } from "vue"; export const isAlive = ref(false); + export const loading = ref(true); export const setConnectionStatus: IpcListenerCallback = (payload) => { + console.log('received payload', payload) if (payload) { isAlive.value = payload; } else { diff --git a/src/session.ts b/src/session.ts index 01a105c..912aa68 100644 --- a/src/session.ts +++ b/src/session.ts @@ -60,9 +60,9 @@ const onMessageCallback = (payload: string) => { } -const onConnectionStatusCallback = (alive: boolean) => { - console.log('[Session]: Connection Alive: ', alive); - // ipcEmit('connection-state', alive); +const onConnectionStatusCallback = (isAlive: boolean | Error) => { + console.log('[Session]: Connection Alive: ', isAlive); + ipcEmit('set-connection-status', isAlive); } const { connect, send, close } = useWebsockets( From 4afa716251ad0fe8f59198b34d08f359c7d727fe Mon Sep 17 00:00:00 2001 From: riqo Date: Thu, 24 Jun 2021 07:50:58 -0500 Subject: [PATCH 05/12] show connection status poc --- src/composables/useWebsockets.ts | 25 ++++++++++++++++------ src/main.ts | 2 -- src/render/components/connectionStatus.vue | 23 ++++++++++++++------ src/render/shared/connectionStatus.ts | 16 ++++---------- src/session.ts | 15 +++++++++---- 5 files changed, 51 insertions(+), 30 deletions(-) diff --git a/src/composables/useWebsockets.ts b/src/composables/useWebsockets.ts index d1cfc02..aed517c 100644 --- a/src/composables/useWebsockets.ts +++ b/src/composables/useWebsockets.ts @@ -11,7 +11,13 @@ let _connectionCheckInterval: ReturnType; export default function useWebSockets( messageCallback: (message: string) => void, - connectionStatusCallback: (alive: boolean) => void, + connectionStatusCallback: (status: string) => void, + statusOptions?: { + openMessage: string, + pongMessage: string, + closeMessage: string, + pingErrorMessage: string, + }, ) { let socket: WebSocket; @@ -41,13 +47,15 @@ export default function useWebSockets( socket.send(JSON.stringify({key: secret})); + connectionStatusCallback(statusOptions ? statusOptions.openMessage : "Connection Opened"); + // ping server _connectionCheckInterval = setInterval(() => { socket.ping(null, true, (e: Error) => { if (e) { socket.close(); - connectionStatusCallback(false); + connectionStatusCallback(statusOptions ? statusOptions.pingErrorMessage : "Connection Lost"); setTimeout(() => connect(socketUrl, secret), _reconnectTimeout); } }); @@ -62,15 +70,20 @@ export default function useWebSockets( }); socket.on("close", (event: WebSocket.CloseEvent) => { - connectionStatusCallback(false); + connectionStatusCallback(statusOptions ? statusOptions.closeMessage : "Connection Closed"); clearInterval(_connectionCheckInterval); if (!event.wasClean) { - setTimeout(() => connect(socketUrl, secret), _reconnectTimeout); + setTimeout(() => { + connect(socketUrl, secret), _reconnectTimeout + }); + } }); - socket.on("pong", () => connectionStatusCallback(true)); + socket.on("pong", () => connectionStatusCallback(statusOptions ? statusOptions.pongMessage : "Pong")); + + socket.on('error', () => {}); } @@ -78,7 +91,7 @@ export default function useWebSockets( if (socket) { socket.close(); } - } + }; return { connect, diff --git a/src/main.ts b/src/main.ts index efa8f14..a360835 100644 --- a/src/main.ts +++ b/src/main.ts @@ -24,8 +24,6 @@ export default async function main() { /* launch browser window */ await createWindow(); - console.log('[Session]: Connection Alive: ', "loading"); - try { authState = await accountAuth() as AuthState; } catch(e) { diff --git a/src/render/components/connectionStatus.vue b/src/render/components/connectionStatus.vue index 7dccf1a..c02b43a 100644 --- a/src/render/components/connectionStatus.vue +++ b/src/render/components/connectionStatus.vue @@ -1,26 +1,37 @@ + + diff --git a/src/render/shared/connectionStatus.ts b/src/render/shared/connectionStatus.ts index 7fb15eb..742a6be 100644 --- a/src/render/shared/connectionStatus.ts +++ b/src/render/shared/connectionStatus.ts @@ -1,20 +1,12 @@ // shared import { ref } from "vue"; -export const isAlive = ref(false); - -export const loading = ref(true); - -export const setConnectionStatus: IpcListenerCallback = (payload) => { - console.log('received payload', payload) - if (payload) { - isAlive.value = payload; - } else { - loading.value = true; - } +export const status = ref(""); +export const setConnectionStatus: IpcListenerCallback = (payload) => { + status.value = payload; }; export default { - isAlive + status }; diff --git a/src/session.ts b/src/session.ts index 912aa68..701cb8a 100644 --- a/src/session.ts +++ b/src/session.ts @@ -60,14 +60,21 @@ const onMessageCallback = (payload: string) => { } -const onConnectionStatusCallback = (isAlive: boolean | Error) => { - console.log('[Session]: Connection Alive: ', isAlive); - ipcEmit('set-connection-status', isAlive); +const connectionStatusCallback = (status: string) => { + ipcEmit('set-connection-status', status); +} + +const statusOptions = { + openMessage: "Connected", + pongMessage: "Nominal", + closeMessage: "Reconnecting", + pingErrorMessage: "Connection Lost", } const { connect, send, close } = useWebsockets( onMessageCallback, - onConnectionStatusCallback + connectionStatusCallback, + statusOptions ); /* send a message to the platform */ From 51c4ad8ed7f89f92399537bda0c7d087fc65379a Mon Sep 17 00:00:00 2001 From: Andrew Gundersen Date: Fri, 25 Jun 2021 18:28:29 -0500 Subject: [PATCH 06/12] new message anim --- src/account.ts | 3 +- src/messages.ts | 24 ++ src/render/components/bubble.vue | 395 +++++------------- src/render/components/controllers/helpers.ts | 49 ++- .../controllers/inputItem.control.audio.ts | 2 +- .../controllers/inputItem.control.text.ts | 11 +- src/render/components/message.vue | 142 +++++++ src/render/components/messenger.vue | 11 +- src/render/ipc.ts | 2 +- src/render/shared/messages.ts | 40 +- src/session.ts | 49 +-- src/types.ts | 16 +- 12 files changed, 377 insertions(+), 367 deletions(-) create mode 100644 src/messages.ts create mode 100644 src/render/components/message.vue diff --git a/src/account.ts b/src/account.ts index 581da0e..ac53b23 100644 --- a/src/account.ts +++ b/src/account.ts @@ -4,6 +4,7 @@ import { endSession, launchSession } from "@/session"; import { getToken, setToken, setProfile, getProfile, clearStore } from "./store"; import { parseAuthRes } from "./auth"; import { ipcEmit } from "@/composables/useEmitter"; +import { messages } from "@/messages"; export const accountAuth = async (): Promise => { @@ -89,7 +90,7 @@ export const updateAppState = (): void => { ipcEmit("set-profile", profile); - // ipcEmit('messages') etc + ipcEmit("init-messages", messages) } diff --git a/src/messages.ts b/src/messages.ts new file mode 100644 index 0000000..fdd9d15 --- /dev/null +++ b/src/messages.ts @@ -0,0 +1,24 @@ +import { ipcEmit } from "@/composables/useEmitter"; + +export let messages: Message[] = []; + +export const setMessages = (init: {messages: Message[]}) => { + messages = init.messages; + ipcEmit("init-messages", messages); +} + +export const newMessage = (message: Message) => { + messages.push(message); + ipcEmit("add-message", message); +} + +export const annotateMessage = (annotation: Annotation) => { + for (let i in messages) { + if (messages[i].uid == annotation.uid) { + messages[i].context = annotation.context; + ipcEmit("update-message", annotation) + break; + } + } +} + diff --git a/src/render/components/bubble.vue b/src/render/components/bubble.vue index 982cef3..a3ca4f7 100644 --- a/src/render/components/bubble.vue +++ b/src/render/components/bubble.vue @@ -1,19 +1,6 @@ @@ -24,27 +11,16 @@ export default defineComponent({ name: "Bubble", - props: ["text", "context", "type", "position"], + props: ["modifier", "content", "child"], - setup() { - - const seen = ref(false); - /* initialize seen state */ - if (document.visibilityState === "visible") { - seen.value = true; - } else seen.value = false; + setup(props) { onMounted(() => { - if(seen.value) - document.addEventListener("visibilitychange", () => { - seen.value = true - }); - }); return { - seen + }; } @@ -55,277 +31,92 @@ - - +} + +.ai-bubble { + background-color: #FFFFFF; +} + +.client-bubble { + color: white; + background-color: #58C4FD; +} + +.ai-first-child { + border-bottom-left-radius: 9px; +} + +.ai-middle-child { + border-top-left-radius: 9px; + border-bottom-left-radius: 9px; +} + +.ai-last-child { + border-top-left-radius: 9px; +} + +.client-first-child { + border-bottom-right-radius: 9px; +} + +.client-middle-child { + border-top-right-radius: 9px; + border-bottom-right-radius: 9px; +} + +.client-last-child { + border-top-right-radius: 9px; +} + +.ai-none-child, .client-none-child { + border-top-right-radius: 9px; +} + + +// .notify { +// position: absolute; +// width: 12px; +// height: 12px; +// border-radius: 50%; +// background-color: #58D9FF; +// top: -5px; +// left: -5px; +// border: 2px solid #EBEBEB; +// transform: scale(0); + +// animation-name: notify-anim; +// animation-duration: 5s; +// } + +// @keyframes notify-anim { +// 0%, 90% { +// transform: scale(1); +// } +// 100% { +// transform: scale(0); +// } +// } + + \ No newline at end of file diff --git a/src/render/components/controllers/helpers.ts b/src/render/components/controllers/helpers.ts index 23976a8..f20a11a 100644 --- a/src/render/components/controllers/helpers.ts +++ b/src/render/components/controllers/helpers.ts @@ -1,4 +1,4 @@ -import { ref } from "vue"; +import { ref, Ref } from "vue"; import anime from "animejs"; import { v4 as uuidv4 } from 'uuid'; @@ -86,17 +86,40 @@ export function animateAudioInput () { } +// animate a message annotation. +//! probably could be improved. +export function annotateAnim (uid: string, val: string, contextRef: Ref) { -export function newMessage ({ - text=false, - audio=false, - context=false, - uid=uuidv4() -}) { - return { - text: text, - audio: audio, - context: context, - uid: uid - }; + anime({ + targets: `#${uid} span`, + opacity: [1, 0], + duration: 500, + easing: 'easeOutExpo' + }) + + .finished.then(() => { + contextRef.value = val; + }) + + .then(() => { + anime({ + targets: `#${uid} span`, + opacity: [0, 1], + duration: 500, + easing: 'easeOutExpo' + }) + }) } + +// Given index and length of content, return message child status. +export function calcChild (index: number, len: number) { + if (len === 1) { + return "none"; + } else if (index === 0) { + return "first-child"; + } else if (index === len - 1) { + return "last-child"; + } else { + return "middle-child"; + } +} \ No newline at end of file diff --git a/src/render/components/controllers/inputItem.control.audio.ts b/src/render/components/controllers/inputItem.control.audio.ts index facf30c..9852e53 100644 --- a/src/render/components/controllers/inputItem.control.audio.ts +++ b/src/render/components/controllers/inputItem.control.audio.ts @@ -1,6 +1,6 @@ import { onMounted, onUnmounted, ref, Ref } from "vue"; import { postMessage } from "@/render/ipc"; -import { newMessage, animateAudioInput } from "./helpers"; +import { animateAudioInput } from "./helpers"; import { invokeReturnAudio, postAudioChunk } from "@/render/ipc"; export default function useAudioInputController (typing: Ref) { diff --git a/src/render/components/controllers/inputItem.control.text.ts b/src/render/components/controllers/inputItem.control.text.ts index 941ae89..51edfe0 100644 --- a/src/render/components/controllers/inputItem.control.text.ts +++ b/src/render/components/controllers/inputItem.control.text.ts @@ -1,6 +1,6 @@ import { Ref, ref, watch, onMounted, onUnmounted } from "vue"; import { postMessage } from "@/render/ipc"; -import { newMessage, animateTextInput } from "./helpers"; +import { animateTextInput } from "./helpers"; export default function useTextInputController(elementX: Ref) { @@ -36,11 +36,12 @@ export default function useTextInputController(elementX: Ref) { if (textInput) { // Send it to the backend for processing. - const message = newMessage({ - text: false - }); + const message: Raw = { + text: textInput.value, + audio: false + }; - // postMessage(message); + postMessage(message); clearInput() } diff --git a/src/render/components/message.vue b/src/render/components/message.vue new file mode 100644 index 0000000..d27df6e --- /dev/null +++ b/src/render/components/message.vue @@ -0,0 +1,142 @@ + + + + + + + diff --git a/src/render/components/messenger.vue b/src/render/components/messenger.vue index 8a7f6ad..619eeff 100644 --- a/src/render/components/messenger.vue +++ b/src/render/components/messenger.vue @@ -7,11 +7,12 @@
-
@@ -22,7 +23,7 @@ import { defineComponent, onMounted, onUnmounted } from "vue"; import InputItem from "@/render/components/inputItem.vue"; import Settings from "@/render/components/settings.vue"; -import Bubble from "@/render/components/bubble.vue"; +import Message from "@/render/components/message.vue"; import { profile } from "@/render/shared/profile"; import { messages } from "@/render/shared/messages"; @@ -33,7 +34,7 @@ export default defineComponent({ components: { InputItem, Settings, - Bubble + Message }, setup() { diff --git a/src/render/ipc.ts b/src/render/ipc.ts index 6601dde..cb14778 100644 --- a/src/render/ipc.ts +++ b/src/render/ipc.ts @@ -41,7 +41,7 @@ export const invokeReturnAudio = async (): Promise => ( * */ -export const postMessage = (payload: Message): void => ( +export const postMessage = (payload: Raw): void => ( post('post-session-send', payload) ); diff --git a/src/render/shared/messages.ts b/src/render/shared/messages.ts index f731584..cd8fa26 100644 --- a/src/render/shared/messages.ts +++ b/src/render/shared/messages.ts @@ -5,26 +5,54 @@ import useScroll from "@/render/composables/useScroll"; export const messages: Ref> = ref([]); export const setMessages: IpcListenerCallback> = (payload) => { - messages.value = payload as Array; + // messages.value = payload as Array; } export const addMessage: IpcListenerCallback = (payload) => { messages.value.push(payload as Message); } -export const updateMessage: IpcListenerCallback = (payload) => { - const message = payload as Message; +export const updateMessage: IpcListenerCallback = (payload) => { + const annotation = payload as Annotation; let targetMessage = messages.value.filter((m: Message) => { - return m.uid = message.uid; + return m.uid === annotation.uid; })[0]; if (targetMessage) { - targetMessage = message; + targetMessage.content = annotation.content; + targetMessage.context = annotation.context; } - } +messages.value = [ +{ + content: ["Welcome Andrew"], + context: "Crimata", + modifier: "ai", + uid: "b5dd3d1b-cef0-4057-aa0f-c72e5c4cd67d" +}, +{ + content: ["show contacts"], + context: "show contacts", + modifier: "client", + uid: "ee2819a3-1125-4881-9942-ca5b3d0ad502" +}] + +setTimeout(() => { + + console.log("updating message"); + + updateMessage({ + content: ["show contacts", "msg"], + context: "show contacts", + uid: "ee2819a3-1125-4881-9942-ca5b3d0ad502" + }); + + + +}, 2000); + export default { messages, setMessages, diff --git a/src/session.ts b/src/session.ts index 01a105c..df0c122 100644 --- a/src/session.ts +++ b/src/session.ts @@ -3,60 +3,53 @@ // import useAudio from "@/audio"; -import { ipcEmit } from "@/composables/useEmitter"; +import { setMessages, newMessage, annotateMessage } from "./messages"; import useWebsockets from "./composables/useWebsockets"; import {config} from "@/config"; -/* data structure of messages that's tied to the UI */ -const uiState: any | null = null; - /* start and stop audio functionality */ // const { initAudio, closeAudio } = useAudio(); -const isInitMessage = (message: any): boolean => { - return true; +const isInitMessage = (object: any): object is Message[] => { + return 'messages' in object; +}; + +const isNewMessage = (object: any): object is Message => { + return 'content' in object; +}; + +const isAnnotation = (object: any): object is Annotation => { + return 'messages' in object; }; const deauthenticate = (): void => { console.log('deauthenticating') }; -const isAddMessage = (message: any): boolean => { - return true; -}; - /** * Controls for interfacing with the platform. * Takes an onMessage callback which we define below. */ - const onMessageCallback = (payload: string) => { - const message = JSON.parse(payload); + const message = JSON.parse(payload); - /* if the platform fails to authenticate, we must back down */ if (message === "CLOSE_AUTH_FAIL") { deauthenticate(); - return; } - ipcEmit("add-message", message) - return + else if (isInitMessage(message)) { + setMessages(message); + } - /* on init, platform sends state, used to init canvas */ - // if (isInitMessage(message)) { - // ipcEmit("init-messages", message) - // } + else if (isNewMessage(message)) { + newMessage(message); + } - - // else if (isAddMessage(message)) { - // ipcEmit("add-messages", message) - // } - - // else { - // ipcEmit("update-messages", message) - // } + else if (isAnnotation(message)) { + annotateMessage(message); + } } diff --git a/src/types.ts b/src/types.ts index 2139a86..7b6d964 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,14 +1,20 @@ interface Message { - text: boolean | string; + modifier: string; + content: string[]; context: boolean | string; - audio: boolean | string; - type: 1 | 2 | 3; uid: string; } -interface ViewMessage extends Message { - child: string; +interface Annotation { + content: string[]; + context: string; + uid: string; +} + +interface Raw { + text: string | boolean; + audio: string | boolean; } interface WindowState { From cb40a811faa1390e7ef64ea7e0c3b548434af086 Mon Sep 17 00:00:00 2001 From: Andrew Gundersen Date: Sun, 27 Jun 2021 09:30:31 -0500 Subject: [PATCH 07/12] message anim --- src/render/components/bubble.vue | 4 ++-- src/render/shared/messages.ts | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/render/components/bubble.vue b/src/render/components/bubble.vue index a3ca4f7..7084a75 100644 --- a/src/render/components/bubble.vue +++ b/src/render/components/bubble.vue @@ -41,8 +41,8 @@ margin-left: 15px; margin-right: 15px; margin-bottom: 4px; - animation-name: bubble-init-anim; - animation-duration: 0.5s; + // animation-name: bubble-init-anim; + // animation-duration: 0.5s; } @keyframes bubble-init-anim { diff --git a/src/render/shared/messages.ts b/src/render/shared/messages.ts index cd8fa26..8d87534 100644 --- a/src/render/shared/messages.ts +++ b/src/render/shared/messages.ts @@ -39,19 +39,19 @@ messages.value = [ uid: "ee2819a3-1125-4881-9942-ca5b3d0ad502" }] -setTimeout(() => { +// setTimeout(() => { - console.log("updating message"); +// console.log("updating message"); - updateMessage({ - content: ["show contacts", "msg"], - context: "show contacts", - uid: "ee2819a3-1125-4881-9942-ca5b3d0ad502" - }); +// updateMessage({ +// content: ["show contacts", "msg"], +// context: "show contacts", +// uid: "ee2819a3-1125-4881-9942-ca5b3d0ad502" +// }); -}, 2000); +// }, 2000); export default { messages, From af421e6bfb9cbfdfdf586c5e3c3b0708435447f3 Mon Sep 17 00:00:00 2001 From: Andrew Gundersen Date: Mon, 28 Jun 2021 11:32:45 -0500 Subject: [PATCH 08/12] more updates --- src/messages.ts | 20 ++++++- .../controllers/messenger.control.ts | 38 ------------- src/render/composables/useIpcRend.ts | 2 +- src/render/listeners.ts | 1 - src/render/shared/messages.ts | 55 +++++++++++-------- src/session.ts | 2 +- src/types.ts | 9 ++- 7 files changed, 60 insertions(+), 67 deletions(-) delete mode 100644 src/render/components/controllers/messenger.control.ts diff --git a/src/messages.ts b/src/messages.ts index fdd9d15..fa2ab22 100644 --- a/src/messages.ts +++ b/src/messages.ts @@ -3,21 +3,37 @@ import { ipcEmit } from "@/composables/useEmitter"; export let messages: Message[] = []; export const setMessages = (init: {messages: Message[]}) => { + console.log("setMessages", init); messages = init.messages; ipcEmit("init-messages", messages); } export const newMessage = (message: Message) => { + console.log("newMessage", message); messages.push(message); ipcEmit("add-message", message); } export const annotateMessage = (annotation: Annotation) => { + console.log("annotateMessage", annotation); + for (let i in messages) { + if (messages[i].uid == annotation.uid) { - messages[i].context = annotation.context; - ipcEmit("update-message", annotation) + + if (annotation.data.hasOwnProperty("content")) { + const data = annotation.data as ContentData; + messages[i].content = data.content; + } + + if (annotation.data.hasOwnProperty("context")) { + const data = annotation.data as ContextData; + messages[i].context = data.context; + } + + ipcEmit("update-message", annotation); break; + } } } diff --git a/src/render/components/controllers/messenger.control.ts b/src/render/components/controllers/messenger.control.ts deleted file mode 100644 index b9c3a96..0000000 --- a/src/render/components/controllers/messenger.control.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { ref } from 'vue'; -import useScroll from "@/render/composables/useScroll"; - -const messagesRef = ref(); - -/* seed the canvas with messages */ -const seedCanvas = (messages: Message[]) => { - messagesRef.value = messages; -} - -const addMessage = (message: Message) => { - messagesRef.value.push(message); -} - -const updateMessage = (message: Message) => { - - let target_message = messagesRef.value.filter((m: Message) => { - return m.uid = message.uid; - })[0]; - - if (target_message) { - target_message = message; - } - -} - -export default function useMessages() { - - const { updateScrollRef, adjustScroll } = useScroll("messenger"); - - return { - messagesRef, - seedCanvas, - addMessage, - updateMessage - }; - -} diff --git a/src/render/composables/useIpcRend.ts b/src/render/composables/useIpcRend.ts index bbfb4e1..477289f 100644 --- a/src/render/composables/useIpcRend.ts +++ b/src/render/composables/useIpcRend.ts @@ -25,7 +25,7 @@ export class IpcRendererListener implements IIpcListener } private _onPost = (_e: IpcRendererEvent, payload: InputParam): void => { - console.log(`[IPC] Post: ${this.channel}`); + console.log(`[IPC] Post: ${this.channel}`, payload); this._listenerCallback(payload); } diff --git a/src/render/listeners.ts b/src/render/listeners.ts index 0a05208..62f7cb7 100644 --- a/src/render/listeners.ts +++ b/src/render/listeners.ts @@ -38,4 +38,3 @@ export const connectionStatusListener = new IpcRendererListener({ channel: UPDATE_MESSAGE_CHANNEL, listenerCallback: setConnectionStatus }); - diff --git a/src/render/shared/messages.ts b/src/render/shared/messages.ts index 8d87534..79b8fe7 100644 --- a/src/render/shared/messages.ts +++ b/src/render/shared/messages.ts @@ -5,39 +5,55 @@ import useScroll from "@/render/composables/useScroll"; export const messages: Ref> = ref([]); export const setMessages: IpcListenerCallback> = (payload) => { - // messages.value = payload as Array; + console.log("setMessages", payload); + messages.value = payload as Array; } export const addMessage: IpcListenerCallback = (payload) => { + console.log("addMessage", payload); messages.value.push(payload as Message); } export const updateMessage: IpcListenerCallback = (payload) => { + console.log("updateMessage", payload); const annotation = payload as Annotation; + /* find the target message */ let targetMessage = messages.value.filter((m: Message) => { return m.uid === annotation.uid; })[0]; if (targetMessage) { - targetMessage.content = annotation.content; - targetMessage.context = annotation.context; + + /* update the content (add a new bubble) */ + if (annotation.data.hasOwnProperty("content")) { + const data = annotation.data as ContentData; + targetMessage.content = data.content; + } + + /* and/or update the context */ + if (annotation.data.hasOwnProperty("context")) { + const data = annotation.data as ContextData; + targetMessage.context = data.context; + } + } + } -messages.value = [ -{ - content: ["Welcome Andrew"], - context: "Crimata", - modifier: "ai", - uid: "b5dd3d1b-cef0-4057-aa0f-c72e5c4cd67d" -}, -{ - content: ["show contacts"], - context: "show contacts", - modifier: "client", - uid: "ee2819a3-1125-4881-9942-ca5b3d0ad502" -}] +// messages.value = [ +// { +// content: ["Welcome Andrew"], +// context: "Crimata", +// modifier: "ai", +// uid: "b5dd3d1b-cef0-4057-aa0f-c72e5c4cd67d" +// }, +// { +// content: ["show contacts"], +// context: "false", +// modifier: "client", +// uid: "ee2819a3-1125-4881-9942-ca5b3d0ad502" +// }] // setTimeout(() => { @@ -52,10 +68,3 @@ messages.value = [ // }, 2000); - -export default { - messages, - setMessages, - addMessage, - updateMessage -}; diff --git a/src/session.ts b/src/session.ts index df0c122..c1aec40 100644 --- a/src/session.ts +++ b/src/session.ts @@ -19,7 +19,7 @@ const isNewMessage = (object: any): object is Message => { }; const isAnnotation = (object: any): object is Annotation => { - return 'messages' in object; + return 'uid' in object; }; const deauthenticate = (): void => { diff --git a/src/types.ts b/src/types.ts index 7b6d964..a0cbb70 100644 --- a/src/types.ts +++ b/src/types.ts @@ -6,9 +6,16 @@ interface Message { uid: string; } -interface Annotation { +interface ContentData { content: string[]; +} + +interface ContextData { context: string; +} + +interface Annotation { + data: ContentData | ContextData; uid: string; } From a791f973dcc3ff4c511412c82b460919019be289 Mon Sep 17 00:00:00 2001 From: Andrew Gundersen Date: Tue, 29 Jun 2021 11:30:35 -0500 Subject: [PATCH 09/12] working protocols --- src/account.ts | 12 +---- src/ipc/listeners.ts | 2 +- src/main.ts | 4 +- src/messages.ts | 83 +++++++++++++++++++++++------------ src/render/listeners.ts | 8 +++- src/render/shared/messages.ts | 53 ++++++++++------------ src/session.ts | 51 ++++++++++----------- src/types.ts | 16 +++---- 8 files changed, 117 insertions(+), 112 deletions(-) diff --git a/src/account.ts b/src/account.ts index ac53b23..965502c 100644 --- a/src/account.ts +++ b/src/account.ts @@ -1,10 +1,9 @@ import { postAuth, postLogin, postLogout } from "@/api/account"; import { endSession, launchSession } from "@/session"; -import { getToken, setToken, setProfile, getProfile, clearStore } from "./store"; +import { getToken, setToken, setProfile, clearStore } from "./store"; import { parseAuthRes } from "./auth"; import { ipcEmit } from "@/composables/useEmitter"; -import { messages } from "@/messages"; export const accountAuth = async (): Promise => { @@ -84,13 +83,4 @@ export const accountLogout = async (): Promise => { } -export const updateAppState = (): void => { - - const profile = getProfile(); - - ipcEmit("set-profile", profile); - - ipcEmit("init-messages", messages) - -} diff --git a/src/ipc/listeners.ts b/src/ipc/listeners.ts index cb17f65..17af103 100644 --- a/src/ipc/listeners.ts +++ b/src/ipc/listeners.ts @@ -2,7 +2,7 @@ import { IpcListener } from "@/composables/useIpcMain" import { sendMessage } from '@/session'; import { collect } from "@/audio"; -import { updateAppState } from "@/account"; +import { updateAppState } from "@/session"; import { onNavBar } from "@/window"; const CLIENT_MESSAGE_CHANNEL = "post-session-send" diff --git a/src/main.ts b/src/main.ts index a360835..6df9f6c 100644 --- a/src/main.ts +++ b/src/main.ts @@ -10,8 +10,8 @@ */ import initIpcMain from "@/ipc/index"; -import { accountAuth, updateAppState } from "./account"; -import { launchSession } from "./session"; +import { accountAuth } from "./account"; +import { launchSession, updateAppState } from "./session"; import createWindow from "./window"; let authState: AuthState | null; diff --git a/src/messages.ts b/src/messages.ts index fa2ab22..9a72b38 100644 --- a/src/messages.ts +++ b/src/messages.ts @@ -1,40 +1,67 @@ import { ipcEmit } from "@/composables/useEmitter"; -export let messages: Message[] = []; +export default class Messages { -export const setMessages = (init: {messages: Message[]}) => { - console.log("setMessages", init); - messages = init.messages; - ipcEmit("init-messages", messages); -} + messages: Message[]; -export const newMessage = (message: Message) => { - console.log("newMessage", message); - messages.push(message); - ipcEmit("add-message", message); -} + constructor(messages: Message[]) { + console.log(messages); + this.messages = messages; + this.emit(); + } -export const annotateMessage = (annotation: Annotation) => { - console.log("annotateMessage", annotation); + emit() { + ipcEmit("init-messages", this.messages); + } - for (let i in messages) { + update(update: Update) { - if (messages[i].uid == annotation.uid) { + if (update.name === "add") { + this.messages.push(update.data as Message); + ipcEmit("add-message", update.data); + } + + else if (update.name === "annotate") { + this._annotate(update.data as Annotation); + ipcEmit("update-message", update.data); + } + + else { + this._delete(update.data as string); + ipcEmit("delete-message", update.data); + } + + } + + _annotate(annotation: Annotation) { + + for (let i in this.messages) { + + if (this.messages[i].uid == annotation.uid) { + + if (annotation.name == "content") { + this.messages[i].content = annotation.data as string[]; + } + + if (annotation.name == "context") { + this.messages[i].context = annotation.data as string; + } + + ipcEmit("update-message", annotation); - if (annotation.data.hasOwnProperty("content")) { - const data = annotation.data as ContentData; - messages[i].content = data.content; } - if (annotation.data.hasOwnProperty("context")) { - const data = annotation.data as ContextData; - messages[i].context = data.context; - } - - ipcEmit("update-message", annotation); - break; - } - } -} + } + + _delete(uid: string) { + for (var i = 0; i < this.messages.length; i++) { + if (this.messages[i].uid === uid) { + this.messages.splice(i, 1); + break; + } + } + } + +} \ No newline at end of file diff --git a/src/render/listeners.ts b/src/render/listeners.ts index 90b3cc4..e6ada7f 100644 --- a/src/render/listeners.ts +++ b/src/render/listeners.ts @@ -4,7 +4,7 @@ import { IpcRendererListener } from "./composables/useIpcRend" * Shared state imports * */ import { setProfile } from "./shared/profile"; -import { setMessages, addMessage, updateMessage } from "./shared/messages"; +import { setMessages, addMessage, updateMessage, deleteMessage } from "./shared/messages"; import { setConnectionStatus } from "./shared/connectionStatus"; const SET_PROFILE_CHANNEL = "set-profile"; @@ -12,6 +12,7 @@ const SET_PROFILE_CHANNEL = "set-profile"; const INIT_MESSAGES_CHANNEL = "init-messages"; const ADD_MESSAGE_CHANNEL = "add-message"; const UPDATE_MESSAGE_CHANNEL = "update-message"; +const DELETE_MESSAGE_CHANNEL = "delete-message"; const CONNECTION_STATUS_CHANNEL = "set-connection-status"; export const setProfileListener = new IpcRendererListener({ @@ -35,6 +36,11 @@ export const updateMessagesListener = new IpcRendererListener({ listenerCallback: updateMessage }); +export const deleteMessagesListener = new IpcRendererListener({ + channel: DELETE_MESSAGE_CHANNEL, + listenerCallback: deleteMessage +}); + export const connectionStatusListener = new IpcRendererListener({ channel: CONNECTION_STATUS_CHANNEL, listenerCallback: setConnectionStatus diff --git a/src/render/shared/messages.ts b/src/render/shared/messages.ts index 79b8fe7..b24dca3 100644 --- a/src/render/shared/messages.ts +++ b/src/render/shared/messages.ts @@ -5,55 +5,46 @@ import useScroll from "@/render/composables/useScroll"; export const messages: Ref> = ref([]); export const setMessages: IpcListenerCallback> = (payload) => { - console.log("setMessages", payload); messages.value = payload as Array; } export const addMessage: IpcListenerCallback = (payload) => { - console.log("addMessage", payload); messages.value.push(payload as Message); } export const updateMessage: IpcListenerCallback = (payload) => { - console.log("updateMessage", payload); const annotation = payload as Annotation; - /* find the target message */ - let targetMessage = messages.value.filter((m: Message) => { - return m.uid === annotation.uid; - })[0]; + for (let i in messages.value) { - if (targetMessage) { + if (messages.value[i].uid == annotation.uid) { + + if (annotation.name == "content") { + messages.value[i].content = annotation.data as string[]; + } + + if (annotation.name == "context") { + messages.value[i].context = annotation.data as string; + } - /* update the content (add a new bubble) */ - if (annotation.data.hasOwnProperty("content")) { - const data = annotation.data as ContentData; - targetMessage.content = data.content; } - /* and/or update the context */ - if (annotation.data.hasOwnProperty("context")) { - const data = annotation.data as ContextData; - targetMessage.context = data.context; - } - } } -// messages.value = [ -// { -// content: ["Welcome Andrew"], -// context: "Crimata", -// modifier: "ai", -// uid: "b5dd3d1b-cef0-4057-aa0f-c72e5c4cd67d" -// }, -// { -// content: ["show contacts"], -// context: "false", -// modifier: "client", -// uid: "ee2819a3-1125-4881-9942-ca5b3d0ad502" -// }] +export const deleteMessage: IpcListenerCallback = (payload) => { + const uid = payload as string; + + for (var i = 0; i < messages.value.length; i++) { + if (messages.value[i].uid === uid) { + messages.value.splice(i, 1); + break; + } + } + +} + // setTimeout(() => { diff --git a/src/session.ts b/src/session.ts index 885d940..6b40e43 100644 --- a/src/session.ts +++ b/src/session.ts @@ -3,52 +3,34 @@ // import useAudio from "@/audio"; -import { setMessages, newMessage, annotateMessage } from "./messages"; +import Messages from "./messages"; +import { getProfile } from "./store"; import useWebsockets from "./composables/useWebsockets"; import {config} from "@/config"; +import { ipcEmit } from "@/composables/useEmitter"; /* start and stop audio functionality */ // const { initAudio, closeAudio } = useAudio(); -const isInitMessage = (object: any): object is Message[] => { - return 'messages' in object; -}; +let messages: Messages; -const isNewMessage = (object: any): object is Message => { - return 'content' in object; -}; - -const isAnnotation = (object: any): object is Annotation => { - return 'uid' in object; -}; - -const deauthenticate = (): void => { - console.log('deauthenticating') -}; - -/** - * Controls for interfacing with the platform. - * Takes an onMessage callback which we define below. - */ const onMessageCallback = (payload: string) => { const message = JSON.parse(payload); if (message === "CLOSE_AUTH_FAIL") { - deauthenticate(); + console.log("deauthenticating"); } - else if (isInitMessage(message)) { - setMessages(message); + else if (message.header === "init") { + messages = new Messages(message.body); } - else if (isNewMessage(message)) { - newMessage(message); - } - - else if (isAnnotation(message)) { - annotateMessage(message); + else { + if (messages) { + messages.update(message.body); + } } } @@ -57,6 +39,17 @@ const connectionStatusCallback = (status: string) => { ipcEmit('set-connection-status', status); } + +export const updateAppState = (): void => { + const profile = getProfile(); + + ipcEmit("set-profile", profile); + + if (messages) + messages.emit(); + +} + const statusOptions = { openMessage: "Connected", pongMessage: "Nominal", diff --git a/src/types.ts b/src/types.ts index a0cbb70..bcadf73 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,4 +1,9 @@ +interface Update { + name: string; + data: Message[] | Message | Annotation | string; +} + interface Message { modifier: string; content: string[]; @@ -6,16 +11,9 @@ interface Message { uid: string; } -interface ContentData { - content: string[]; -} - -interface ContextData { - context: string; -} - interface Annotation { - data: ContentData | ContextData; + name: string; + data: string | string[]; uid: string; } From b6b1f3e8948eb6a17160a0670fd5287a2db97ea0 Mon Sep 17 00:00:00 2001 From: Andrew Gundersen Date: Tue, 29 Jun 2021 16:21:18 -0500 Subject: [PATCH 10/12] working new protocols --- src/render/components/bubble.vue | 29 +--- src/render/components/context.vue | 88 ++++++++++++ src/render/components/controllers/helpers.ts | 5 +- src/render/components/inputItem.vue | 5 +- src/render/components/message.vue | 135 ++++++------------- src/render/components/messenger.vue | 2 +- src/render/composables/useScroll.ts | 26 ++-- src/render/shared/messages.ts | 15 ++- 8 files changed, 167 insertions(+), 138 deletions(-) create mode 100644 src/render/components/context.vue diff --git a/src/render/components/bubble.vue b/src/render/components/bubble.vue index 7084a75..fe71f94 100644 --- a/src/render/components/bubble.vue +++ b/src/render/components/bubble.vue @@ -6,22 +6,14 @@ + + \ No newline at end of file diff --git a/src/render/components/controllers/helpers.ts b/src/render/components/controllers/helpers.ts index f20a11a..017cfb3 100644 --- a/src/render/components/controllers/helpers.ts +++ b/src/render/components/controllers/helpers.ts @@ -1,6 +1,5 @@ import { ref, Ref } from "vue"; import anime from "animejs"; -import { v4 as uuidv4 } from 'uuid'; export function animateTextInput () { @@ -93,7 +92,7 @@ export function annotateAnim (uid: string, val: string, contextRef: Ref) { anime({ targets: `#${uid} span`, opacity: [1, 0], - duration: 500, + duration: 250, easing: 'easeOutExpo' }) @@ -105,7 +104,7 @@ export function annotateAnim (uid: string, val: string, contextRef: Ref) { anime({ targets: `#${uid} span`, opacity: [0, 1], - duration: 500, + duration: 250, easing: 'easeOutExpo' }) }) diff --git a/src/render/components/inputItem.vue b/src/render/components/inputItem.vue index edd844f..5deb739 100644 --- a/src/render/components/inputItem.vue +++ b/src/render/components/inputItem.vue @@ -32,7 +32,7 @@ import draggify from "@/render/composables/useDraggify"; import useTextInputController from "@/render/components/controllers/inputItem.control.text"; -import useAudioInputController from +// import useAudioInputController from "@/render/components/controllers/inputItem.control.audio"; export default defineComponent({ @@ -51,7 +51,8 @@ export default defineComponent({ // Controllers for text and audio. const { typing } = useTextInputController(elementX) - const { recording } = useAudioInputController(typing) + // const { recording } = useAudioInputController(typing) + const recording = false; return { elementX, diff --git a/src/render/components/message.vue b/src/render/components/message.vue index d27df6e..f7876f7 100644 --- a/src/render/components/message.vue +++ b/src/render/components/message.vue @@ -8,19 +8,21 @@ :child="calcChild(index, content.length)" /> - -
- {{ contextRef }} - + @@ -74,28 +70,4 @@ border-top-right-radius: 9px; } -// .notify { -// position: absolute; -// width: 12px; -// height: 12px; -// border-radius: 50%; -// background-color: #58D9FF; -// top: -5px; -// left: -5px; -// border: 2px solid #EBEBEB; -// transform: scale(0); - -// animation-name: notify-anim; -// animation-duration: 5s; -// } - -// @keyframes notify-anim { -// 0%, 90% { -// transform: scale(1); -// } -// 100% { -// transform: scale(0); -// } -// } - \ No newline at end of file diff --git a/src/render/shared/messages.ts b/src/render/shared/messages.ts index 71871a3..acf24e4 100644 --- a/src/render/shared/messages.ts +++ b/src/render/shared/messages.ts @@ -56,17 +56,9 @@ export const deleteMessage: IpcListenerCallback = (payload) => { } - -// setTimeout(() => { - -// console.log("updating message"); - -// updateMessage({ -// content: ["show contacts", "msg"], -// context: "show contacts", -// uid: "ee2819a3-1125-4881-9942-ca5b3d0ad502" -// }); - - - -// }, 2000); +export default { + messages, + setMessages, + addMessage, + updateMessage +}; From 5872d245f0cc881363601a3f39ad95055a865f65 Mon Sep 17 00:00:00 2001 From: Enrique Hernandez Date: Wed, 7 Jul 2021 07:10:13 -0500 Subject: [PATCH 12/12] add audio worklet --- audio/audioNode.ts | 0 package.json | 22 +----- public/processor.js | 12 ++++ public/worklet/audioProcessor.ts | 33 +++++++++ src/ipc/handlers.ts | 2 +- src/render/audio/audio.worklet.ts | 33 +++++++++ src/render/audio/audioNode.ts | 9 +++ src/render/audio/setupAudio.ts | 74 +++++++++++++++++++++ src/render/main.ts | 3 + src/render/shims-vue.d.ts | 5 ++ tests/audio.js | 107 +++++------------------------- tests/transcribe.js | 38 +++++++++++ tsconfig.worklet.json | 23 +++++++ vue.config.js | 50 ++++++++++++++ yarn.lock | 53 +++++++++++++-- 15 files changed, 347 insertions(+), 117 deletions(-) create mode 100644 audio/audioNode.ts create mode 100644 public/processor.js create mode 100644 public/worklet/audioProcessor.ts create mode 100644 src/render/audio/audio.worklet.ts create mode 100644 src/render/audio/audioNode.ts create mode 100644 src/render/audio/setupAudio.ts create mode 100644 tests/transcribe.js create mode 100644 tsconfig.worklet.json create mode 100644 vue.config.js diff --git a/audio/audioNode.ts b/audio/audioNode.ts new file mode 100644 index 0000000..e69de29 diff --git a/package.json b/package.json index 24d13b1..081698f 100644 --- a/package.json +++ b/package.json @@ -66,25 +66,9 @@ "spectron": "11.0.0", "typescript": "~3.9.3", "vue-cli-plugin-electron-builder": "~2.0.0-rc.6", - "vue-jest": "^5.0.0-0" - }, - "vue": { - "lintOnSave": false, - "pluginOptions": { - "electronBuilder": { - "mainProcessFile": "./src/init.ts", - "rendererProcessFile": "./src/render/main.ts", - "preload": "./src/render/preload.ts", - "builderOptions": { - "appId": "com.crimata.ElectronUpdaterApp", - "artifactName": "${productName}-${version}.${ext}", - "publish": { - "provider": "generic", - "url": "https://gitlab.com/api/v4/projects/25637892/jobs/artifacts/main/raw/dist_electron?job=build" - } - } - } - } + "vue-jest": "^5.0.0-0", + "worker-loader": "^3.0.8", + "worklet-loader": "^1.0.0" }, "gitHooks": { "pre-commit": "lint-staged" diff --git a/public/processor.js b/public/processor.js new file mode 100644 index 0000000..df735a5 --- /dev/null +++ b/public/processor.js @@ -0,0 +1,12 @@ + +class AudioProcessor extends AudioWorkletProcessor { + process (inputs, outputs, parameters) { + console.log(inputs); + console.log(outputs); + console.log(parameters); + return true + } +} + +registerProcessor('processor', AudioProcessor) + diff --git a/public/worklet/audioProcessor.ts b/public/worklet/audioProcessor.ts new file mode 100644 index 0000000..8c8336f --- /dev/null +++ b/public/worklet/audioProcessor.ts @@ -0,0 +1,33 @@ +interface AudioWorkletProcessor { + readonly port: MessagePort; + process( + inputs: Float32Array[][], + outputs: Float32Array[][], + parameters: Record + ): boolean; +} + +declare let AudioWorkletProcessor: { + prototype: AudioWorkletProcessor; + new (options?: AudioWorkletNodeOptions): AudioWorkletProcessor; +}; + +declare function registerProcessor( + name: string, + processorCtor: (new ( + options?: AudioWorkletNodeOptions + ) => AudioWorkletProcessor) & { + parameterDescriptors?: AudioParamDescriptor[]; + } +): undefined; + +class AudioProcessor extends AudioWorkletProcessor { + process (inputs: Float32Array[][], outputs: Float32Array[][], parameters: Record) { + console.log(inputs); + console.log(outputs); + console.log(parameters); + return true + } +} + +registerProcessor('audio-processor', AudioProcessor) diff --git a/src/ipc/handlers.ts b/src/ipc/handlers.ts index 5529662..7a9ba49 100644 --- a/src/ipc/handlers.ts +++ b/src/ipc/handlers.ts @@ -1,7 +1,7 @@ "use strict"; -import { accountLogin, accountLogout, accountProfile } from "@/account"; +import { accountLogin, accountLogout } from "@/account"; import { IpcHandler } from "@/composables/useIpcMain"; import { flush } from "@/audio"; diff --git a/src/render/audio/audio.worklet.ts b/src/render/audio/audio.worklet.ts new file mode 100644 index 0000000..8c8336f --- /dev/null +++ b/src/render/audio/audio.worklet.ts @@ -0,0 +1,33 @@ +interface AudioWorkletProcessor { + readonly port: MessagePort; + process( + inputs: Float32Array[][], + outputs: Float32Array[][], + parameters: Record + ): boolean; +} + +declare let AudioWorkletProcessor: { + prototype: AudioWorkletProcessor; + new (options?: AudioWorkletNodeOptions): AudioWorkletProcessor; +}; + +declare function registerProcessor( + name: string, + processorCtor: (new ( + options?: AudioWorkletNodeOptions + ) => AudioWorkletProcessor) & { + parameterDescriptors?: AudioParamDescriptor[]; + } +): undefined; + +class AudioProcessor extends AudioWorkletProcessor { + process (inputs: Float32Array[][], outputs: Float32Array[][], parameters: Record) { + console.log(inputs); + console.log(outputs); + console.log(parameters); + return true + } +} + +registerProcessor('audio-processor', AudioProcessor) diff --git a/src/render/audio/audioNode.ts b/src/render/audio/audioNode.ts new file mode 100644 index 0000000..245bfed --- /dev/null +++ b/src/render/audio/audioNode.ts @@ -0,0 +1,9 @@ + +// MyWorkletNode.js +export default class MyWorkletNode extends AudioWorkletNode { + constructor(context) { + super(context, '-processor') + console.log(this.channelCount) + } +} + diff --git a/src/render/audio/setupAudio.ts b/src/render/audio/setupAudio.ts new file mode 100644 index 0000000..f69d4b0 --- /dev/null +++ b/src/render/audio/setupAudio.ts @@ -0,0 +1,74 @@ +import AudioProcessor from "./audio.worklet.ts"; +console.log('YAYAYAYYA', AudioProcessor) +//TODO: need to set export directory for audio worklet +// webpack!!! + + const constraints = { + audio: { + echoCancellation: true, + autoGainControl: true, + noiseSuppression: true, + channelCount: 1 + }, + video: false + } +async function getWebAudioMediaStream() { + if (!window.navigator.mediaDevices) { + throw new Error( + "This browser does not support web audio or it is not enabled." + ); + } + + try { + const result = await window.navigator.mediaDevices.getUserMedia(constraints); + + return result; + } catch (e) { + switch (e.name) { + case "NotAllowedError": + throw new Error( + "A recording device was found but has been disallowed for this application. Enable the device in the browser settings." + ); + + case "NotFoundError": + throw new Error( + "No recording device was found. Please attach a microphone and click Retry." + ); + + default: + throw e; + } + } +} + +export async function setupAudio() { + // Get the browser audio. Awaits user "allowing" it for the current tab. + const mediaStream = await getWebAudioMediaStream(); + + const context = new window.AudioContext(); + const audioSource = context.createMediaStreamSource(mediaStream); + + let node; + + // Add our audio processor worklet to the context. + + try { + await context.audioWorklet.addModule("audio.worklet.js"); + } catch (e) { + throw new Error( + `Failed to load audio analyzer worklet. Further info: ${e.message}` + ); + } + + node = new AudioWorkletNode(context, 'audio-processor') + + // Connect the audio source (microphone output) to our analysis node. + audioSource.connect(node); + + // Connect our analysis node to the output. Required even though we do not + // output any audio. Allows further downstream audio processing or output to + // occur. + node.connect(context.destination); + + return { context, node }; +} diff --git a/src/render/main.ts b/src/render/main.ts index 5375889..9f986ed 100644 --- a/src/render/main.ts +++ b/src/render/main.ts @@ -3,6 +3,7 @@ import App from "./App.vue"; +import { setupAudio } from "./audio/setupAudio"; import mitt from "mitt"; import { createApp } from "vue"; @@ -12,6 +13,8 @@ import { initIpcRendererListeners } from "./ipc" // Handle ipcMain events. initIpcRendererListeners(); +setupAudio(); + // Handle events. const emitter = mitt(); diff --git a/src/render/shims-vue.d.ts b/src/render/shims-vue.d.ts index 35a61a9..35b2544 100644 --- a/src/render/shims-vue.d.ts +++ b/src/render/shims-vue.d.ts @@ -11,3 +11,8 @@ declare module "anime-js" { export = anime; } } + +declare module "*.worklet.ts" { + const exportString: string; + export default exportString; +} diff --git a/tests/audio.js b/tests/audio.js index b8ab22f..ba21748 100644 --- a/tests/audio.js +++ b/tests/audio.js @@ -1,84 +1,38 @@ -const speech = require('@google-cloud/speech'); +// const speech = require('@google-cloud/speech'); const portAudio = require('naudiodon'); // const rs = fs.createReadStream('rawAudio.wav'); -// Creates a client -const client = new speech.SpeechClient(); - -const encoding = 'LINEAR16'; -const sampleRateHertz = 16000; -const languageCode = 'en-US'; - const audioContainer = { input: '', buffers: [] } -const config = { - encoding: encoding, - sampleRateHertz: sampleRateHertz, - languageCode: languageCode, -}; - -/** - * Note that transcription is limited to 60 seconds audio. - * Use a GCS file for audio longer than 1 minute. - */ -async function transcribeSpeech (audio) { - const request = { - config: config, - audio: audio, - }; - - // Detects speech in the audio file. This creates a recognition job that you - // can wait for now, or get its result later. - const [operation] = await client.longRunningRecognize(request); - - // Get a Promise representation of the final result of the job - const [response] = await operation.promise(); - - const transcription = response.results - .map(result => result.alternatives[0].transcript) - .join('\n'); - console.log(`Transcription: ${transcription}`); -} let record = true; // Create an instance of AudioIO with inOptions (defaults are as below), which will return a ReadableStream -const ia = new portAudio.AudioIO({ +const aio = new portAudio.AudioIO({ inOptions: { - channelCount: 1, - sampleFormat: 16, - sampleRate: 16000, - deviceId: -1, - closeOnError: false, + channelCount: 2, + sampleFormat: portAudio.SampleFormat16Bit, + sampleRate: 44100, + deviceId: -1 // Use -1 or omit the deviceId to select the default device }, -}); -ia.setEncoding('base64'); -ia.start(); -ia.on('data', (chunk) => { - if (record) { - console.log('recording data'); - audioContainer.input += chunk; - } else { - if (audioContainer.input.length) audioContainer.input = ""; - } + outOptions: { + channelCount: 2, + sampleFormat: portAudio.SampleFormat16Bit, + sampleRate: 44100, + deviceId: -1 // Use -1 or omit the deviceId to select the default device + } }); -const ao = new portAudio.AudioIO({ - outOptions: { - sampleFormat: 16, - channelCount: 1, - sampleRate: 16000, - deviceId: -1, - closeOnError: false, - } -}); -ao.start(); -let counter = 0; +aio.start() +aio.read() +aio.on('data', buf => console.log(buf.timestamp)); + +const counter = 0; const tests = []; function testCallback() { @@ -143,13 +97,6 @@ function bufSplit(input){ } async function test() { - transcribeSpeech({ - content: Buffer.from(audioContainer.input, 'base64') - }); - tests.push(audioContainer.input) - counter++; - console.log('audio string length:', audioContainer.input.length) - console.log('buffers written: ', audioContainer.buffers.length) } setTimeout(async () => { @@ -157,26 +104,6 @@ setTimeout(async () => { test() }, 4000); -// setTimeout(() => { -// record = true; -// }, 6000) -// -// setTimeout(async () => { -// record = false; -// test(); -// }, 9000); -// -// setTimeout(() => { -// record = true; -// }, 11000) -// -// setTimeout(async () => { -// record = false; -// test(); -// }, 14000); -// setTimeout(async () => { - ia.quit(); - testCallback() return; }, 6000); diff --git a/tests/transcribe.js b/tests/transcribe.js new file mode 100644 index 0000000..11b38d7 --- /dev/null +++ b/tests/transcribe.js @@ -0,0 +1,38 @@ + +const speech = require('@google-cloud/speech'); + +// Creates a client +const client = new speech.SpeechClient(); + +const encoding = 'LINEAR16'; +const sampleRateHertz = 16000; +const languageCode = 'en-US'; + +const config = { + encoding: encoding, + sampleRateHertz: sampleRateHertz, + languageCode: languageCode, +}; + +/** + * Note that transcription is limited to 60 seconds audio. + * Use a GCS file for audio longer than 1 minute. + */ +async function transcribeSpeech (audio) { + const request = { + config: config, + audio: audio, + }; + + // Detects speech in the audio file. This creates a recognition job that you + // can wait for now, or get its result later. + const [operation] = await client.longRunningRecognize(request); + + // Get a Promise representation of the final result of the job + const [response] = await operation.promise(); + + const transcription = response.results + .map(result => result.alternatives[0].transcript) + .join('\n'); + console.log(`Transcription: ${transcription}`); +} diff --git a/tsconfig.worklet.json b/tsconfig.worklet.json new file mode 100644 index 0000000..f304b84 --- /dev/null +++ b/tsconfig.worklet.json @@ -0,0 +1,23 @@ +{ + "compilerOptions": { + "target": "ES2018", + "module": "esnext", + "strict": true, + "importHelpers": true, + "moduleResolution": "node", + "resolveJsonModule": true, + "experimentalDecorators": true, + "skipLibCheck": true, + "esModuleInterop": true, + "allowSyntheticDefaultImports": true, + "sourceMap": true, + "baseUrl": ".", + "types": ["webpack-env"], + "paths": { + "@/*": ["src/*"] + }, + "lib": ["esnext", "scripthost"] + }, + "include": ["src/**/*.worklet.ts"], + "exclude": ["node_modules"] +} diff --git a/vue.config.js b/vue.config.js new file mode 100644 index 0000000..2f1f526 --- /dev/null +++ b/vue.config.js @@ -0,0 +1,50 @@ + +const path = require("path"); + +module.exports = { + assetsDir: "../../static/SPA" +} +module.exports = { + lintOnSave: false, + pluginOptions: { + electronBuilder: { + mainProcessFile: "./src/init.ts", + rendererProcessFile: "./src/render/main.ts", + preload: "./src/render/preload.ts", + + + builderOptions: { + "appId": "com.crimata.ElectronUpdaterApp", + "artifactName": "${productName}-${version}.${ext}", + "publish": { + "provider": "generic", + "url": "https://gitlab.com/api/v4/projects/25637892/jobs/artifacts/main/raw/dist_electron?job=build" + } + }, + + + chainWebpackRendererProcess: (config) => { + // Chain webpack config for electron renderer process only (won't be applied to web builds) + config.outputDir = path.resolve(__dirnamei, "js"); + config.module + .rule('worklet') + .test(/\.worklet\.ts$/) + .use('worklet-loader') + .loader('worklet-loader') + .tap(options => { + options.name = "js/[hash].worklet.js"; + return options + }) + .end() + // Add another loader + .use('ts-loader') + .loader('ts-loader') + .tap(options => { + options.configFile = "tsconfig.worklet.json"; + return options + }) + .end() + }, + } + } +}; diff --git a/yarn.lock b/yarn.lock index 465adc9..15c2b5a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1456,6 +1456,11 @@ resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.6.tgz#f4c7ec43e81b319a9815115031709f26987891f0" integrity sha512-3c+yGKvVP5Y9TYBEibGNR+kLtijnj7mYrXRg+WpFb2X9xm04g/DXYkfg4hmzJQosc9snFNUPkbYIhu+KAm6jJw== +"@types/json-schema@^7.0.6": + version "7.0.7" + resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.7.tgz#98a993516c859eb0d5c4c8f098317a9ea68db9ad" + integrity sha512-cxWFQVseBm6O9Gbw1IWb8r6OS4OhSt3hPZLkFApLjM8TEXROBuQGLAH2i2gZpcXdLBIrpXuTDhH7Vbm1iXmNGA== + "@types/long@^4.0.0", "@types/long@^4.0.1": version "4.0.1" resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.1.tgz#459c65fa1867dafe6a8f322c4c51695663cc55e9" @@ -2367,7 +2372,7 @@ ajv-keywords@^3.1.0, ajv-keywords@^3.4.1, ajv-keywords@^3.5.2: resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-3.5.2.tgz#31f29da5ab6e00d1c2d329acf7b5929614d5014d" integrity sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ== -ajv@^6.1.0, ajv@^6.10.0, ajv@^6.10.2, ajv@^6.12.0, ajv@^6.12.2, ajv@^6.12.3, ajv@^6.12.4: +ajv@^6.1.0, ajv@^6.10.0, ajv@^6.10.2, ajv@^6.12.0, ajv@^6.12.2, ajv@^6.12.3, ajv@^6.12.4, ajv@^6.12.5: version "6.12.6" resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4" integrity sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g== @@ -4898,11 +4903,6 @@ dotenv-expand@^5.1.0: resolved "https://registry.yarnpkg.com/dotenv-expand/-/dotenv-expand-5.1.0.tgz#3fbaf020bfd794884072ea26b1e9791d45a629f0" integrity sha512-YXQl1DSa4/PQyRfgrv6aoNjhasp/p4qs9FjJ4q4cQk+8m4r6k4ZSiEyytKG8f8W9gi8WsQtIObNmKd+tMzNTmA== -dotenv@^10.0.0: - version "10.0.0" - resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-10.0.0.tgz#3d4227b8fb95f81096cdd2b66653fb2c7085ba81" - integrity sha512-rlBi9d8jpv9Sf1klPjNfFAuWDjKLwTIJJ/VxtoTwIR6hnZxcEOQCZg2oIL3MWBYw5GpUDKOEnND7LXTbIpQ03Q== - dotenv@^8.2.0: version "8.2.0" resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-8.2.0.tgz#97e619259ada750eea3e4ea3e26bceea5424b16a" @@ -6649,6 +6649,11 @@ hmac-drbg@^1.0.0: minimalistic-assert "^1.0.0" minimalistic-crypto-utils "^1.0.1" +hoek@^4.2.1: + version "4.2.1" + resolved "https://registry.yarnpkg.com/hoek/-/hoek-4.2.1.tgz#9634502aa12c445dd5a7c5734b572bb8738aacbb" + integrity sha512-QLg82fGkfnJ/4iy1xZ81/9SIJiq1NGFUMGs6ParyjBZr6jW2Ufj/snDqTHixNlHdPNwN2RLVD0Pi3igeK9+JfA== + hoopy@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/hoopy/-/hoopy-0.1.4.tgz#609207d661100033a9a9402ad3dea677381c1b1d" @@ -8447,7 +8452,7 @@ loader-utils@^0.2.16: json5 "^0.5.0" object-assign "^4.0.1" -loader-utils@^1.0.2, loader-utils@^1.1.0, loader-utils@^1.2.3, loader-utils@^1.4.0: +loader-utils@^1.0.0, loader-utils@^1.0.2, loader-utils@^1.1.0, loader-utils@^1.2.3, loader-utils@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-1.4.0.tgz#c579b5e34cb34b1a74edc6c1fb36bfa371d5a613" integrity sha512-qH0WSMBtn/oHuwjy/NucEgbx5dbxxnxup9s4PVXJUDHZBQY+s0NWA9rJf53RBnQZxfch7euUui7hpoAPvALZdA== @@ -11154,6 +11159,14 @@ schema-utils@2.7.0: ajv "^6.12.2" ajv-keywords "^3.4.1" +schema-utils@^0.4.0: + version "0.4.7" + resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-0.4.7.tgz#ba74f597d2be2ea880131746ee17d0a093c68187" + integrity sha512-v/iwU6wvwGK8HbU9yi3/nhGzP0yGSuhQMzL6ySiec1FSrZZDkhm4noOSWzrNFo/jEc+SJY6jRTwuwbSXJPDUnQ== + dependencies: + ajv "^6.1.0" + ajv-keywords "^3.1.0" + schema-utils@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-1.0.0.tgz#0b79a93204d7b600d4b2850d1f66c2a34951c770" @@ -11172,6 +11185,15 @@ schema-utils@^2.0.0, schema-utils@^2.5.0, schema-utils@^2.6.1, schema-utils@^2.6 ajv "^6.12.4" ajv-keywords "^3.5.2" +schema-utils@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-3.0.0.tgz#67502f6aa2b66a2d4032b4279a2944978a0913ef" + integrity sha512-6D82/xSzO094ajanoOSbe4YvXWMfn2A//8Y1+MUqFAJul5Bs+yn36xbK9OtNDcRVSBJ9jjeoXftM6CfztsjOAA== + dependencies: + "@types/json-schema" "^7.0.6" + ajv "^6.12.5" + ajv-keywords "^3.5.2" + scss-tokenizer@^0.2.3: version "0.2.3" resolved "https://registry.yarnpkg.com/scss-tokenizer/-/scss-tokenizer-0.2.3.tgz#8eb06db9a9723333824d3f5530641149847ce5d1" @@ -13300,6 +13322,14 @@ worker-farm@^1.7.0: dependencies: errno "~0.1.7" +worker-loader@^3.0.8: + version "3.0.8" + resolved "https://registry.yarnpkg.com/worker-loader/-/worker-loader-3.0.8.tgz#5fc5cda4a3d3163d9c274a4e3a811ce8b60dbb37" + integrity sha512-XQyQkIFeRVC7f7uRhFdNMe/iJOdO6zxAaR3EWbDp45v3mDhrTi+++oswKNxShUNjPC/1xUp5DB29YKLhFo129g== + dependencies: + loader-utils "^2.0.0" + schema-utils "^3.0.0" + worker-rpc@^0.1.0: version "0.1.1" resolved "https://registry.yarnpkg.com/worker-rpc/-/worker-rpc-0.1.1.tgz#cb565bd6d7071a8f16660686051e969ad32f54d5" @@ -13307,6 +13337,15 @@ worker-rpc@^0.1.0: dependencies: microevent.ts "~0.1.1" +worklet-loader@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/worklet-loader/-/worklet-loader-1.0.0.tgz#17e2eef75981de469c1e1e200ad1ffb54efe7a29" + integrity sha512-4yFqiGDwICoJB4ZbWHzCzyTyDrRnCU1XfvSJtjiBBDreuWDYpA6wf8yqQjNckcjL1jm/sT9ocSvP5tJnmsMOLA== + dependencies: + hoek "^4.2.1" + loader-utils "^1.0.0" + schema-utils "^0.4.0" + wrap-ansi@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-3.0.1.tgz#288a04d87eda5c286e060dfe8f135ce8d007f8ba"