From 65303df96acc840617b28613f480851ec274411c Mon Sep 17 00:00:00 2001 From: josedario87 Date: Mon, 23 Feb 2026 20:52:11 -0600 Subject: [PATCH] feat: Samsung lock screen face widget, voice assistant services, PiP mode and gitignore installers Add Samsung proprietary Face Widget (lock screen/AOD) with terminal status display. Add voice interaction services (AgentVoiceInteractionService, RecognitionService) for digital assistant registration. Add PiP mode with voice/expand actions. Add session-state proxy, voice transcript routes, window controls component. Ignore installers/ directory. --- .gitignore | 15 + frontend/src/App.vue | 72 +- frontend/src/components/WindowControls.vue | 131 +++ frontend/src/lib/tauri.ts | 5 + frontend/src/pages/TranscriptDebugPage.vue | 795 +++++++++++++----- frontend/src/router/index.ts | 5 + package.json | 5 +- server/routes/index.ts | 13 + server/routes/session-state-proxy.ts | 26 + server/routes/voice-transcript.ts | 105 +++ src-tauri/capabilities/default.json | 8 +- src-tauri/gen/android/app/build.gradle.kts | 39 +- .../android/app/src/main/AndroidManifest.xml | 68 +- .../desktop/AgentRecognitionService.kt | 41 + .../desktop/AgentVoiceInteractionService.kt | 21 + .../AgentVoiceInteractionSessionService.kt | 75 ++ .../desktop/LockScreenWidgetReceiver.kt | 238 ++++++ .../java/com/agentui/desktop/MainActivity.kt | 271 +++++- .../java/com/agentui/desktop/ServerConfig.kt | 132 ++- .../desktop/TerminalListWidgetService.kt | 260 ++++++ .../desktop/TranscriptWidgetProvider.kt | 60 +- .../agentui/desktop/TranscriptWidgetWorker.kt | 124 +-- .../agentui/desktop/VoiceCommandActivity.kt | 157 ++-- .../main/res/drawable/face_widget_bg_aod.xml | 6 + .../main/res/drawable/face_widget_bg_dark.xml | 6 + .../src/main/res/layout/face_widget_aod.xml | 124 +++ .../res/layout/face_widget_lockscreen.xml | 140 +++ .../main/res/layout/widget_terminal_item.xml | 64 ++ .../src/main/res/layout/widget_transcript.xml | 90 +- .../app/src/main/res/raw/facewidgets.json | 7 + .../app/src/main/res/values/strings.xml | 1 + .../src/main/res/xml/recognition_service.xml | 4 + .../main/res/xml/transcript_widget_info.xml | 4 +- .../res/xml/voice_interaction_service.xml | 8 + src-tauri/tauri.conf.json | 4 +- 35 files changed, 2640 insertions(+), 484 deletions(-) create mode 100644 frontend/src/components/WindowControls.vue create mode 100644 server/routes/session-state-proxy.ts create mode 100644 server/routes/voice-transcript.ts create mode 100644 src-tauri/gen/android/app/src/main/java/com/agentui/desktop/AgentRecognitionService.kt create mode 100644 src-tauri/gen/android/app/src/main/java/com/agentui/desktop/AgentVoiceInteractionService.kt create mode 100644 src-tauri/gen/android/app/src/main/java/com/agentui/desktop/AgentVoiceInteractionSessionService.kt create mode 100644 src-tauri/gen/android/app/src/main/java/com/agentui/desktop/LockScreenWidgetReceiver.kt create mode 100644 src-tauri/gen/android/app/src/main/java/com/agentui/desktop/TerminalListWidgetService.kt create mode 100644 src-tauri/gen/android/app/src/main/res/drawable/face_widget_bg_aod.xml create mode 100644 src-tauri/gen/android/app/src/main/res/drawable/face_widget_bg_dark.xml create mode 100644 src-tauri/gen/android/app/src/main/res/layout/face_widget_aod.xml create mode 100644 src-tauri/gen/android/app/src/main/res/layout/face_widget_lockscreen.xml create mode 100644 src-tauri/gen/android/app/src/main/res/layout/widget_terminal_item.xml create mode 100644 src-tauri/gen/android/app/src/main/res/raw/facewidgets.json create mode 100644 src-tauri/gen/android/app/src/main/res/xml/recognition_service.xml create mode 100644 src-tauri/gen/android/app/src/main/res/xml/voice_interaction_service.xml diff --git a/.gitignore b/.gitignore index 12e81aa..7243b0e 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,9 @@ nul # Voice recordings (training data) server/recordings/*.webm +# Installers / APKs +installers/ + # Tauri build artifacts src-tauri/target/ src-tauri/installers/ @@ -31,12 +34,24 @@ src-tauri/gen/android/app/src/main/res/* !src-tauri/gen/android/app/src/main/res/layout/ src-tauri/gen/android/app/src/main/res/layout/* !src-tauri/gen/android/app/src/main/res/layout/widget_transcript.xml +!src-tauri/gen/android/app/src/main/res/layout/widget_terminal_item.xml +!src-tauri/gen/android/app/src/main/res/layout/face_widget_lockscreen.xml +!src-tauri/gen/android/app/src/main/res/layout/face_widget_aod.xml !src-tauri/gen/android/app/src/main/res/xml/ src-tauri/gen/android/app/src/main/res/xml/* !src-tauri/gen/android/app/src/main/res/xml/transcript_widget_info.xml +!src-tauri/gen/android/app/src/main/res/xml/voice_interaction_service.xml +!src-tauri/gen/android/app/src/main/res/xml/recognition_service.xml !src-tauri/gen/android/app/src/main/res/values/ src-tauri/gen/android/app/src/main/res/values/* !src-tauri/gen/android/app/src/main/res/values/strings.xml +!src-tauri/gen/android/app/src/main/res/raw/ +src-tauri/gen/android/app/src/main/res/raw/* +!src-tauri/gen/android/app/src/main/res/raw/facewidgets.json +!src-tauri/gen/android/app/src/main/res/drawable/ +src-tauri/gen/android/app/src/main/res/drawable/* +!src-tauri/gen/android/app/src/main/res/drawable/face_widget_bg_dark.xml +!src-tauri/gen/android/app/src/main/res/drawable/face_widget_bg_aod.xml src-tauri/gen/android/keystore.jks # Old frontend Tauri location diff --git a/frontend/src/App.vue b/frontend/src/App.vue index 153e829..b242339 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -11,6 +11,7 @@ import TerminalFabStack from './components/transcript-debug/TerminalFabStack.vue import PwaInstallBanner from './components/PwaInstallBanner.vue' import HooksApprovalModal from './components/HooksApprovalModal.vue' import ServerConfigDialog from './components/ServerConfigDialog.vue' +import WindowControls from './components/WindowControls.vue' import { useGlobalApproval } from './composables/useGlobalApproval' import { initWebMCP, getWebMCP } from './services/webmcp' import { initTorch, destroyTorch } from './services/torch' @@ -20,7 +21,7 @@ import { setResponseControls } from './services/tools/handlers/responseHandlers' import { useCanvasStore } from './stores/canvas' import { useProjectCanvasStore } from './stores/projectCanvas' import { useSessionState } from './stores/session-state' -import { isTauri } from './lib/tauri' +import { isTauri, isMobileTauri, getTauriNotification } from './lib/tauri' import { useServerConfig } from './stores/server-config' const route = useRoute() @@ -35,7 +36,7 @@ const showVoice = ref(false) const showTranscriptDebug = ref(false) const showDebugConsole = ref(false) const toolbarVisible = ref(true) -const forceWco = ref(false) +const forceWco = ref(isTauri && !isMobileTauri()) const debugLogs = ref>([]) // Intercept console.log for debug panel @@ -222,6 +223,31 @@ function syncThemeColor() { } onMounted(async () => { + // Bridge for Android widget navigation (called from MainActivity via evaluateJavascript) + ;(window as any).__WIDGET_NAVIGATE__ = (route: string) => { + router.push(route) + return true + } + + // Bridge for Android voice assistant — opens FloatingTranscriptDebug on the target terminal + ;(window as any).__VOICE_OPEN_TERMINAL__ = (ephemeralSessionId: string) => { + const entry = sessionState.terminalRegistry.find( + t => t.ephemeralSessionId === ephemeralSessionId + ) + if (entry && transcriptDebugRef.value) { + transcriptDebugRef.value.switchToTerminal(entry.transcriptSessionId) + showTranscriptDebug.value = true + return true + } + // Fallback: open on first terminal + if (sessionState.terminalRegistry.length && transcriptDebugRef.value) { + transcriptDebugRef.value.switchToTerminal(sessionState.terminalRegistry[0].transcriptSessionId) + showTranscriptDebug.value = true + return true + } + return false + } + // Sync Windows titlebar color with CSS variable syncThemeColor() @@ -287,6 +313,34 @@ onMounted(async () => { await torchReady }) +async function sendTestNotification() { + const title = 'Agent UI' + const body = 'Test notification from Agent UI — all platforms!' + + if (isTauri) { + try { + const { isPermissionGranted, requestPermission, sendNotification } = await getTauriNotification() + let granted = await isPermissionGranted() + if (!granted) { + const perm = await requestPermission() + granted = perm === 'granted' + } + if (granted) { + sendNotification({ title, body }) + } + } catch (e) { + console.warn('[Notification] Tauri plugin failed:', e) + } + } else if ('Notification' in window) { + if (Notification.permission === 'granted') { + new Notification(title, { body }) + } else if (Notification.permission !== 'denied') { + const perm = await Notification.requestPermission() + if (perm === 'granted') new Notification(title, { body }) + } + } +} + onUnmounted(() => { document.removeEventListener('mousemove', trackMouse) document.removeEventListener('keydown', handleGlobalKeydown) @@ -367,6 +421,12 @@ if (serverConfig) { {{ totalPending }} + +
@@ -503,9 +564,9 @@ if (serverConfig) { align-items: center; justify-content: space-between; padding: 0.5rem 1rem; - padding-top: calc(0.5rem + env(safe-area-inset-top, 0px)); - padding-left: calc(1rem + env(safe-area-inset-left, 0px)); - padding-right: calc(1rem + env(safe-area-inset-right, 0px)); + padding-top: calc(0.5rem + var(--sat, env(safe-area-inset-top, 0px))); + padding-left: calc(1rem + var(--sal, env(safe-area-inset-left, 0px))); + padding-right: calc(1rem + var(--sar, env(safe-area-inset-right, 0px))); background: var(--bg-primary); border-bottom: 1px solid var(--border-color); flex-shrink: 0; @@ -537,6 +598,7 @@ if (serverConfig) { min-height: 32px; max-height: 32px; padding: 0 0.5rem; + padding-right: 0; border-bottom: none; overflow: visible; } diff --git a/frontend/src/components/WindowControls.vue b/frontend/src/components/WindowControls.vue new file mode 100644 index 0000000..15302cf --- /dev/null +++ b/frontend/src/components/WindowControls.vue @@ -0,0 +1,131 @@ + + + + + diff --git a/frontend/src/lib/tauri.ts b/frontend/src/lib/tauri.ts index ea2fc16..f06312c 100644 --- a/frontend/src/lib/tauri.ts +++ b/frontend/src/lib/tauri.ts @@ -83,3 +83,8 @@ export async function getTauriClipboard() { export async function getTauriDialog() { return import('@tauri-apps/plugin-dialog') } + +export async function getTauriWindow() { + const { getCurrentWindow } = await import('@tauri-apps/api/window') + return getCurrentWindow() +} diff --git a/frontend/src/pages/TranscriptDebugPage.vue b/frontend/src/pages/TranscriptDebugPage.vue index 15ad856..065bcbf 100644 --- a/frontend/src/pages/TranscriptDebugPage.vue +++ b/frontend/src/pages/TranscriptDebugPage.vue @@ -1,30 +1,38 @@