feat: Samsung lock screen face widget, voice assistant services, PiP mode and gitignore installers

Add Samsung proprietary Face Widget (lock screen/AOD) with terminal status display.
Add voice interaction services (AgentVoiceInteractionService, RecognitionService) for
digital assistant registration. Add PiP mode with voice/expand actions. Add session-state
proxy, voice transcript routes, window controls component. Ignore installers/ directory.
This commit is contained in:
2026-02-23 20:52:11 -06:00
parent e1aa8b1bdb
commit 65303df96a
35 changed files with 2640 additions and 484 deletions

View File

@@ -45,8 +45,9 @@ android {
}
}
getByName("release") {
manifestPlaceholders["usesCleartextTraffic"] = "true"
signingConfig = signingConfigs.getByName("release")
isMinifyEnabled = true
isMinifyEnabled = false
proguardFiles(
*fileTree(".") { include("**/*.pro") }
.plus(getDefaultProguardFile("proguard-android-optimize.txt"))
@@ -80,7 +81,8 @@ dependencies {
apply(from = "tauri.build.gradle.kts")
// Copy APK to src-tauri/installers after build
// Copy APK after build: local installers/ + network backup
// Only the universal signed variant gets copied as agent-ui.apk
android.applicationVariants.all {
val variant = this
variant.outputs.all {
@@ -88,9 +90,36 @@ android.applicationVariants.all {
variant.assembleProvider.get().doLast {
val src = output.outputFile
if (src.exists()) {
val dest = file("../../../../installers/AgentUI-${variant.versionName}-${variant.name}.apk")
src.copyTo(dest, overwrite = true)
println(">> Copied APK to ${dest.absolutePath}")
val localDir = file("../../../../installers")
localDir.mkdirs()
// Always save versioned copy per variant
val localVersioned = File(localDir, "AgentUI-${variant.versionName}-${variant.name}.apk")
src.copyTo(localVersioned, overwrite = true)
println(">> Copied APK to ${localVersioned.absolutePath}")
// agent-ui.apk = only the universal signed build
val isUniversal = variant.name.contains("universal", ignoreCase = true)
val isSigned = variant.signingConfig != null
if (isUniversal && isSigned) {
val localFixed = File(localDir, "agent-ui.apk")
src.copyTo(localFixed, overwrite = true)
println(">> Copied universal signed APK to ${localFixed.absolutePath}")
// Network backup
val networkDir = File("\\\\Memoria-1\\ActiveBackupforBusiness\\Nucleo v3\\agent-ui-apk")
try {
if (networkDir.exists() || networkDir.mkdirs()) {
val networkFile = File(networkDir, "agent-ui.apk")
src.copyTo(networkFile, overwrite = true)
println(">> Copied APK to network: ${networkFile.absolutePath}")
} else {
println(">> WARNING: Network path not available: $networkDir")
}
} catch (e: Exception) {
println(">> WARNING: Failed to copy to network: ${e.message}")
}
}
}
}
}

View File

@@ -2,6 +2,7 @@
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
<uses-permission android:name="android.permission.INTERNET" />
<uses-permission android:name="android.permission.RECORD_AUDIO" />
<uses-permission android:name="com.samsung.systemui.permission.FACE_WIDGET" />
<!-- AndroidTV support -->
<uses-feature android:name="android.software.leanback" android:required="false" />
@@ -16,7 +17,8 @@
android:launchMode="singleTask"
android:label="@string/main_activity_title"
android:name=".MainActivity"
android:exported="true">
android:exported="true"
android:supportsPictureInPicture="true">
<intent-filter>
<action android:name="android.intent.action.MAIN" />
<category android:name="android.intent.category.LAUNCHER" />
@@ -37,7 +39,7 @@
android:resource="@xml/transcript_widget_info" />
</receiver>
<!-- Voice Command / Share Activity -->
<!-- Voice Command / Share / Assist Activity -->
<activity
android:name=".VoiceCommandActivity"
android:label="Agent UI Voice"
@@ -48,8 +50,70 @@
<category android:name="android.intent.category.DEFAULT" />
<data android:mimeType="text/plain" />
</intent-filter>
<intent-filter>
<action android:name="android.intent.action.ASSIST" />
<category android:name="android.intent.category.DEFAULT" />
</intent-filter>
<intent-filter>
<action android:name="android.intent.action.VOICE_COMMAND" />
<category android:name="android.intent.category.DEFAULT" />
</intent-filter>
</activity>
<!-- Digital Assistant Service (enables Samsung side-key, long-press home, etc.) -->
<service
android:name=".AgentVoiceInteractionService"
android:label="@string/app_name"
android:permission="android.permission.BIND_VOICE_INTERACTION"
android:exported="true">
<meta-data
android:name="android.voice_interaction"
android:resource="@xml/voice_interaction_service" />
<intent-filter>
<action android:name="android.service.voice.VoiceInteractionService" />
</intent-filter>
</service>
<service
android:name=".AgentVoiceInteractionSessionService"
android:permission="android.permission.BIND_VOICE_INTERACTION"
android:exported="false" />
<!-- RecognitionService: required by Android to fully register as digital assistant -->
<service
android:name=".AgentRecognitionService"
android:label="@string/app_name"
android:permission="android.permission.BIND_RECOGNITION_SERVICE"
android:exported="true">
<intent-filter>
<action android:name="android.speech.RecognitionService" />
<category android:name="android.intent.category.DEFAULT" />
</intent-filter>
<meta-data
android:name="android.speech"
android:resource="@xml/recognition_service" />
</service>
<!-- Widget ListView adapter service -->
<service
android:name=".TerminalListWidgetService"
android:permission="android.permission.BIND_REMOTEVIEWS"
android:exported="false" />
<!-- Samsung Lock Screen Face Widget -->
<receiver
android:name=".LockScreenWidgetReceiver"
android:permission="com.samsung.systemui.permission.FACE_WIDGET"
android:exported="true">
<intent-filter>
<action android:name="com.samsung.android.intent.action.REQUEST_SERVICEBOX_REMOTEVIEWS" />
</intent-filter>
</receiver>
<meta-data
android:name="com.samsung.systemui.facewidget.executable"
android:resource="@raw/facewidgets" />
<provider
android:name="androidx.core.content.FileProvider"
android:authorities="${applicationId}.fileprovider"

View File

@@ -0,0 +1,41 @@
package com.agentui.desktop
import android.content.Intent
import android.os.Bundle
import android.speech.RecognitionService
import android.speech.SpeechRecognizer
import android.util.Log
/**
* Stub RecognitionService required by Android to register the app
* as a complete voice interaction service / digital assistant.
*
* Without this, the system won't fully recognize the app as an assistant
* and settings will show "None" even when selected.
*
* We delegate actual speech recognition to the system's default
* RecognizerIntent in VoiceCommandActivity.
*/
class AgentRecognitionService : RecognitionService() {
companion object {
private const val TAG = "AgentUI.Recognition"
}
override fun onStartListening(intent: Intent?, callback: Callback?) {
Log.d(TAG, "onStartListening called")
// We don't do real recognition here - VoiceCommandActivity uses
// the system RecognizerIntent directly. This stub satisfies the
// Android requirement that a VoiceInteractionService must have
// an associated RecognitionService.
callback?.error(SpeechRecognizer.ERROR_CLIENT)
}
override fun onCancel(callback: Callback?) {
Log.d(TAG, "onCancel called")
}
override fun onStopListening(callback: Callback?) {
Log.d(TAG, "onStopListening called")
}
}

View File

@@ -0,0 +1,21 @@
package com.agentui.desktop
import android.service.voice.VoiceInteractionService
import android.util.Log
class AgentVoiceInteractionService : VoiceInteractionService() {
override fun onReady() {
super.onReady()
Log.d(TAG, "VoiceInteractionService ready - registered as digital assistant")
}
override fun onShutdown() {
Log.d(TAG, "VoiceInteractionService shutting down")
super.onShutdown()
}
companion object {
private const val TAG = "AgentUI.VIS"
}
}

View File

@@ -0,0 +1,75 @@
package com.agentui.desktop
import android.content.Intent
import android.os.Build
import android.os.Bundle
import android.service.voice.VoiceInteractionSession
import android.service.voice.VoiceInteractionSessionService
import android.util.Log
class AgentVoiceInteractionSessionService : VoiceInteractionSessionService() {
override fun onNewSession(args: Bundle?): VoiceInteractionSession {
Log.d(TAG, "onNewSession called")
return AgentSession(this)
}
private class AgentSession(
private val ctx: android.content.Context
) : VoiceInteractionSession(ctx) {
override fun onPrepareShow(args: Bundle?, showFlags: Int) {
super.onPrepareShow(args, showFlags)
// Disable the default VoiceInteractionSession window/UI -
// we launch our own activity instead
setUiEnabled(false)
Log.d(TAG, "onPrepareShow: UI disabled, will launch activity")
}
override fun onShow(args: Bundle?, showFlags: Int) {
super.onShow(args, showFlags)
Log.d(TAG, "onShow called, flags=$showFlags")
val intent = Intent(ctx, VoiceCommandActivity::class.java).apply {
action = Intent.ACTION_VOICE_COMMAND
addFlags(Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_CLEAR_TOP)
}
try {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
// API 30+: startAssistantActivity is the proper method
// for launching activities from a VoiceInteractionSession
startAssistantActivity(intent)
Log.d(TAG, "startAssistantActivity dispatched (API 30+)")
} else {
// API 23-29: use startVoiceActivity
startVoiceActivity(intent)
Log.d(TAG, "startVoiceActivity dispatched (API <30)")
}
} catch (e: Exception) {
Log.e(TAG, "Assistant/Voice activity launch failed, trying startActivity", e)
try {
ctx.startActivity(intent)
Log.d(TAG, "Fallback startActivity dispatched")
} catch (e2: Exception) {
Log.e(TAG, "All launch methods failed", e2)
}
}
// Finish the session - the activity handles everything from here
finish()
}
override fun onHandleAssist(
data: Bundle?,
structure: android.app.assist.AssistStructure?,
content: android.app.assist.AssistContent?
) {
Log.d(TAG, "onHandleAssist called")
}
}
companion object {
private const val TAG = "AgentUI.VISS"
}
}

View File

@@ -0,0 +1,238 @@
package com.agentui.desktop
import android.content.BroadcastReceiver
import android.content.Context
import android.content.Intent
import android.util.Log
import android.view.View
import android.widget.RemoteViews
import okhttp3.OkHttpClient
import okhttp3.Request
import org.json.JSONObject
import java.util.concurrent.TimeUnit
/**
* Samsung Face Widget receiver for the lock screen / AOD.
*
* Samsung's proprietary system sends REQUEST_SERVICEBOX_REMOTEVIEWS
* and expects RESPONSE_SERVICEBOX_REMOTEVIEWS back with RemoteViews
* for both the lock screen ("origin") and AOD ("aod").
*/
class LockScreenWidgetReceiver : BroadcastReceiver() {
companion object {
private const val TAG = "AgentUI.FaceWidget"
private const val ACTION_REQUEST =
"com.samsung.android.intent.action.REQUEST_SERVICEBOX_REMOTEVIEWS"
private const val ACTION_RESPONSE =
"com.samsung.android.intent.action.RESPONSE_SERVICEBOX_REMOTEVIEWS"
private const val PAGE_ID = "agent_ui_transcript"
private val STATUS_COLORS = mapOf(
"idle" to 0xFF6b7280.toInt(),
"thinking" to 0xFF60a5fa.toInt(),
"reading" to 0xFF22d3ee.toInt(),
"writing" to 0xFF4ade80.toInt(),
"toolUse" to 0xFFfbbf24.toInt(),
"permissionRequest" to 0xFFfb923c.toInt(),
"interrupted" to 0xFFf87171.toInt(),
"error" to 0xFFf87171.toInt(),
"sessionStart" to 0xFF60a5fa.toInt(),
"sessionEnd" to 0xFF6b7280.toInt()
)
// Dimmed versions for AOD
private val AOD_STATUS_COLORS = mapOf(
"idle" to 0xFF3b3f47.toInt(),
"thinking" to 0xFF304f7a.toInt(),
"reading" to 0xFF116670.toInt(),
"writing" to 0xFF256b40.toInt(),
"toolUse" to 0xFF7a5f12.toInt(),
"permissionRequest" to 0xFF7a491e.toInt(),
"interrupted" to 0xFF7a3838.toInt(),
"error" to 0xFF7a3838.toInt(),
"sessionStart" to 0xFF304f7a.toInt(),
"sessionEnd" to 0xFF3b3f47.toInt()
)
private val client = OkHttpClient.Builder()
.connectTimeout(5, TimeUnit.SECONDS)
.readTimeout(5, TimeUnit.SECONDS)
.build()
}
override fun onReceive(context: Context, intent: Intent) {
if (intent.action != ACTION_REQUEST) return
val pageId = intent.getStringExtra("pageId")
if (pageId != PAGE_ID) return
Log.d(TAG, "Face widget update requested for pageId=$pageId")
// Fetch data on a background thread to avoid ANR
val pendingResult = goAsync()
Thread {
try {
val terminals = fetchTerminals(context)
val lockViews = buildLockScreenViews(context, terminals)
val aodViews = buildAodViews(context, terminals)
val response = Intent(ACTION_RESPONSE).apply {
setPackage("com.android.systemui")
putExtra("package", context.packageName)
putExtra("pageId", PAGE_ID)
putExtra("show", true)
putExtra("origin", lockViews)
putExtra("aod", aodViews)
}
context.sendBroadcast(response)
Log.d(TAG, "Face widget response sent with ${terminals.size} terminals")
} catch (e: Exception) {
Log.e(TAG, "Failed to build face widget", e)
// Send empty response so widget doesn't get stuck
val response = Intent(ACTION_RESPONSE).apply {
setPackage("com.android.systemui")
putExtra("package", context.packageName)
putExtra("pageId", PAGE_ID)
putExtra("show", true)
putExtra("origin", buildEmptyLockScreenViews(context))
putExtra("aod", buildEmptyAodViews(context))
}
context.sendBroadcast(response)
} finally {
pendingResult.finish()
}
}.start()
}
private fun buildLockScreenViews(
context: Context,
terminals: List<FaceWidgetTerminal>
): RemoteViews {
val views = RemoteViews(context.packageName, R.layout.face_widget_lockscreen)
val statusText = if (terminals.isEmpty()) "offline"
else "${terminals.size} agent${if (terminals.size > 1) "s" else ""}"
views.setTextViewText(R.id.fw_status, statusText)
if (terminals.isEmpty()) {
views.setViewVisibility(R.id.fw_empty, View.VISIBLE)
} else {
views.setViewVisibility(R.id.fw_empty, View.GONE)
}
// Lockscreen terminal slot IDs
val slotIds = listOf(
Triple(R.id.fw_terminal_1, R.id.fw_dot_1, R.id.fw_name_1),
Triple(R.id.fw_terminal_2, R.id.fw_dot_2, R.id.fw_name_2),
Triple(R.id.fw_terminal_3, R.id.fw_dot_3, R.id.fw_name_3)
)
for (i in slotIds.indices) {
val (container, dot, name) = slotIds[i]
if (i < terminals.size) {
val t = terminals[i]
views.setViewVisibility(container, View.VISIBLE)
views.setTextColor(dot, t.statusColor)
views.setTextViewText(name, "T${t.index} ${t.agent} ${t.status}")
} else {
views.setViewVisibility(container, View.GONE)
}
}
return views
}
private fun buildAodViews(
context: Context,
terminals: List<FaceWidgetTerminal>
): RemoteViews {
val views = RemoteViews(context.packageName, R.layout.face_widget_aod)
if (terminals.isEmpty()) {
views.setViewVisibility(R.id.fw_aod_empty, View.VISIBLE)
} else {
views.setViewVisibility(R.id.fw_aod_empty, View.GONE)
}
val slotIds = listOf(
Triple(R.id.fw_aod_terminal_1, R.id.fw_aod_dot_1, R.id.fw_aod_name_1),
Triple(R.id.fw_aod_terminal_2, R.id.fw_aod_dot_2, R.id.fw_aod_name_2),
Triple(R.id.fw_aod_terminal_3, R.id.fw_aod_dot_3, R.id.fw_aod_name_3)
)
for (i in slotIds.indices) {
val (container, dot, name) = slotIds[i]
if (i < terminals.size) {
val t = terminals[i]
views.setViewVisibility(container, View.VISIBLE)
val aodColor = AOD_STATUS_COLORS[t.status] ?: AOD_STATUS_COLORS["idle"]!!
views.setTextColor(dot, aodColor)
views.setTextViewText(name, "T${t.index} ${t.agent}")
} else {
views.setViewVisibility(container, View.GONE)
}
}
return views
}
private fun buildEmptyLockScreenViews(context: Context): RemoteViews {
return RemoteViews(context.packageName, R.layout.face_widget_lockscreen)
}
private fun buildEmptyAodViews(context: Context): RemoteViews {
return RemoteViews(context.packageName, R.layout.face_widget_aod)
}
private fun fetchTerminals(context: Context): List<FaceWidgetTerminal> {
val apiBase = ServerConfig.apiBaseUrl(context) ?: return emptyList()
try {
val url = "$apiBase/session-state"
val req = Request.Builder().url(url).build()
val resp = client.newCall(req).execute()
if (!resp.isSuccessful) return emptyList()
val json = JSONObject(resp.body?.string() ?: "{}")
val registry = json.optJSONArray("registry") ?: return emptyList()
val agents = json.optJSONObject("agents")
val result = mutableListOf<FaceWidgetTerminal>()
for (i in 0 until minOf(registry.length(), 3)) { // Max 3 for face widget
val entry = registry.getJSONObject(i)
val agentName = entry.optString("agent", "")
val alive = entry.optBoolean("alive", false)
val agentState = agents?.optJSONObject(agentName)
val status = agentState?.optString("status", if (alive) "idle" else "closed")
?: if (alive) "idle" else "closed"
val statusColor = STATUS_COLORS[status] ?: STATUS_COLORS["idle"]!!
result.add(
FaceWidgetTerminal(
index = i + 1,
agent = agentName,
status = status,
statusColor = statusColor
)
)
}
return result
} catch (e: Exception) {
Log.w(TAG, "Failed to fetch terminals for face widget", e)
return emptyList()
}
}
private data class FaceWidgetTerminal(
val index: Int,
val agent: String,
val status: String,
val statusColor: Int
)
}

View File

@@ -1,40 +1,279 @@
package com.agentui.desktop
import android.app.PendingIntent
import android.app.PictureInPictureParams
import android.app.RemoteAction
import android.content.BroadcastReceiver
import android.content.Context
import android.content.Intent
import android.content.IntentFilter
import android.content.res.Configuration
import android.graphics.drawable.Icon
import android.os.Build
import android.os.Bundle
import android.util.Log
import android.util.Rational
import android.webkit.WebView
import androidx.activity.enableEdgeToEdge
import org.json.JSONObject
import java.io.File
import androidx.core.view.ViewCompat
import androidx.core.view.WindowInsetsCompat
class MainActivity : TauriActivity() {
companion object {
private const val ACTION_PIP_MIC = "com.agentui.desktop.PIP_MIC"
private const val ACTION_PIP_EXPAND = "com.agentui.desktop.PIP_EXPAND"
private const val PIP_REQUEST_CODE_MIC = 2001
private const val PIP_REQUEST_CODE_EXPAND = 2002
}
private var pendingRoute: String? = null
private var pendingVoiceTerminal: String? = null
private val pipReceiver = object : BroadcastReceiver() {
override fun onReceive(context: Context, intent: Intent) {
when (intent.action) {
ACTION_PIP_MIC -> {
Log.d("AgentUI", "PiP mic button pressed")
// Launch voice command from PiP
val voiceIntent = Intent(context, VoiceCommandActivity::class.java).apply {
action = Intent.ACTION_VOICE_COMMAND
addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
}
startActivity(voiceIntent)
}
ACTION_PIP_EXPAND -> {
Log.d("AgentUI", "PiP expand button pressed")
// Bring app to foreground full-screen
val expandIntent = Intent(context, MainActivity::class.java).apply {
addFlags(Intent.FLAG_ACTIVITY_REORDER_TO_FRONT or Intent.FLAG_ACTIVITY_SINGLE_TOP)
}
startActivity(expandIntent)
}
}
}
}
override fun onCreate(savedInstanceState: Bundle?) {
enableEdgeToEdge()
super.onCreate(savedInstanceState)
syncServerUrlToPrefs()
injectSafeAreaInsets()
handleWidgetIntent(intent)
handleVoiceIntent(intent)
registerPipReceiver()
}
override fun onDestroy() {
super.onDestroy()
try { unregisterReceiver(pipReceiver) } catch (_: Exception) {}
}
private fun registerPipReceiver() {
val filter = IntentFilter().apply {
addAction(ACTION_PIP_MIC)
addAction(ACTION_PIP_EXPAND)
}
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
registerReceiver(pipReceiver, filter, RECEIVER_NOT_EXPORTED)
} else {
registerReceiver(pipReceiver, filter)
}
}
/**
* Enter PiP when user leaves the app (home button / swipe).
*/
override fun onUserLeaveHint() {
super.onUserLeaveHint()
// Only enter PiP via voice command flow, not on regular minimize
}
override fun onPictureInPictureModeChanged(
isInPictureInPictureMode: Boolean,
newConfig: Configuration
) {
super.onPictureInPictureModeChanged(isInPictureInPictureMode, newConfig)
Log.d("AgentUI", "PiP mode changed: $isInPictureInPictureMode")
}
private fun enterPipIfSupported() {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
try {
val actions = buildPipActions()
val builder = PictureInPictureParams.Builder()
.setAspectRatio(Rational(9, 16))
.setActions(actions)
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
builder.setAutoEnterEnabled(false)
builder.setSeamlessResizeEnabled(true)
}
enterPictureInPictureMode(builder.build())
} catch (e: Exception) {
Log.w("AgentUI", "Failed to enter PiP: $e")
}
}
}
private fun buildPipActions(): List<RemoteAction> {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) return emptyList()
val actions = mutableListOf<RemoteAction>()
// Mic button — launch voice command
val micIntent = PendingIntent.getBroadcast(
this, PIP_REQUEST_CODE_MIC,
Intent(ACTION_PIP_MIC).setPackage(packageName),
PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE
)
actions.add(RemoteAction(
Icon.createWithResource(this, android.R.drawable.ic_btn_speak_now),
"Voice", "Send voice command",
micIntent
))
// Expand button — bring app to foreground
val expandIntent = PendingIntent.getBroadcast(
this, PIP_REQUEST_CODE_EXPAND,
Intent(ACTION_PIP_EXPAND).setPackage(packageName),
PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE
)
actions.add(RemoteAction(
Icon.createWithResource(this, android.R.drawable.ic_menu_view),
"Open", "Open full app",
expandIntent
))
return actions
}
override fun onNewIntent(intent: android.content.Intent) {
super.onNewIntent(intent)
handleWidgetIntent(intent)
handleVoiceIntent(intent)
}
override fun onResume() {
super.onResume()
syncServerUrlToPrefs()
pendingRoute?.let { route ->
pendingRoute = null
navigateWebView(route)
}
}
private fun handleWidgetIntent(intent: android.content.Intent?) {
val terminalIndex = intent?.getIntExtra("terminalIndex", -1) ?: -1
if (terminalIndex > 0) {
val route = "/transcript-debug/$terminalIndex"
Log.d("AgentUI", "Widget click → navigate to $route")
pendingRoute = route
navigateWebView(route)
}
}
private fun handleVoiceIntent(intent: android.content.Intent?) {
if (intent?.action == "com.agentui.desktop.VOICE_TERMINAL") {
val sessionId = intent.getStringExtra("ephemeralSessionId") ?: return
if (sessionId.isNotEmpty()) {
Log.d("AgentUI", "Voice intent → open terminal $sessionId")
pendingVoiceTerminal = sessionId
// Don't call openVoiceTerminal here — WebView may not exist yet.
// Instead, poll until the WebView is ready.
pollForWebViewAndOpenTerminal(sessionId, 0)
}
}
}
/**
* Reads the Tauri plugin-store settings.json and syncs serverUrl
* to SharedPreferences so native components (widget, voice) can use it.
* Retry finding the WebView up to ~3 seconds (15 attempts x 200ms).
* Once found, inject JS to open floating transcript and enter PiP.
*/
private fun syncServerUrlToPrefs() {
try {
val storeFile = File(filesDir, "app_tauri-plugin-store/settings.json")
if (!storeFile.exists()) return
private fun pollForWebViewAndOpenTerminal(ephemeralSessionId: String, attempt: Int) {
if (attempt > 15) {
Log.w("AgentUI", "Gave up waiting for WebView after ${attempt} attempts")
pendingVoiceTerminal = null
return
}
val json = JSONObject(storeFile.readText())
val serverUrl = json.optString("serverUrl", "")
if (serverUrl.isNotEmpty()) {
ServerConfig.setServerUrl(this, serverUrl)
Log.d("AgentUI", "Synced serverUrl to prefs: $serverUrl")
}
} catch (e: Exception) {
Log.w("AgentUI", "Failed to sync server URL", e)
val webView = try { findWebView(window.decorView) } catch (_: Exception) { null }
if (webView != null) {
val js = "window.__VOICE_OPEN_TERMINAL__ && window.__VOICE_OPEN_TERMINAL__('$ephemeralSessionId')"
webView.evaluateJavascript(js, null)
pendingVoiceTerminal = null
Log.d("AgentUI", "Voice terminal JS dispatched (attempt $attempt): $ephemeralSessionId")
// Enter PiP after the WebView has time to render
webView.postDelayed({ enterPipIfSupported() }, 500)
} else {
Log.d("AgentUI", "WebView not ready (attempt $attempt), retrying in 200ms")
window.decorView.postDelayed({
pollForWebViewAndOpenTerminal(ephemeralSessionId, attempt + 1)
}, 200)
}
}
private fun navigateWebView(route: String) {
try {
val decorView = window.decorView
val webView = findWebView(decorView)
if (webView != null) {
val js = "window.__WIDGET_NAVIGATE__ && window.__WIDGET_NAVIGATE__('$route') || (window.location.href = '$route')"
webView.evaluateJavascript(js, null)
pendingRoute = null
}
} catch (e: Exception) {
Log.w("AgentUI", "Failed to navigate WebView: $e")
}
}
private fun injectSafeAreaInsets() {
val decorView = window.decorView
ViewCompat.setOnApplyWindowInsetsListener(decorView) { view, insets ->
val systemBars = insets.getInsets(WindowInsetsCompat.Type.systemBars())
val density = resources.displayMetrics.density
val topPx = systemBars.top
val bottomPx = systemBars.bottom
val leftPx = systemBars.left
val rightPx = systemBars.right
val topDp = topPx / density
val bottomDp = bottomPx / density
val leftDp = leftPx / density
val rightDp = rightPx / density
val js = """
document.documentElement.style.setProperty('--sat', '${topDp}px');
document.documentElement.style.setProperty('--sab', '${bottomDp}px');
document.documentElement.style.setProperty('--sal', '${leftDp}px');
document.documentElement.style.setProperty('--sar', '${rightDp}px');
""".trimIndent()
try {
val webView = view.findViewWithTag<WebView>("tauri_webview")
?: view.findViewById<WebView>(android.R.id.content)?.let {
findWebView(it)
}
webView?.evaluateJavascript(js, null)
Log.d("AgentUI", "Injected safe-area: top=${topDp}dp bottom=${bottomDp}dp")
} catch (e: Exception) {
Log.w("AgentUI", "Failed to inject safe-area insets: $e")
}
ViewCompat.onApplyWindowInsets(view, insets)
}
}
private fun findWebView(view: android.view.View): WebView? {
if (view is WebView) return view
if (view is android.view.ViewGroup) {
for (i in 0 until view.childCount) {
val found = findWebView(view.getChildAt(i))
if (found != null) return found
}
}
return null
}
private fun syncServerUrlToPrefs() {
val url = ServerConfig.getServerUrl(this)
Log.d("AgentUI", "syncServerUrlToPrefs: resolved url=$url")
}
}

View File

@@ -1,21 +1,149 @@
package com.agentui.desktop
import android.content.Context
import android.util.Log
import org.json.JSONObject
import java.io.File
object ServerConfig {
private const val TAG = "ServerConfig"
private const val PREFS_NAME = "agent_ui_config"
private const val KEY_SERVER_URL = "server_url"
fun getServerUrl(context: Context): String? {
return context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE)
val stored = context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE)
.getString(KEY_SERVER_URL, null)
if (!stored.isNullOrEmpty()) {
Log.d(TAG, "Found serverUrl in SharedPreferences: $stored")
return stored
}
// Fallback: search for Tauri store file in multiple possible locations
return readFromTauriStore(context)
}
fun setServerUrl(context: Context, url: String) {
val normalized = url.trimEnd('/')
context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE)
.edit()
.putString(KEY_SERVER_URL, url.trimEnd('/'))
.putString(KEY_SERVER_URL, normalized)
.apply()
Log.d(TAG, "Saved serverUrl to SharedPreferences: $normalized")
}
/**
* Searches for serverUrl in the Tauri plugin-store settings.json.
* Tries multiple possible file locations since the Tauri store path
* can vary by plugin version and platform.
*/
private fun readFromTauriStore(context: Context): String? {
Log.d(TAG, "SharedPreferences empty, searching Tauri store files...")
// Try multiple possible paths where Tauri plugin-store may save settings.json
val possiblePaths = listOf(
File(context.filesDir, "app_tauri-plugin-store/settings.json"),
File(context.filesDir, "settings.json"),
File(context.dataDir, "app_tauri-plugin-store/settings.json"),
File(context.dataDir, "settings.json"),
File(context.filesDir, ".tauri/settings.json"),
File(context.filesDir, "tauri-plugin-store/settings.json")
)
for (file in possiblePaths) {
Log.d(TAG, " Checking: ${file.absolutePath} exists=${file.exists()}")
if (file.exists()) {
try {
val content = file.readText()
Log.d(TAG, " Found store file! Content: ${content.take(200)}")
val url = extractServerUrl(content)
if (url != null) {
setServerUrl(context, url)
Log.d(TAG, " Extracted and cached serverUrl: $url")
return url
}
} catch (e: Exception) {
Log.w(TAG, " Failed to parse: ${file.absolutePath}", e)
}
}
}
// Also search recursively for any settings.json in the data directory
val found = findFile(context.filesDir, "settings.json")
if (found != null) {
Log.d(TAG, " Found via recursive search: ${found.absolutePath}")
try {
val content = found.readText()
Log.d(TAG, " Content: ${content.take(200)}")
val url = extractServerUrl(content)
if (url != null) {
setServerUrl(context, url)
return url
}
} catch (e: Exception) {
Log.w(TAG, " Failed to parse found file", e)
}
}
// Debug: dump directory tree so we can see where files actually are
Log.w(TAG, " Could not find settings.json. Directory tree of filesDir:")
dumpTree(context.filesDir, " ")
Log.w(TAG, " Directory tree of dataDir:")
dumpTree(context.dataDir, " ")
return null
}
/**
* Extracts serverUrl from JSON content.
* Handles both flat format {"serverUrl":"..."} and
* possible wrapped formats.
*/
private fun extractServerUrl(content: String): String? {
try {
val json = JSONObject(content)
// Direct key
val direct = json.optString("serverUrl", "")
if (direct.isNotEmpty()) return direct.trimEnd('/')
// Maybe nested under a "store" or "data" key
for (key in json.keys()) {
val inner = json.optJSONObject(key)
if (inner != null) {
val nested = inner.optString("serverUrl", "")
if (nested.isNotEmpty()) return nested.trimEnd('/')
}
}
} catch (e: Exception) {
Log.w(TAG, "extractServerUrl parse error", e)
}
return null
}
/** Recursively find a file by name */
private fun findFile(dir: File, name: String): File? {
val files = dir.listFiles() ?: return null
for (file in files) {
if (file.isFile && file.name == name) return file
if (file.isDirectory) {
val found = findFile(file, name)
if (found != null) return found
}
}
return null
}
/** Dump directory tree for debugging */
private fun dumpTree(dir: File, indent: String) {
val files = dir.listFiles() ?: return
for (file in files) {
if (file.isDirectory) {
Log.d(TAG, "$indent${file.name}/")
dumpTree(file, "$indent ")
} else {
Log.d(TAG, "$indent${file.name} (${file.length()} bytes)")
}
}
}
/** e.g. "http://192.168.1.10:4103" */

View File

@@ -0,0 +1,260 @@
package com.agentui.desktop
import android.appwidget.AppWidgetManager
import android.content.ComponentName
import android.content.Context
import android.content.Intent
import android.os.Handler
import android.os.Looper
import android.util.Log
import android.widget.RemoteViews
import android.widget.RemoteViewsService
import okhttp3.OkHttpClient
import okhttp3.Request
import org.json.JSONObject
import java.util.concurrent.TimeUnit
class TerminalListWidgetService : RemoteViewsService() {
override fun onGetViewFactory(intent: Intent): RemoteViewsFactory {
return TerminalListFactory(applicationContext)
}
}
data class TerminalItem(
val ephemeralSessionId: String,
val agent: String,
val label: String,
val status: String,
val alive: Boolean,
val hookBadges: String,
val lastUserPrompt: String,
val statusColor: Int,
val terminalIndex: Int // 1-based, maps to /transcript-debug/:terminalIndex
)
class TerminalListFactory(private val context: Context) : RemoteViewsService.RemoteViewsFactory {
companion object {
private const val TAG = "WidgetListFactory"
// Refresh button states
private const val ICON_NORMAL = "\u21BB" // ↻
private const val ICON_LOADING = "\u23F3" // ⏳
private const val ICON_OK = "\u2713" // ✓
private const val ICON_ERROR = "\u26A0" // ⚠
private const val COLOR_NORMAL = 0xFF8888FF.toInt()
private const val COLOR_LOADING = 0xFF60a5fa.toInt()
private const val COLOR_OK = 0xFF4ade80.toInt()
private const val COLOR_ERROR = 0xFFf87171.toInt()
private val client = OkHttpClient.Builder()
.connectTimeout(8, TimeUnit.SECONDS)
.readTimeout(8, TimeUnit.SECONDS)
.build()
private val STATUS_COLORS = mapOf(
"idle" to 0xFF6b7280.toInt(),
"thinking" to 0xFF60a5fa.toInt(),
"reading" to 0xFF22d3ee.toInt(),
"writing" to 0xFF4ade80.toInt(),
"toolUse" to 0xFFfbbf24.toInt(),
"permissionRequest" to 0xFFfb923c.toInt(),
"interrupted" to 0xFFf87171.toInt(),
"error" to 0xFFf87171.toInt(),
"sessionStart" to 0xFF60a5fa.toInt(),
"sessionEnd" to 0xFF6b7280.toInt()
)
private val TOOL_EVENTS = setOf("PreToolUse", "PostToolUse", "PostToolUseFailure")
private val PERM_EVENTS = setOf("PermissionRequest")
private val SESSION_EVENTS = setOf("SessionStart", "UserPromptSubmit", "Stop", "SessionEnd")
}
private var items = listOf<TerminalItem>()
private val mainHandler = Handler(Looper.getMainLooper())
override fun onCreate() {}
override fun onDataSetChanged() {
setRefreshButton(ICON_LOADING, COLOR_LOADING)
val result = fetchTerminals()
if (result == null) {
// Network error — keep previous items, show error on button
setRefreshButton(ICON_ERROR, COLOR_ERROR)
scheduleResetButton(3000)
} else {
items = result
// Brief success flash, then back to normal
setRefreshButton(ICON_OK, COLOR_OK)
scheduleResetButton(1500)
}
}
override fun onDestroy() {
items = emptyList()
}
override fun getCount(): Int = items.size
override fun getViewAt(position: Int): RemoteViews {
val views = RemoteViews(context.packageName, R.layout.widget_terminal_item)
if (position >= items.size) return views
val item = items[position]
views.setTextColor(R.id.item_dot, item.statusColor)
val statusLabel = if (item.alive) item.status else "closed"
views.setTextViewText(R.id.item_name, "T${item.terminalIndex} ${item.agent} $statusLabel")
views.setTextViewText(R.id.item_badges, item.hookBadges)
// Always show the registry label (unique per terminal)
views.setTextViewText(R.id.item_label, item.label)
val fillIntent = Intent().apply {
putExtra("terminalIndex", item.terminalIndex)
putExtra("agent", item.agent)
}
views.setOnClickFillInIntent(R.id.item_root, fillIntent)
return views
}
override fun getLoadingView(): RemoteViews? = null
override fun getViewTypeCount(): Int = 1
override fun getItemId(position: Int): Long = position.toLong()
override fun hasStableIds(): Boolean = false
// ── Refresh button state management ──
/**
* Update just the refresh button via partiallyUpdateAppWidget.
* This doesn't reset the ListView adapter.
*/
private fun setRefreshButton(icon: String, color: Int) {
try {
val views = RemoteViews(context.packageName, R.layout.widget_transcript)
views.setTextViewText(R.id.btn_refresh, icon)
views.setTextColor(R.id.btn_refresh, color)
val mgr = AppWidgetManager.getInstance(context)
val ids = mgr.getAppWidgetIds(
ComponentName(context, TranscriptWidgetProvider::class.java)
)
mgr.partiallyUpdateAppWidget(ids, views)
} catch (e: Exception) {
Log.w(TAG, "Failed to update refresh button", e)
}
}
/**
* Schedule resetting the button back to normal after a delay.
*/
private fun scheduleResetButton(delayMs: Long) {
mainHandler.postDelayed({
setRefreshButton(ICON_NORMAL, COLOR_NORMAL)
}, delayMs)
}
// ── Data fetching ──
/**
* Returns list on success, null on error.
* Items keep the same order as the terminal registry (T1, T2, T3...)
* so the index maps directly to /transcript-debug/:terminalIndex
*/
private fun fetchTerminals(): List<TerminalItem>? {
val apiBase = ServerConfig.apiBaseUrl(context) ?: return emptyList()
try {
val url = "$apiBase/session-state"
val req = Request.Builder().url(url).build()
val resp = client.newCall(req).execute()
if (!resp.isSuccessful) return null
val json = JSONObject(resp.body?.string() ?: "{}")
val registry = json.optJSONArray("registry")
val agents = json.optJSONObject("agents")
if (registry == null || registry.length() == 0) return emptyList()
val result = mutableListOf<TerminalItem>()
for (i in 0 until registry.length()) {
val entry = registry.getJSONObject(i)
val agentName = entry.optString("agent", "")
val ephId = entry.optString("ephemeralSessionId", "")
val label = entry.optString("label", "")
val alive = entry.optBoolean("alive", false)
val terminalIndex = i + 1 // 1-based, maps to /transcript-debug/:terminalIndex
val agentState = agents?.optJSONObject(agentName)
val status = agentState?.optString("status", if (alive) "idle" else "closed")
?: if (alive) "idle" else "closed"
val statusColor = STATUS_COLORS[status] ?: STATUS_COLORS["idle"]!!
val lastUserPrompt = if (agentState != null) extractLastUserPrompt(agentState) else ""
val hookBadges = if (agentState != null) buildBadgeString(agentState) else ""
result.add(
TerminalItem(
ephemeralSessionId = ephId,
agent = agentName,
label = label,
status = status,
alive = alive,
hookBadges = hookBadges,
lastUserPrompt = lastUserPrompt,
statusColor = statusColor,
terminalIndex = terminalIndex
)
)
}
// Keep registry order (don't sort) — index must match T1, T2, T3...
return result
} catch (e: Exception) {
Log.w(TAG, "Failed to fetch terminals", e)
return null
}
}
private fun extractLastUserPrompt(state: JSONObject): String {
val history = state.optJSONArray("hookHistory") ?: return ""
for (i in history.length() - 1 downTo 0) {
val entry = history.optJSONObject(i) ?: continue
if (entry.optString("event") == "UserPromptSubmit") {
val detail = entry.optString("detail", "")
if (detail.isNotEmpty()) return detail.take(120)
}
}
val stopResp = state.optString("lastStopResponse", "")
if (stopResp.isNotEmpty()) return "< ${stopResp.take(100)}"
return ""
}
private fun buildBadgeString(state: JSONObject): String {
val history = state.optJSONArray("hookHistory") ?: return ""
var tools = 0; var perms = 0; var sessions = 0
for (i in 0 until history.length()) {
val entry = history.optJSONObject(i) ?: continue
val event = entry.optString("event", "")
when {
event in TOOL_EVENTS -> tools++
event in PERM_EVENTS -> perms++
event in SESSION_EVENTS -> sessions++
}
}
val parts = mutableListOf<String>()
if (tools > 0) parts.add("T:$tools")
if (perms > 0) parts.add("P:$perms")
if (sessions > 0) parts.add("S:$sessions")
return parts.joinToString(" ")
}
}

View File

@@ -3,12 +3,19 @@ package com.agentui.desktop
import android.app.PendingIntent
import android.appwidget.AppWidgetManager
import android.appwidget.AppWidgetProvider
import android.content.ComponentName
import android.content.Context
import android.content.Intent
import android.net.Uri
import android.widget.RemoteViews
class TranscriptWidgetProvider : AppWidgetProvider() {
companion object {
const val ACTION_REFRESH = "com.agentui.desktop.WIDGET_REFRESH"
const val ACTION_ITEM_CLICK = "com.agentui.desktop.WIDGET_ITEM_CLICK"
}
override fun onUpdate(
context: Context,
appWidgetManager: AppWidgetManager,
@@ -17,22 +24,61 @@ class TranscriptWidgetProvider : AppWidgetProvider() {
for (appWidgetId in appWidgetIds) {
val views = RemoteViews(context.packageName, R.layout.widget_transcript)
// Tap widget → open main app
val intent = Intent(context, MainActivity::class.java)
val pendingIntent = PendingIntent.getActivity(
context, 0, intent,
// ListView adapter
val serviceIntent = Intent(context, TerminalListWidgetService::class.java).apply {
putExtra(AppWidgetManager.EXTRA_APPWIDGET_ID, appWidgetId)
data = Uri.parse(toUri(Intent.URI_INTENT_SCHEME))
}
views.setRemoteAdapter(R.id.terminal_list, serviceIntent)
views.setEmptyView(R.id.terminal_list, R.id.empty_view)
// Item click template → opens app
val itemClickIntent = Intent(context, MainActivity::class.java).apply {
action = ACTION_ITEM_CLICK
}
val itemClickPending = PendingIntent.getActivity(
context, 0, itemClickIntent,
PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_MUTABLE
)
views.setPendingIntentTemplate(R.id.terminal_list, itemClickPending)
// Refresh button
val refreshIntent = Intent(context, TranscriptWidgetProvider::class.java).apply {
action = ACTION_REFRESH
}
val refreshPending = PendingIntent.getBroadcast(
context, 1, refreshIntent,
PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE
)
views.setOnClickPendingIntent(R.id.widget_root, pendingIntent)
views.setOnClickPendingIntent(R.id.btn_refresh, refreshPending)
// Title tap → open app
val appIntent = Intent(context, MainActivity::class.java)
val appPending = PendingIntent.getActivity(
context, 2, appIntent,
PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE
)
views.setOnClickPendingIntent(R.id.widget_title, appPending)
appWidgetManager.updateAppWidget(appWidgetId, views)
}
}
// Start periodic refresh worker
TranscriptWidgetWorker.enqueue(context)
override fun onReceive(context: Context, intent: Intent) {
super.onReceive(context, intent)
if (intent.action == ACTION_REFRESH) {
// Notify the ListView adapter to refresh its data
val appWidgetManager = AppWidgetManager.getInstance(context)
val widgetIds = appWidgetManager.getAppWidgetIds(
ComponentName(context, TranscriptWidgetProvider::class.java)
)
appWidgetManager.notifyAppWidgetViewDataChanged(widgetIds, R.id.terminal_list)
}
}
override fun onEnabled(context: Context) {
// Periodic refresh via WorkManager for background updates
TranscriptWidgetWorker.enqueue(context)
}

View File

@@ -3,14 +3,13 @@ package com.agentui.desktop
import android.appwidget.AppWidgetManager
import android.content.ComponentName
import android.content.Context
import android.widget.RemoteViews
import androidx.work.*
import okhttp3.OkHttpClient
import okhttp3.Request
import org.json.JSONArray
import org.json.JSONObject
import java.util.concurrent.TimeUnit
/**
* Periodic background worker that triggers the widget ListView to refresh.
* The actual data fetching happens in TerminalListFactory.onDataSetChanged().
*/
class TranscriptWidgetWorker(
private val context: Context,
params: WorkerParameters
@@ -18,13 +17,6 @@ class TranscriptWidgetWorker(
companion object {
const val WORK_NAME = "transcript_widget_refresh"
private val MSG_IDS = intArrayOf(
R.id.msg1, R.id.msg2, R.id.msg3, R.id.msg4, R.id.msg5
)
private val client = OkHttpClient.Builder()
.connectTimeout(10, TimeUnit.SECONDS)
.readTimeout(10, TimeUnit.SECONDS)
.build()
fun enqueue(context: Context) {
val request = OneTimeWorkRequestBuilder<TranscriptWidgetWorker>()
@@ -39,108 +31,20 @@ class TranscriptWidgetWorker(
}
override suspend fun doWork(): Result {
val messages = fetchMessages()
updateWidget(messages)
scheduleNext()
return Result.success()
}
private fun fetchMessages(): List<String> {
val apiBase = ServerConfig.apiBaseUrl(context) ?: return listOf("No server configured")
try {
// Get most recent session for 'ejecutor' agent
val sessionsUrl = "$apiBase/transcript-debug/sessions?agent=ejecutor"
val sessionsReq = Request.Builder().url(sessionsUrl).build()
val sessionsResp = client.newCall(sessionsReq).execute()
if (!sessionsResp.isSuccessful) return listOf("Failed to fetch sessions")
val sessionsJson = JSONArray(sessionsResp.body?.string() ?: "[]")
if (sessionsJson.length() == 0) return listOf("No sessions found")
// Get the most recent session (sorted by mtime desc from server)
val latestSession = sessionsJson.getJSONObject(0)
val sessionId = latestSession.getString("id")
// Fetch raw JSONL
val rawUrl = "$apiBase/transcript-debug/$sessionId/raw?agent=ejecutor"
val rawReq = Request.Builder().url(rawUrl).build()
val rawResp = client.newCall(rawReq).execute()
if (!rawResp.isSuccessful) return listOf("Failed to fetch transcript")
val rawText = rawResp.body?.string() ?: return listOf("Empty transcript")
return parseJsonlMessages(rawText)
} catch (e: Exception) {
return listOf("Error: ${e.message?.take(40)}")
}
}
private fun parseJsonlMessages(jsonl: String): List<String> {
val messages = mutableListOf<String>()
val lines = jsonl.trim().split("\n").filter { it.isNotBlank() }
for (line in lines) {
try {
val obj = JSONObject(line)
val type = obj.optString("type", "")
when (type) {
"human" -> {
val content = obj.optString("message", "")
.ifEmpty { extractMessageContent(obj) }
if (content.isNotEmpty()) {
messages.add("> ${content.take(80)}")
}
}
"assistant" -> {
val content = extractMessageContent(obj)
if (content.isNotEmpty()) {
messages.add("< ${content.take(80)}")
}
}
}
} catch (_: Exception) { }
}
return messages.takeLast(5).ifEmpty { listOf("No messages yet") }
}
private fun extractMessageContent(obj: JSONObject): String {
// Try "message" field first (simple string)
val msg = obj.optString("message", "")
if (msg.isNotEmpty()) return msg
// Try "message" as array of content blocks
val msgArray = obj.optJSONArray("message")
if (msgArray != null) {
for (i in 0 until msgArray.length()) {
val block = msgArray.optJSONObject(i) ?: continue
if (block.optString("type") == "text") {
return block.optString("text", "")
}
}
}
return ""
}
private fun updateWidget(messages: List<String>) {
val views = RemoteViews(context.packageName, R.layout.widget_transcript)
for (i in MSG_IDS.indices) {
views.setTextViewText(MSG_IDS[i], messages.getOrElse(i) { "" })
}
// Tell the widget's ListView adapter to re-fetch data
val appWidgetManager = AppWidgetManager.getInstance(context)
val widgetComponent = ComponentName(context, TranscriptWidgetProvider::class.java)
appWidgetManager.updateAppWidget(widgetComponent, views)
}
val widgetIds = appWidgetManager.getAppWidgetIds(
ComponentName(context, TranscriptWidgetProvider::class.java)
)
appWidgetManager.notifyAppWidgetViewDataChanged(widgetIds, R.id.terminal_list)
private fun scheduleNext() {
val request = OneTimeWorkRequestBuilder<TranscriptWidgetWorker>()
// Schedule next refresh
val next = OneTimeWorkRequestBuilder<TranscriptWidgetWorker>()
.setInitialDelay(30, TimeUnit.SECONDS)
.build()
WorkManager.getInstance(context)
.enqueueUniqueWork(WORK_NAME, ExistingWorkPolicy.REPLACE, request)
.enqueueUniqueWork(WORK_NAME, ExistingWorkPolicy.REPLACE, next)
return Result.success()
}
}

View File

@@ -10,7 +10,9 @@ import okhttp3.*
import okhttp3.MediaType.Companion.toMediaType
import okhttp3.RequestBody.Companion.toRequestBody
import org.json.JSONObject
import java.util.concurrent.CountDownLatch
import java.text.SimpleDateFormat
import java.util.Date
import java.util.Locale
import java.util.concurrent.TimeUnit
class VoiceCommandActivity : Activity() {
@@ -27,28 +29,44 @@ class VoiceCommandActivity : Activity() {
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
Log.d(TAG, "VoiceCommandActivity created, action=${intent?.action}")
// Check if launched via share intent
if (intent?.action == Intent.ACTION_SEND && intent.type == "text/plain") {
val sharedText = intent.getStringExtra(Intent.EXTRA_TEXT)
if (!sharedText.isNullOrBlank()) {
sendToServer(sharedText)
when (intent?.action) {
// Launched via share intent
Intent.ACTION_SEND -> {
if (intent.type == "text/plain") {
val sharedText = intent.getStringExtra(Intent.EXTRA_TEXT)
if (!sharedText.isNullOrBlank()) {
sendTranscript(sharedText, "share-intent")
return
}
}
}
// Launched as assist app or voice command
Intent.ACTION_ASSIST,
Intent.ACTION_VOICE_COMMAND,
"android.intent.action.VOICE_COMMAND" -> {
Log.d(TAG, "Launched as assistant, starting speech recognition")
startSpeechRecognition()
return
}
}
// Otherwise, start speech recognition
// Default fallback: start speech recognition
startSpeechRecognition()
}
private fun startSpeechRecognition() {
val intent = Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply {
putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM)
putExtra(RecognizerIntent.EXTRA_PROMPT, "Agent UI - Speak your command")
putExtra(RecognizerIntent.EXTRA_LANGUAGE, "es-HN")
putExtra(RecognizerIntent.EXTRA_LANGUAGE_PREFERENCE, "es-HN")
putExtra(RecognizerIntent.EXTRA_PROMPT, "Agent UI - Decí tu comando")
}
try {
startActivityForResult(intent, SPEECH_REQUEST_CODE)
} catch (e: Exception) {
Log.e(TAG, "Speech recognition not available", e)
Toast.makeText(this, "Speech recognition not available", Toast.LENGTH_SHORT).show()
finish()
}
@@ -62,95 +80,90 @@ class VoiceCommandActivity : Activity() {
val results = data.getStringArrayListExtra(RecognizerIntent.EXTRA_RESULTS)
val text = results?.firstOrNull()
if (!text.isNullOrBlank()) {
sendToServer(text)
Log.d(TAG, "Speech recognized: $text")
sendTranscript(text, "voice-assistant")
return
}
}
Log.d(TAG, "Speech recognition cancelled or no result")
finish()
}
}
private fun sendToServer(text: String) {
/**
* Sends the transcribed text to the server's /api/voice-transcript endpoint,
* then launches MainActivity with the target terminal so the floating
* transcript debug opens on the correct session.
*/
private fun sendTranscript(text: String, source: String) {
Thread {
try {
val terminalBase = ServerConfig.terminalBaseUrl(this)
val terminalWs = ServerConfig.terminalWsUrl(this)
val apiBase = ServerConfig.apiBaseUrl(this)
if (terminalBase == null || terminalWs == null) {
if (apiBase == null) {
Log.w(TAG, "No server configured, cannot send transcript")
showToastAndFinish("No server configured")
return@Thread
}
// 1. Create a new terminal session
val createBody = JSONObject().apply {
put("agent", "ejecutor")
put("transcriptSessionId", "__new__")
put("label", text.take(60))
put("command", "ejecutor")
val timestamp = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US)
.format(Date())
val body = JSONObject().apply {
put("text", text)
put("timestamp", timestamp)
put("source", source)
}.toString().toRequestBody("application/json".toMediaType())
val createReq = Request.Builder()
.url("$terminalBase/create-terminal")
.post(createBody)
val request = Request.Builder()
.url("$apiBase/voice-transcript")
.post(body)
.build()
val createResp = client.newCall(createReq).execute()
if (!createResp.isSuccessful) {
showToastAndFinish("Failed to create terminal")
return@Thread
Log.d(TAG, "Sending transcript to $apiBase/voice-transcript")
val response = client.newCall(request).execute()
val responseBody = response.body?.string() ?: ""
if (response.isSuccessful) {
Log.d(TAG, "Server response: $responseBody")
// Parse the ephemeralSessionId from the response
val json = JSONObject(responseBody)
val terminal = json.optString("ephemeralSessionId", "")
// Launch MainActivity with the terminal info to open floating transcript
openMainWithTerminal(terminal, text)
} else {
Log.e(TAG, "Server error ${response.code}: $responseBody")
showToastAndFinish("Server error: ${response.code}")
}
val respJson = JSONObject(createResp.body?.string() ?: "{}")
val sessionId = respJson.optString("ephemeralSessionId", "")
if (sessionId.isEmpty()) {
showToastAndFinish("No session ID returned")
return@Thread
}
// 2. Connect via WebSocket and send the input
val wsUrl = "$terminalWs/ws/terminal?session=$sessionId"
val wsReq = Request.Builder().url(wsUrl).build()
val latch = CountDownLatch(1)
client.newWebSocket(wsReq, object : WebSocketListener() {
override fun onOpen(webSocket: WebSocket, response: Response) {
// Send the text as input
val inputMsg = JSONObject().apply {
put("type", "input")
put("data", text)
}.toString()
webSocket.send(inputMsg)
// Send Enter key after a short delay
Thread.sleep(80)
val enterMsg = JSONObject().apply {
put("type", "input")
put("data", "\r")
}.toString()
webSocket.send(enterMsg)
// Close after sending
Thread.sleep(200)
webSocket.close(1000, "done")
latch.countDown()
}
override fun onFailure(webSocket: WebSocket, t: Throwable, response: Response?) {
Log.e(TAG, "WebSocket failed", t)
latch.countDown()
}
})
latch.await(5, TimeUnit.SECONDS)
showToastAndFinish("Sent: ${text.take(40)}")
} catch (e: Exception) {
Log.e(TAG, "Failed to send command", e)
showToastAndFinish("Error: ${e.message?.take(40)}")
Log.e(TAG, "Failed to send transcript", e)
showToastAndFinish("Error: ${e.message?.take(50)}")
}
}.start()
}
/**
* Opens MainActivity and tells the WebView to open FloatingTranscriptDebug
* on the terminal that received the voice command.
*/
private fun openMainWithTerminal(terminal: String, text: String) {
runOnUiThread {
Toast.makeText(this, "Sent: ${text.take(50)}", Toast.LENGTH_SHORT).show()
val intent = Intent(this, MainActivity::class.java).apply {
action = "com.agentui.desktop.VOICE_TERMINAL"
putExtra("ephemeralSessionId", terminal)
addFlags(Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_CLEAR_TOP)
}
startActivity(intent)
finish()
}
}
private fun showToastAndFinish(message: String) {
runOnUiThread {
Toast.makeText(this, message, Toast.LENGTH_SHORT).show()

View File

@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<shape xmlns:android="http://schemas.android.com/apk/res/android"
android:shape="rectangle">
<solid android:color="#ff171717" />
<corners android:radius="22dp" />
</shape>

View File

@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<shape xmlns:android="http://schemas.android.com/apk/res/android"
android:shape="rectangle">
<solid android:color="#99010101" />
<corners android:radius="22dp" />
</shape>

View File

@@ -0,0 +1,124 @@
<?xml version="1.0" encoding="utf-8"?>
<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
android:layout_width="match_parent"
android:layout_height="match_parent"
android:orientation="vertical"
android:padding="12dp"
android:background="@drawable/face_widget_bg_aod">
<!-- Minimal AOD layout: title + terminal count -->
<TextView
android:id="@+id/fw_aod_title"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:text="Agent UI"
android:textColor="#555588"
android:textSize="11sp"
android:fontFamily="monospace"
android:paddingBottom="4dp" />
<!-- Terminal 1 -->
<LinearLayout
android:id="@+id/fw_aod_terminal_1"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:orientation="horizontal"
android:gravity="center_vertical"
android:paddingTop="1dp"
android:paddingBottom="1dp"
android:visibility="gone">
<TextView
android:id="@+id/fw_aod_dot_1"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="\u25CF"
android:textColor="#444466"
android:textSize="7sp"
android:paddingEnd="4dp" />
<TextView
android:id="@+id/fw_aod_name_1"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:textColor="#777777"
android:textSize="9sp"
android:fontFamily="monospace"
android:maxLines="1"
android:ellipsize="end" />
</LinearLayout>
<!-- Terminal 2 -->
<LinearLayout
android:id="@+id/fw_aod_terminal_2"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:orientation="horizontal"
android:gravity="center_vertical"
android:paddingTop="1dp"
android:paddingBottom="1dp"
android:visibility="gone">
<TextView
android:id="@+id/fw_aod_dot_2"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="\u25CF"
android:textColor="#444466"
android:textSize="7sp"
android:paddingEnd="4dp" />
<TextView
android:id="@+id/fw_aod_name_2"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:textColor="#777777"
android:textSize="9sp"
android:fontFamily="monospace"
android:maxLines="1"
android:ellipsize="end" />
</LinearLayout>
<!-- Terminal 3 -->
<LinearLayout
android:id="@+id/fw_aod_terminal_3"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:orientation="horizontal"
android:gravity="center_vertical"
android:paddingTop="1dp"
android:paddingBottom="1dp"
android:visibility="gone">
<TextView
android:id="@+id/fw_aod_dot_3"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="\u25CF"
android:textColor="#444466"
android:textSize="7sp"
android:paddingEnd="4dp" />
<TextView
android:id="@+id/fw_aod_name_3"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:textColor="#777777"
android:textSize="9sp"
android:fontFamily="monospace"
android:maxLines="1"
android:ellipsize="end" />
</LinearLayout>
<!-- Empty state -->
<TextView
android:id="@+id/fw_aod_empty"
android:layout_width="match_parent"
android:layout_height="match_parent"
android:text="No agents"
android:textColor="#444444"
android:textSize="9sp"
android:fontFamily="monospace"
android:gravity="center" />
</LinearLayout>

View File

@@ -0,0 +1,140 @@
<?xml version="1.0" encoding="utf-8"?>
<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
android:layout_width="match_parent"
android:layout_height="match_parent"
android:orientation="vertical"
android:padding="12dp"
android:background="@drawable/face_widget_bg_dark">
<!-- Title bar -->
<LinearLayout
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:orientation="horizontal"
android:gravity="center_vertical"
android:paddingBottom="6dp">
<TextView
android:id="@+id/fw_title"
android:layout_width="0dp"
android:layout_height="wrap_content"
android:layout_weight="1"
android:text="Agent UI"
android:textColor="#8888FF"
android:textSize="12sp"
android:fontFamily="monospace" />
<TextView
android:id="@+id/fw_status"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:textColor="#888888"
android:textSize="10sp"
android:fontFamily="monospace" />
</LinearLayout>
<!-- Terminal 1 -->
<LinearLayout
android:id="@+id/fw_terminal_1"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:orientation="horizontal"
android:gravity="center_vertical"
android:paddingTop="2dp"
android:paddingBottom="2dp"
android:visibility="gone">
<TextView
android:id="@+id/fw_dot_1"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="\u25CF"
android:textColor="#6b7280"
android:textSize="8sp"
android:paddingEnd="4dp" />
<TextView
android:id="@+id/fw_name_1"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:textColor="#DDDDDD"
android:textSize="10sp"
android:fontFamily="monospace"
android:maxLines="1"
android:ellipsize="end" />
</LinearLayout>
<!-- Terminal 2 -->
<LinearLayout
android:id="@+id/fw_terminal_2"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:orientation="horizontal"
android:gravity="center_vertical"
android:paddingTop="2dp"
android:paddingBottom="2dp"
android:visibility="gone">
<TextView
android:id="@+id/fw_dot_2"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="\u25CF"
android:textColor="#6b7280"
android:textSize="8sp"
android:paddingEnd="4dp" />
<TextView
android:id="@+id/fw_name_2"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:textColor="#DDDDDD"
android:textSize="10sp"
android:fontFamily="monospace"
android:maxLines="1"
android:ellipsize="end" />
</LinearLayout>
<!-- Terminal 3 -->
<LinearLayout
android:id="@+id/fw_terminal_3"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:orientation="horizontal"
android:gravity="center_vertical"
android:paddingTop="2dp"
android:paddingBottom="2dp"
android:visibility="gone">
<TextView
android:id="@+id/fw_dot_3"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="\u25CF"
android:textColor="#6b7280"
android:textSize="8sp"
android:paddingEnd="4dp" />
<TextView
android:id="@+id/fw_name_3"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:textColor="#DDDDDD"
android:textSize="10sp"
android:fontFamily="monospace"
android:maxLines="1"
android:ellipsize="end" />
</LinearLayout>
<!-- Empty state -->
<TextView
android:id="@+id/fw_empty"
android:layout_width="match_parent"
android:layout_height="match_parent"
android:text="No agents active"
android:textColor="#666666"
android:textSize="10sp"
android:fontFamily="monospace"
android:gravity="center" />
</LinearLayout>

View File

@@ -0,0 +1,64 @@
<?xml version="1.0" encoding="utf-8"?>
<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
android:id="@+id/item_root"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:orientation="vertical"
android:paddingTop="3dp"
android:paddingBottom="3dp"
android:background="?android:attr/selectableItemBackground">
<!-- Header: dot + agent name + status + badges -->
<LinearLayout
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:orientation="horizontal"
android:gravity="center_vertical">
<!-- Status dot -->
<TextView
android:id="@+id/item_dot"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="\u25CF"
android:textColor="#6b7280"
android:textSize="8sp"
android:paddingEnd="4dp" />
<!-- Agent name + status -->
<TextView
android:id="@+id/item_name"
android:layout_width="0dp"
android:layout_height="wrap_content"
android:layout_weight="1"
android:textColor="#DDDDDD"
android:textSize="10sp"
android:fontFamily="monospace"
android:maxLines="1"
android:ellipsize="end" />
<!-- Hook badges -->
<TextView
android:id="@+id/item_badges"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:textColor="#888888"
android:textSize="9sp"
android:fontFamily="monospace"
android:paddingStart="4dp" />
</LinearLayout>
<!-- Label / last user prompt -->
<TextView
android:id="@+id/item_label"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:textColor="#9999CC"
android:textSize="10sp"
android:fontFamily="monospace"
android:maxLines="2"
android:ellipsize="end"
android:paddingStart="12dp"
android:paddingTop="1dp" />
</LinearLayout>

View File

@@ -4,67 +4,57 @@
android:layout_width="match_parent"
android:layout_height="match_parent"
android:orientation="vertical"
android:padding="12dp"
android:padding="10dp"
android:background="#DD1A1A2E">
<TextView
android:id="@+id/widget_title"
<!-- Title bar -->
<LinearLayout
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:text="Agent UI Transcript"
android:textColor="#8888FF"
android:textSize="12sp"
android:fontFamily="monospace"
android:paddingBottom="4dp" />
android:orientation="horizontal"
android:gravity="center_vertical"
android:paddingBottom="4dp">
<TextView
android:id="@+id/msg1"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:textColor="#CCCCCC"
android:textSize="11sp"
android:fontFamily="monospace"
android:maxLines="1"
android:ellipsize="end" />
<TextView
android:id="@+id/widget_title"
android:layout_width="0dp"
android:layout_height="wrap_content"
android:layout_weight="1"
android:text="Agent UI"
android:textColor="#8888FF"
android:textSize="11sp"
android:fontFamily="monospace" />
<TextView
android:id="@+id/msg2"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:textColor="#CCCCCC"
android:textSize="11sp"
android:fontFamily="monospace"
android:maxLines="1"
android:ellipsize="end" />
<TextView
android:id="@+id/btn_refresh"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="\u21BB"
android:textColor="#8888FF"
android:textSize="14sp"
android:paddingStart="8dp"
android:paddingEnd="4dp"
android:background="?android:attr/selectableItemBackground" />
</LinearLayout>
<TextView
android:id="@+id/msg3"
<!-- Terminal list (dynamic, scrollable) -->
<ListView
android:id="@+id/terminal_list"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:textColor="#CCCCCC"
android:textSize="11sp"
android:fontFamily="monospace"
android:maxLines="1"
android:ellipsize="end" />
android:layout_height="match_parent"
android:divider="@null"
android:dividerHeight="0dp"
android:scrollbars="none" />
<!-- Fallback when empty -->
<TextView
android:id="@+id/msg4"
android:id="@+id/empty_view"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:textColor="#CCCCCC"
android:textSize="11sp"
android:layout_height="match_parent"
android:text="No terminals open"
android:textColor="#666666"
android:textSize="10sp"
android:fontFamily="monospace"
android:maxLines="1"
android:ellipsize="end" />
<TextView
android:id="@+id/msg5"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:textColor="#CCCCCC"
android:textSize="11sp"
android:fontFamily="monospace"
android:maxLines="1"
android:ellipsize="end" />
android:gravity="center" />
</LinearLayout>

View File

@@ -0,0 +1,7 @@
{
"agent_ui_transcript": {
"menuInSetting": 1,
"labelResNameInSetting": "face_widget_label",
"actionDetailSetting": "com.agentui.desktop.FACE_WIDGET_SETTINGS"
}
}

View File

@@ -2,4 +2,5 @@
<string name="app_name">Agent UI</string>
<string name="main_activity_title">Agent UI</string>
<string name="widget_description">Shows recent transcript messages from Agent UI</string>
<string name="face_widget_label">Agent UI Terminals</string>
</resources>

View File

@@ -0,0 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<recognition-service
xmlns:android="http://schemas.android.com/apk/res/android"
android:settingsActivity="com.agentui.desktop.MainActivity" />

View File

@@ -1,9 +1,9 @@
<?xml version="1.0" encoding="utf-8"?>
<appwidget-provider xmlns:android="http://schemas.android.com/apk/res/android"
android:minWidth="250dp"
android:minHeight="110dp"
android:minHeight="140dp"
android:updatePeriodMillis="1800000"
android:initialLayout="@layout/widget_transcript"
android:resizeMode="horizontal|vertical"
android:widgetCategory="home_screen"
android:widgetCategory="home_screen|keyguard"
android:description="@string/widget_description" />

View File

@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="utf-8"?>
<voice-interaction-service xmlns:android="http://schemas.android.com/apk/res/android"
android:sessionService="com.agentui.desktop.AgentVoiceInteractionSessionService"
android:recognitionService="com.agentui.desktop.AgentRecognitionService"
android:supportsAssist="true"
android:supportsLaunchVoiceAssistFromKeyguard="true"
android:supportsLocalInteraction="true"
android:settingsActivity="com.agentui.desktop.MainActivity" />