nucleo000/agent-ui
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: voice assistant integration, PiP window fixes, widget improvements and pixel art scrollbar

Browse Source 
- Android voice assistant: RecognitionService, VoiceInteractionSession with startAssistantActivity, es-HN speech recognition
- Voice transcript sent to first alive terminal via WebSocket, opens FloatingTranscriptDebug on correct session
- PiP window: fix close button using getCurrentWebviewWindow(), add mini/restore toggle, remove alwaysOnTop
- Add webview-close and window-destroy permissions to capabilities
- Pixel art ocean scrollbar on /transcript-debug respecting scroll nav mode settings
- Widget improvements: terminal list service, input widget provider, updated layouts
This commit is contained in:
josedario87
2026-02-23 22:35:58 -06:00
parent f6ec5ba5de
commit c46b1283d1
14 changed files with 837 additions and 252 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src-tauri/capabilities/default.json
View File

@@ -2,7 +2,7 @@
"$schema": "https://raw.githubusercontent.com/tauri-apps/tauri/dev/crates/tauri-utils/schema.json",
"identifier": "default",
"description": "Default permissions for Agent UI",
"windows": ["main"],
"windows": ["main", "pip-terminal"],
"permissions": [
"core:default",
{
@@ -17,10 +17,20 @@
"clipboard-manager:default",
"dialog:default",
"core:window:default",
"core:window:allow-create",
"core:window:allow-minimize",
"core:window:allow-maximize",
"core:window:allow-unmaximize",
"core:window:allow-toggle-maximize",
"core:window:allow-close"
"core:window:allow-close",
"core:window:allow-set-always-on-top",
"core:window:allow-set-size",
"core:window:allow-set-position",
"core:window:allow-set-focus",
"core:window:allow-set-decorations",
"core:webview:default",
"core:webview:allow-create-webview-window",
"core:webview:allow-webview-close",
"core:window:allow-destroy"
]
}

18
src-tauri/gen/android/app/src/main/AndroidManifest.xml
View File

@@ -9,7 +9,8 @@
android:icon="@mipmap/ic_launcher"
android:label="@string/app_name"
android:theme="@style/Theme.agent_ui"
android:usesCleartextTraffic="${usesCleartextTraffic}">
android:usesCleartextTraffic="${usesCleartextTraffic}"
android:networkSecurityConfig="@xml/network_security_config">
<activity
android:configChanges="orientation|keyboardHidden|keyboard|screenSize|locale|smallestScreenSize|screenLayout|uiMode"
android:launchMode="singleTask"
@@ -37,6 +38,21 @@
android:resource="@xml/transcript_widget_info" />
</receiver>
<!-- Input Widget -->
<receiver
android:name=".InputWidgetProvider"
android:exported="false">
<intent-filter>
<action android:name="android.appwidget.action.APPWIDGET_UPDATE" />
<action android:name="com.agentui.desktop.INPUT_WIDGET_SEND" />
<action android:name="com.agentui.desktop.INPUT_WIDGET_MIC" />
<action android:name="com.agentui.desktop.INPUT_WIDGET_CANCEL" />
</intent-filter>
<meta-data
android:name="android.appwidget.provider"
android:resource="@xml/input_widget_info" />
</receiver>
<!-- Voice Command / Share / Assist Activity -->
<activity
android:name=".VoiceCommandActivity"

217
src-tauri/gen/android/app/src/main/java/com/agentui/desktop/InputWidgetProvider.kt Normal file
View File

@@ -0,0 +1,217 @@
package com.agentui.desktop
import android.app.PendingIntent
import android.appwidget.AppWidgetManager
import android.appwidget.AppWidgetProvider
import android.content.ComponentName
import android.content.Context
import android.content.Intent
import android.net.Uri
import android.util.Log
import android.widget.RemoteViews
class InputWidgetProvider : AppWidgetProvider() {
companion object {
private const val TAG = "InputWidget"
const val ACTION_SEND = "com.agentui.desktop.INPUT_WIDGET_SEND"
const val ACTION_MIC = "com.agentui.desktop.INPUT_WIDGET_MIC"
const val ACTION_CANCEL = "com.agentui.desktop.INPUT_WIDGET_CANCEL"
private const val STATE_IDLE = 0
private const val STATE_SENDING = 1
private const val STATE_PROCESSING = 2
private const val STATE_DONE = 3
private const val STATE_ERROR = 4
private const val PREFS_NAME = "input_widget_prefs"
private const val KEY_STATE = "hook_state"
// Colors
private const val COLOR_IDLE = 0x8040C040.toInt()
private const val COLOR_SENDING = 0xFF60a5fa.toInt()
private const val COLOR_PROCESSING = 0xFFfbbf24.toInt()
private const val COLOR_DONE = 0xFF4ade80.toInt()
private const val COLOR_ERROR = 0xFFf87171.toInt()
private const val COLOR_BTN_ACTIVE = 0xEEFFFFFF.toInt()
private const val COLOR_BTN_NORMAL = 0xAAFFFFFF.toInt()
private const val COLOR_BTN_DIM = 0x50FFFFFF.toInt()
private const val COLOR_TEXT_DIM = 0x80FFFFFF.toInt()
private val STATUS_LABELS = mapOf(
STATE_IDLE to "idle",
STATE_SENDING to "sending...",
STATE_PROCESSING to "processing...",
STATE_DONE to "done",
STATE_ERROR to "error"
)
}
override fun onUpdate(
context: Context,
appWidgetManager: AppWidgetManager,
appWidgetIds: IntArray
) {
Log.d(TAG, "onUpdate called for ${appWidgetIds.size} widgets")
for (id in appWidgetIds) {
updateWidget(context, appWidgetManager, id)
}
}
override fun onReceive(context: Context, intent: Intent) {
super.onReceive(context, intent)
Log.d(TAG, "onReceive: action=${intent.action}")
when (intent.action) {
ACTION_SEND -> {
val next = when (getState(context)) {
STATE_IDLE -> STATE_SENDING
STATE_SENDING -> STATE_PROCESSING
STATE_PROCESSING -> STATE_DONE
else -> STATE_IDLE
}
Log.d(TAG, "SEND: state -> $next")
setState(context, next)
refreshAll(context)
}
ACTION_MIC -> {
val next = if (getState(context) == STATE_IDLE) STATE_SENDING else STATE_IDLE
Log.d(TAG, "MIC: state -> $next")
setState(context, next)
refreshAll(context)
}
ACTION_CANCEL -> {
val current = getState(context)
val next = when (current) {
STATE_SENDING, STATE_PROCESSING -> STATE_ERROR
else -> STATE_IDLE
}
Log.d(TAG, "CANCEL: state -> $next")
setState(context, next)
refreshAll(context)
}
}
}
private fun updateWidget(context: Context, mgr: AppWidgetManager, id: Int) {
val views = RemoteViews(context.packageName, R.layout.widget_input)
val state = getState(context)
// Open app — only the ↗ button
val appIntent = Intent(context, MainActivity::class.java).apply {
flags = Intent.FLAG_ACTIVITY_NEW_TASK
}
val appPending = PendingIntent.getActivity(
context, 100, appIntent,
PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE
)
views.setOnClickPendingIntent(R.id.btn_open_app, appPending)
// Input field → send
views.setOnClickPendingIntent(R.id.input_field,
makeBroadcast(context, ACTION_SEND, "input_field"))
// Buttons — each with unique data URI to avoid PendingIntent deduplication
views.setOnClickPendingIntent(R.id.btn_send,
makeBroadcast(context, ACTION_SEND, "btn_send"))
views.setOnClickPendingIntent(R.id.btn_mic,
makeBroadcast(context, ACTION_MIC, "btn_mic"))
views.setOnClickPendingIntent(R.id.btn_cancel,
makeBroadcast(context, ACTION_CANCEL, "btn_cancel"))
applyState(views, state)
mgr.updateAppWidget(id, views)
}
/**
* Create a unique PendingIntent per button using data URI differentiation.
* This prevents Android from deduplicating PendingIntents that share the same action.
*/
private fun makeBroadcast(context: Context, action: String, tag: String): PendingIntent {
val intent = Intent(action).apply {
component = ComponentName(context, InputWidgetProvider::class.java)
data = Uri.parse("agentui://input-widget/$tag")
}
return PendingIntent.getBroadcast(
context, 0, intent,
PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_MUTABLE
)
}
private fun applyState(views: RemoteViews, state: Int) {
val stateColor = when (state) {
STATE_IDLE -> COLOR_IDLE
STATE_SENDING -> COLOR_SENDING
STATE_PROCESSING -> COLOR_PROCESSING
STATE_DONE -> COLOR_DONE
STATE_ERROR -> COLOR_ERROR
else -> COLOR_IDLE
}
// Status dot
val dotGlyph = when (state) {
STATE_SENDING, STATE_PROCESSING -> "\u25CC"
else -> "\u25CF"
}
views.setTextViewText(R.id.input_status_dot, dotGlyph)
views.setTextColor(R.id.input_status_dot, stateColor)
// Status text
views.setTextViewText(R.id.input_status_text, STATUS_LABELS[state] ?: "idle")
views.setTextColor(R.id.input_status_text, stateColor)
// Input field prompt
val prompt = when (state) {
STATE_IDLE -> "$ _"
STATE_SENDING -> "$ sending..."
STATE_PROCESSING -> "$ processing..."
STATE_DONE -> "$ done \u2713"
STATE_ERROR -> "$ error \u26A0"
else -> "$ _"
}
views.setTextViewText(R.id.input_field, prompt)
views.setTextColor(R.id.input_field, when (state) {
STATE_IDLE -> COLOR_TEXT_DIM
else -> stateColor
})
// Send button
views.setTextColor(R.id.btn_send, when (state) {
STATE_IDLE -> COLOR_BTN_ACTIVE
STATE_DONE, STATE_ERROR -> COLOR_BTN_NORMAL
else -> COLOR_BTN_DIM
})
// Mic button
views.setTextColor(R.id.btn_mic, when (state) {
STATE_IDLE -> COLOR_BTN_NORMAL
STATE_SENDING -> COLOR_SENDING
else -> COLOR_BTN_DIM
})
// Cancel button
views.setTextColor(R.id.btn_cancel, when (state) {
STATE_SENDING, STATE_PROCESSING -> COLOR_ERROR
else -> COLOR_BTN_DIM
})
}
private fun getState(context: Context): Int =
context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE)
.getInt(KEY_STATE, STATE_IDLE)
private fun setState(context: Context, state: Int) {
context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE)
.edit().putInt(KEY_STATE, state).apply()
}
private fun refreshAll(context: Context) {
val mgr = AppWidgetManager.getInstance(context)
val ids = mgr.getAppWidgetIds(ComponentName(context, InputWidgetProvider::class.java))
Log.d(TAG, "refreshAll: ${ids.size} widgets")
for (id in ids) updateWidget(context, mgr, id)
}
}

269
src-tauri/gen/android/app/src/main/java/com/agentui/desktop/MainActivity.kt
View File

@@ -13,6 +13,7 @@ import android.os.Build
import android.os.Bundle
import android.util.Log
import android.util.Rational
import android.webkit.JavascriptInterface
import android.webkit.WebView
import androidx.activity.enableEdgeToEdge
import androidx.core.view.ViewCompat
@@ -21,6 +22,7 @@ import androidx.core.view.WindowInsetsCompat
class MainActivity : TauriActivity() {
companion object {
private const val TAG = "AgentUI"
private const val ACTION_PIP_MIC = "com.agentui.desktop.PIP_MIC"
private const val ACTION_PIP_EXPAND = "com.agentui.desktop.PIP_EXPAND"
private const val PIP_REQUEST_CODE_MIC = 2001
@@ -34,8 +36,7 @@ class MainActivity : TauriActivity() {
override fun onReceive(context: Context, intent: Intent) {
when (intent.action) {
ACTION_PIP_MIC -> {
Log.d("AgentUI", "PiP mic button pressed")
// Launch voice command from PiP
Log.d(TAG, "PiP mic button pressed")
val voiceIntent = Intent(context, VoiceCommandActivity::class.java).apply {
action = Intent.ACTION_VOICE_COMMAND
addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
@@ -43,8 +44,7 @@ class MainActivity : TauriActivity() {
startActivity(voiceIntent)
}
ACTION_PIP_EXPAND -> {
Log.d("AgentUI", "PiP expand button pressed")
// Bring app to foreground full-screen
Log.d(TAG, "PiP expand button pressed")
val expandIntent = Intent(context, MainActivity::class.java).apply {
addFlags(Intent.FLAG_ACTIVITY_REORDER_TO_FRONT or Intent.FLAG_ACTIVITY_SINGLE_TOP)
}
@@ -62,6 +62,7 @@ class MainActivity : TauriActivity() {
handleWidgetIntent(intent)
handleVoiceIntent(intent)
registerPipReceiver()
injectJsBridge()
}
override fun onDestroy() {
@@ -69,84 +70,6 @@ class MainActivity : TauriActivity() {
try { unregisterReceiver(pipReceiver) } catch (_: Exception) {}
}
private fun registerPipReceiver() {
val filter = IntentFilter().apply {
addAction(ACTION_PIP_MIC)
addAction(ACTION_PIP_EXPAND)
}
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
registerReceiver(pipReceiver, filter, RECEIVER_NOT_EXPORTED)
} else {
registerReceiver(pipReceiver, filter)
}
}
/**
* Enter PiP when user leaves the app (home button / swipe).
*/
override fun onUserLeaveHint() {
super.onUserLeaveHint()
// Only enter PiP via voice command flow, not on regular minimize
}
override fun onPictureInPictureModeChanged(
isInPictureInPictureMode: Boolean,
newConfig: Configuration
) {
super.onPictureInPictureModeChanged(isInPictureInPictureMode, newConfig)
Log.d("AgentUI", "PiP mode changed: $isInPictureInPictureMode")
}
private fun enterPipIfSupported() {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
try {
val actions = buildPipActions()
val builder = PictureInPictureParams.Builder()
.setAspectRatio(Rational(9, 16))
.setActions(actions)
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
builder.setAutoEnterEnabled(false)
builder.setSeamlessResizeEnabled(true)
}
enterPictureInPictureMode(builder.build())
} catch (e: Exception) {
Log.w("AgentUI", "Failed to enter PiP: $e")
}
}
}
private fun buildPipActions(): List<RemoteAction> {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) return emptyList()
val actions = mutableListOf<RemoteAction>()
// Mic button — launch voice command
val micIntent = PendingIntent.getBroadcast(
this, PIP_REQUEST_CODE_MIC,
Intent(ACTION_PIP_MIC).setPackage(packageName),
PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE
)
actions.add(RemoteAction(
Icon.createWithResource(this, android.R.drawable.ic_btn_speak_now),
"Voice", "Send voice command",
micIntent
))
// Expand button — bring app to foreground
val expandIntent = PendingIntent.getBroadcast(
this, PIP_REQUEST_CODE_EXPAND,
Intent(ACTION_PIP_EXPAND).setPackage(packageName),
PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE
)
actions.add(RemoteAction(
Icon.createWithResource(this, android.R.drawable.ic_menu_view),
"Open", "Open full app",
expandIntent
))
return actions
}
override fun onNewIntent(intent: android.content.Intent) {
super.onNewIntent(intent)
handleWidgetIntent(intent)
@@ -162,36 +85,121 @@ class MainActivity : TauriActivity() {
}
}
private fun handleWidgetIntent(intent: android.content.Intent?) {
val terminalIndex = intent?.getIntExtra("terminalIndex", -1) ?: -1
if (terminalIndex > 0) {
val route = "/transcript-debug/$terminalIndex"
Log.d("AgentUI", "Widget click → navigate to $route")
pendingRoute = route
navigateWebView(route)
}
override fun onPictureInPictureModeChanged(
isInPictureInPictureMode: Boolean,
newConfig: Configuration
) {
super.onPictureInPictureModeChanged(isInPictureInPictureMode, newConfig)
Log.d(TAG, "PiP mode changed: $isInPictureInPictureMode")
}
private fun handleVoiceIntent(intent: android.content.Intent?) {
if (intent?.action == "com.agentui.desktop.VOICE_TERMINAL") {
val sessionId = intent.getStringExtra("ephemeralSessionId") ?: return
if (sessionId.isNotEmpty()) {
Log.d("AgentUI", "Voice intent → open terminal $sessionId")
pendingVoiceTerminal = sessionId
// Don't call openVoiceTerminal here — WebView may not exist yet.
// Instead, poll until the WebView is ready.
pollForWebViewAndOpenTerminal(sessionId, 0)
// ── PiP ──
private fun enterPipIfSupported() {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
try {
val actions = buildPipActions()
val builder = PictureInPictureParams.Builder()
.setAspectRatio(Rational(9, 16))
.setActions(actions)
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
builder.setSeamlessResizeEnabled(true)
}
enterPictureInPictureMode(builder.build())
} catch (e: Exception) {
Log.w(TAG, "Failed to enter PiP: $e")
}
}
}
/**
* Retry finding the WebView up to ~3 seconds (15 attempts x 200ms).
* Once found, inject JS to open floating transcript and enter PiP.
*/
private fun pollForWebViewAndOpenTerminal(ephemeralSessionId: String, attempt: Int) {
private fun buildPipActions(): List<RemoteAction> {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) return emptyList()
val actions = mutableListOf<RemoteAction>()
val micIntent = PendingIntent.getBroadcast(
this, PIP_REQUEST_CODE_MIC,
Intent(ACTION_PIP_MIC).setPackage(packageName),
PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE
)
actions.add(RemoteAction(
Icon.createWithResource(this, android.R.drawable.ic_btn_speak_now),
"Voice", "Send voice command",
micIntent
))
val expandIntent = PendingIntent.getBroadcast(
this, PIP_REQUEST_CODE_EXPAND,
Intent(ACTION_PIP_EXPAND).setPackage(packageName),
PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE
)
actions.add(RemoteAction(
Icon.createWithResource(this, android.R.drawable.ic_menu_view),
"Open", "Open full app",
expandIntent
))
return actions
}
private fun registerPipReceiver() {
val filter = IntentFilter().apply {
addAction(ACTION_PIP_MIC)
addAction(ACTION_PIP_EXPAND)
}
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
registerReceiver(pipReceiver, filter, RECEIVER_NOT_EXPORTED)
} else {
registerReceiver(pipReceiver, filter)
}
}
// ── JS Bridge (exposes AgentUI.enterPip() to the WebView) ──
private fun injectJsBridge() {
window.decorView.postDelayed({
try {
val webView = findWebView(window.decorView)
if (webView != null) {
webView.addJavascriptInterface(JsBridge(), "AgentUI")
Log.d(TAG, "JS bridge injected (AgentUI.enterPip)")
} else {
// Retry once more after delay
window.decorView.postDelayed({
val wv = findWebView(window.decorView)
if (wv != null) {
wv.addJavascriptInterface(JsBridge(), "AgentUI")
Log.d(TAG, "JS bridge injected on retry")
}
}, 1000)
}
} catch (e: Exception) {
Log.w(TAG, "Failed to inject JS bridge: $e")
}
}, 500)
}
inner class JsBridge {
@JavascriptInterface
fun enterPip() {
runOnUiThread { enterPipIfSupported() }
}
}
// ── Voice intent → navigate to transcript page ──
private fun handleVoiceIntent(intent: android.content.Intent?) {
if (intent?.action == "com.agentui.desktop.VOICE_TERMINAL") {
val sessionId = intent.getStringExtra("ephemeralSessionId") ?: ""
Log.d(TAG, "Voice intent → navigate to transcript-debug, terminal=$sessionId")
pendingVoiceTerminal = sessionId
pollForWebViewAndNavigate(sessionId, 0)
}
}
private fun pollForWebViewAndNavigate(ephemeralSessionId: String, attempt: Int) {
if (attempt > 15) {
Log.w("AgentUI", "Gave up waiting for WebView after ${attempt} attempts")
Log.w(TAG, "Gave up waiting for WebView after $attempt attempts")
pendingVoiceTerminal = null
return
}
@@ -199,32 +207,43 @@ class MainActivity : TauriActivity() {
val webView = try { findWebView(window.decorView) } catch (_: Exception) { null }
if (webView != null) {
// Navigate to transcript-debug page and tell it which terminal to focus
val js = "window.__VOICE_OPEN_TERMINAL__ && window.__VOICE_OPEN_TERMINAL__('$ephemeralSessionId')"
webView.evaluateJavascript(js, null)
pendingVoiceTerminal = null
Log.d("AgentUI", "Voice terminal JS dispatched (attempt $attempt): $ephemeralSessionId")
// Enter PiP after the WebView has time to render
webView.postDelayed({ enterPipIfSupported() }, 500)
Log.d(TAG, "Voice navigate dispatched (attempt $attempt): $ephemeralSessionId")
} else {
Log.d("AgentUI", "WebView not ready (attempt $attempt), retrying in 200ms")
Log.d(TAG, "WebView not ready (attempt $attempt), retrying in 200ms")
window.decorView.postDelayed({
pollForWebViewAndOpenTerminal(ephemeralSessionId, attempt + 1)
pollForWebViewAndNavigate(ephemeralSessionId, attempt + 1)
}, 200)
}
}
// ── Widget intent ──
private fun handleWidgetIntent(intent: android.content.Intent?) {
val terminalIndex = intent?.getIntExtra("terminalIndex", -1) ?: -1
if (terminalIndex > 0) {
val route = "/transcript-debug/$terminalIndex"
Log.d(TAG, "Widget click → navigate to $route")
pendingRoute = route
navigateWebView(route)
}
}
// ── WebView helpers ──
private fun navigateWebView(route: String) {
try {
val decorView = window.decorView
val webView = findWebView(decorView)
val webView = findWebView(window.decorView)
if (webView != null) {
val js = "window.__WIDGET_NAVIGATE__ && window.__WIDGET_NAVIGATE__('$route') || (window.location.href = '$route')"
webView.evaluateJavascript(js, null)
pendingRoute = null
}
} catch (e: Exception) {
Log.w("AgentUI", "Failed to navigate WebView: $e")
Log.w(TAG, "Failed to navigate WebView: $e")
}
}
@@ -233,14 +252,10 @@ class MainActivity : TauriActivity() {
ViewCompat.setOnApplyWindowInsetsListener(decorView) { view, insets ->
val systemBars = insets.getInsets(WindowInsetsCompat.Type.systemBars())
val density = resources.displayMetrics.density
val topPx = systemBars.top
val bottomPx = systemBars.bottom
val leftPx = systemBars.left
val rightPx = systemBars.right
val topDp = topPx / density
val bottomDp = bottomPx / density
val leftDp = leftPx / density
val rightDp = rightPx / density
val topDp = systemBars.top / density
val bottomDp = systemBars.bottom / density
val leftDp = systemBars.left / density
val rightDp = systemBars.right / density
val js = """
document.documentElement.style.setProperty('--sat', '${topDp}px');
document.documentElement.style.setProperty('--sab', '${bottomDp}px');
@@ -249,13 +264,11 @@ class MainActivity : TauriActivity() {
""".trimIndent()
try {
val webView = view.findViewWithTag<WebView>("tauri_webview")
?: view.findViewById<WebView>(android.R.id.content)?.let {
findWebView(it)
}
?: view.findViewById<WebView>(android.R.id.content)?.let { findWebView(it) }
webView?.evaluateJavascript(js, null)
Log.d("AgentUI", "Injected safe-area: top=${topDp}dp bottom=${bottomDp}dp")
Log.d(TAG, "Injected safe-area: top=${topDp}dp bottom=${bottomDp}dp")
} catch (e: Exception) {
Log.w("AgentUI", "Failed to inject safe-area insets: $e")
Log.w(TAG, "Failed to inject safe-area insets: $e")
}
ViewCompat.onApplyWindowInsets(view, insets)
}
@@ -274,6 +287,6 @@ class MainActivity : TauriActivity() {
private fun syncServerUrlToPrefs() {
val url = ServerConfig.getServerUrl(this)
Log.d("AgentUI", "syncServerUrlToPrefs: resolved url=$url")
Log.d(TAG, "syncServerUrlToPrefs: resolved url=$url")
}
}

130
src-tauri/gen/android/app/src/main/java/com/agentui/desktop/TerminalListWidgetService.kt
View File

@@ -43,14 +43,33 @@ class TerminalListFactory(private val context: Context) : RemoteViewsService.Rem
private const val ICON_OK = "\u2713" // ✓
private const val ICON_ERROR = "\u26A0" // ⚠
private const val COLOR_NORMAL = 0xFF8888FF.toInt()
private const val COLOR_NORMAL = 0xAAFFFFFF.toInt()
private const val COLOR_LOADING = 0xFF60a5fa.toInt()
private const val COLOR_OK = 0xFF4ade80.toInt()
private const val COLOR_ERROR = 0xFFf87171.toInt()
// Terminal-style status labels shown after agent name
private val STATUS_LABEL = mapOf(
"idle" to "idle",
"thinking" to "thinking...",
"reading" to "reading",
"writing" to "writing",
"toolUse" to "exec",
"permissionRequest" to "await",
"interrupted" to "SIGINT",
"error" to "ERR",
"sessionStart" to "init",
"sessionEnd" to "exit",
"closed" to "exit 0"
)
private const val MAX_RETRIES = 3
private const val RETRY_DELAY_MS = 1500L
private val client = OkHttpClient.Builder()
.connectTimeout(8, TimeUnit.SECONDS)
.readTimeout(8, TimeUnit.SECONDS)
.connectTimeout(5, TimeUnit.SECONDS)
.readTimeout(5, TimeUnit.SECONDS)
.retryOnConnectionFailure(true)
.build()
private val STATUS_COLORS = mapOf(
@@ -73,10 +92,15 @@ class TerminalListFactory(private val context: Context) : RemoteViewsService.Rem
private var items = listOf<TerminalItem>()
private val mainHandler = Handler(Looper.getMainLooper())
private var pendingReset: Runnable? = null
override fun onCreate() {}
override fun onDataSetChanged() {
// Cancel any pending reset from a previous refresh cycle
pendingReset?.let { mainHandler.removeCallbacks(it) }
pendingReset = null
setRefreshButton(ICON_LOADING, COLOR_LOADING)
val result = fetchTerminals()
@@ -105,15 +129,25 @@ class TerminalListFactory(private val context: Context) : RemoteViewsService.Rem
if (position >= items.size) return views
val item = items[position]
// Status dot color
views.setTextColor(R.id.item_dot, item.statusColor)
// Terminal-style: "$ agent [status]"
val statusLabel = if (item.alive) item.status else "closed"
views.setTextViewText(R.id.item_name, "T${item.terminalIndex} ${item.agent} $statusLabel")
val termLabel = STATUS_LABEL[statusLabel] ?: statusLabel
views.setTextViewText(
R.id.item_name,
"$ ${item.agent} [$termLabel]"
)
// Badges (compact)
views.setTextViewText(R.id.item_badges, item.hookBadges)
// Always show the registry label (unique per terminal)
views.setTextViewText(R.id.item_label, item.label)
// Combine label + lastUserPrompt to show maximum content
val parts = mutableListOf<String>()
if (item.label.isNotEmpty()) parts.add("> ${item.label}")
if (item.lastUserPrompt.isNotEmpty()) parts.add("$ ${item.lastUserPrompt}")
views.setTextViewText(R.id.item_label, parts.joinToString("\n"))
val fillIntent = Intent().apply {
putExtra("terminalIndex", item.terminalIndex)
@@ -141,6 +175,16 @@ class TerminalListFactory(private val context: Context) : RemoteViewsService.Rem
views.setTextViewText(R.id.btn_refresh, icon)
views.setTextColor(R.id.btn_refresh, color)
// Update status dot indicator
val statusDot = when (color) {
COLOR_LOADING -> "◌"
COLOR_OK -> "●"
COLOR_ERROR -> "●"
else -> "●"
}
views.setTextViewText(R.id.widget_status_bar, statusDot)
views.setTextColor(R.id.widget_status_bar, color)
val mgr = AppWidgetManager.getInstance(context)
val ids = mgr.getAppWidgetIds(
ComponentName(context, TranscriptWidgetProvider::class.java)
@@ -153,34 +197,73 @@ class TerminalListFactory(private val context: Context) : RemoteViewsService.Rem
/**
* Schedule resetting the button back to normal after a delay.
* Cancels any pending reset to prevent flickering from overlapping refreshes.
*/
private fun scheduleResetButton(delayMs: Long) {
mainHandler.postDelayed({
setRefreshButton(ICON_NORMAL, COLOR_NORMAL)
}, delayMs)
pendingReset?.let { mainHandler.removeCallbacks(it) }
val runnable = Runnable { setRefreshButton(ICON_NORMAL, COLOR_NORMAL) }
pendingReset = runnable
mainHandler.postDelayed(runnable, delayMs)
}
// ── Data fetching ──
/**
* Returns list on success, null on error.
* Retries up to MAX_RETRIES on transient failures.
* Items keep the same order as the terminal registry (T1, T2, T3...)
* so the index maps directly to /transcript-debug/:terminalIndex
*/
private fun fetchTerminals(): List<TerminalItem>? {
val apiBase = ServerConfig.apiBaseUrl(context) ?: return emptyList()
val apiBase = ServerConfig.apiBaseUrl(context)
if (apiBase == null) {
Log.w(TAG, "No server URL configured")
return null // Show error, not empty success
}
try {
val url = "$apiBase/session-state"
val req = Request.Builder().url(url).build()
val resp = client.newCall(req).execute()
if (!resp.isSuccessful) return null
var lastException: Exception? = null
val json = JSONObject(resp.body?.string() ?: "{}")
for (attempt in 1..MAX_RETRIES) {
try {
val result = doFetch(apiBase)
if (result != null) return result
// null = HTTP error, retry
Log.w(TAG, "Fetch attempt $attempt: HTTP error, retrying...")
} catch (e: Exception) {
lastException = e
Log.w(TAG, "Fetch attempt $attempt failed: ${e.message}")
}
if (attempt < MAX_RETRIES) {
try { Thread.sleep(RETRY_DELAY_MS) } catch (_: InterruptedException) { break }
}
}
Log.w(TAG, "All $MAX_RETRIES fetch attempts failed", lastException)
return null
}
/**
* Single fetch attempt. Returns list on success, null on HTTP error.
* Throws on network/parse errors. Always closes the response body.
*/
private fun doFetch(apiBase: String): List<TerminalItem>? {
val url = "$apiBase/session-state"
val req = Request.Builder().url(url).build()
val resp = client.newCall(req).execute()
return resp.use { response ->
if (!response.isSuccessful) {
Log.w(TAG, "HTTP ${response.code} from $url")
return@use null
}
val body = response.body?.string() ?: "{}"
val json = JSONObject(body)
val registry = json.optJSONArray("registry")
val agents = json.optJSONObject("agents")
if (registry == null || registry.length() == 0) return emptyList()
if (registry == null || registry.length() == 0) return@use emptyList()
val result = mutableListOf<TerminalItem>()
@@ -190,7 +273,7 @@ class TerminalListFactory(private val context: Context) : RemoteViewsService.Rem
val ephId = entry.optString("ephemeralSessionId", "")
val label = entry.optString("label", "")
val alive = entry.optBoolean("alive", false)
val terminalIndex = i + 1 // 1-based, maps to /transcript-debug/:terminalIndex
val terminalIndex = i + 1
val agentState = agents?.optJSONObject(agentName)
@@ -216,12 +299,7 @@ class TerminalListFactory(private val context: Context) : RemoteViewsService.Rem
)
}
// Keep registry order (don't sort) — index must match T1, T2, T3...
return result
} catch (e: Exception) {
Log.w(TAG, "Failed to fetch terminals", e)
return null
result
}
}
@@ -231,11 +309,11 @@ class TerminalListFactory(private val context: Context) : RemoteViewsService.Rem
val entry = history.optJSONObject(i) ?: continue
if (entry.optString("event") == "UserPromptSubmit") {
val detail = entry.optString("detail", "")
if (detail.isNotEmpty()) return detail.take(120)
if (detail.isNotEmpty()) return detail.take(200)
}
}
val stopResp = state.optString("lastStopResponse", "")
if (stopResp.isNotEmpty()) return "< ${stopResp.take(100)}"
if (stopResp.isNotEmpty()) return "< ${stopResp.take(200)}"
return ""
}

28
src-tauri/gen/android/app/src/main/res/layout/widget_terminal_item.xml
View File

@@ -6,57 +6,55 @@
android:orientation="vertical"
android:paddingTop="3dp"
android:paddingBottom="3dp"
android:background="?android:attr/selectableItemBackground">
android:paddingStart="2dp"
android:paddingEnd="2dp"
android:background="@drawable/widget_item_bg">
<!-- Header: dot + agent name + status + badges -->
<LinearLayout
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:orientation="horizontal"
android:gravity="center_vertical">
<!-- Status dot -->
<TextView
android:id="@+id/item_dot"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="\u25CF"
android:text="●"
android:textColor="#6b7280"
android:textSize="8sp"
android:fontFamily="monospace"
android:paddingEnd="4dp" />
<!-- Agent name + status -->
<TextView
android:id="@+id/item_name"
android:layout_width="0dp"
android:layout_height="wrap_content"
android:layout_weight="1"
android:textColor="#DDDDDD"
android:textSize="10sp"
android:textColor="#DDFFFFFF"
android:textSize="13sp"
android:fontFamily="monospace"
android:maxLines="1"
android:ellipsize="end" />
<!-- Hook badges -->
<TextView
android:id="@+id/item_badges"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:textColor="#888888"
android:textSize="9sp"
android:textColor="#60FFFFFF"
android:textSize="10sp"
android:fontFamily="monospace"
android:paddingStart="4dp" />
android:paddingStart="6dp" />
</LinearLayout>
<!-- Label / last user prompt -->
<TextView
android:id="@+id/item_label"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:textColor="#9999CC"
android:textSize="10sp"
android:textColor="#80FFFFFF"
android:textSize="11sp"
android:fontFamily="monospace"
android:maxLines="2"
android:maxLines="3"
android:ellipsize="end"
android:paddingStart="12dp"
android:paddingTop="1dp" />

75
src-tauri/gen/android/app/src/main/res/layout/widget_transcript.xml
View File

@@ -4,56 +4,93 @@
android:layout_width="match_parent"
android:layout_height="match_parent"
android:orientation="vertical"
android:padding="10dp"
android:background="#DD1A1A2E">
android:background="@drawable/widget_bg_pixel"
android:paddingStart="20dp"
android:paddingEnd="20dp"
android:paddingTop="18dp"
android:paddingBottom="18dp">
<!-- Title bar -->
<!-- Terminal title bar -->
<LinearLayout
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:orientation="horizontal"
android:gravity="center_vertical"
android:paddingBottom="4dp">
android:paddingBottom="6dp">
<TextView
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="~$"
android:textColor="#B0FFFFFF"
android:textSize="13sp"
android:fontFamily="monospace"
android:paddingEnd="6dp" />
<TextView
android:id="@+id/widget_title"
android:layout_width="0dp"
android:layout_height="wrap_content"
android:layout_weight="1"
android:text="Agent UI"
android:textColor="#8888FF"
android:textSize="11sp"
android:text="agent-ui"
android:textColor="#CCFFFFFF"
android:textSize="14sp"
android:fontFamily="monospace" />
<TextView
android:id="@+id/widget_status_bar"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="●"
android:textColor="#8040C040"
android:textSize="9sp"
android:fontFamily="monospace"
android:paddingEnd="8dp" />
<TextView
android:id="@+id/btn_refresh"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="\u21BB"
android:textColor="#8888FF"
android:textSize="14sp"
android:paddingStart="8dp"
android:paddingEnd="4dp"
android:text="↻"
android:textColor="#AAFFFFFF"
android:textSize="18sp"
android:paddingStart="4dp"
android:paddingEnd="2dp"
android:background="?android:attr/selectableItemBackground" />
</LinearLayout>
<!-- Terminal list (dynamic, scrollable) -->
<!-- Separator -->
<TextView
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:text=""
android:textSize="1sp"
android:background="#20FFFFFF"
android:layout_marginBottom="6dp"
android:includeFontPadding="false"
android:minHeight="1dp"
android:maxHeight="1dp" />
<!-- Terminal list -->
<ListView
android:id="@+id/terminal_list"
android:layout_width="match_parent"
android:layout_height="match_parent"
android:layout_height="0dp"
android:layout_weight="1"
android:divider="@null"
android:dividerHeight="0dp"
android:scrollbars="none" />
android:scrollbars="none"
android:clipToPadding="false" />
<!-- Fallback when empty -->
<TextView
android:id="@+id/empty_view"
android:layout_width="match_parent"
android:layout_height="match_parent"
android:text="No terminals open"
android:textColor="#666666"
android:textSize="10sp"
android:layout_height="0dp"
android:layout_weight="1"
android:text="no active sessions"
android:textColor="#50FFFFFF"
android:textSize="12sp"
android:fontFamily="monospace"
android:gravity="center" />

1
src-tauri/gen/android/app/src/main/res/values/strings.xml
View File

@@ -2,4 +2,5 @@
<string name="app_name">Agent UI</string>
<string name="main_activity_title">Agent UI</string>
<string name="widget_description">Shows recent transcript messages from Agent UI</string>
<string name="input_widget_description">Terminal-style input prompt for Agent UI</string>
</resources>

2
src-tauri/gen/android/app/src/main/res/xml/transcript_widget_info.xml
View File

@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="utf-8"?>
<appwidget-provider xmlns:android="http://schemas.android.com/apk/res/android"
android:minWidth="250dp"
android:minHeight="140dp"
android:minHeight="110dp"
android:updatePeriodMillis="1800000"
android:initialLayout="@layout/widget_transcript"
android:resizeMode="horizontal|vertical"

3
src-tauri/tauri.conf.json
View File

@@ -6,7 +6,7 @@
"build": {
"frontendDist": "../frontend/dist",
"devUrl": "http://localhost:4100",
"beforeDevCommand": "",
"beforeDevCommand": "cd frontend && bun run dev --host --port 4100",
"beforeBuildCommand": "cd frontend && npx vite build"
},
"app": {
@@ -18,6 +18,7 @@
"minWidth": 800,
"minHeight": 600,
"decorations": false,
"transparent": true,
"resizable": true
}
],