feat: Samsung lock screen face widget, voice assistant services, PiP mode and gitignore installers
Add Samsung proprietary Face Widget (lock screen/AOD) with terminal status display. Add voice interaction services (AgentVoiceInteractionService, RecognitionService) for digital assistant registration. Add PiP mode with voice/expand actions. Add session-state proxy, voice transcript routes, window controls component. Ignore installers/ directory.
This commit is contained in:
@@ -45,8 +45,9 @@ android {
|
||||
}
|
||||
}
|
||||
getByName("release") {
|
||||
manifestPlaceholders["usesCleartextTraffic"] = "true"
|
||||
signingConfig = signingConfigs.getByName("release")
|
||||
isMinifyEnabled = true
|
||||
isMinifyEnabled = false
|
||||
proguardFiles(
|
||||
*fileTree(".") { include("**/*.pro") }
|
||||
.plus(getDefaultProguardFile("proguard-android-optimize.txt"))
|
||||
@@ -80,7 +81,8 @@ dependencies {
|
||||
|
||||
apply(from = "tauri.build.gradle.kts")
|
||||
|
||||
// Copy APK to src-tauri/installers after build
|
||||
// Copy APK after build: local installers/ + network backup
|
||||
// Only the universal signed variant gets copied as agent-ui.apk
|
||||
android.applicationVariants.all {
|
||||
val variant = this
|
||||
variant.outputs.all {
|
||||
@@ -88,9 +90,36 @@ android.applicationVariants.all {
|
||||
variant.assembleProvider.get().doLast {
|
||||
val src = output.outputFile
|
||||
if (src.exists()) {
|
||||
val dest = file("../../../../installers/AgentUI-${variant.versionName}-${variant.name}.apk")
|
||||
src.copyTo(dest, overwrite = true)
|
||||
println(">> Copied APK to ${dest.absolutePath}")
|
||||
val localDir = file("../../../../installers")
|
||||
localDir.mkdirs()
|
||||
|
||||
// Always save versioned copy per variant
|
||||
val localVersioned = File(localDir, "AgentUI-${variant.versionName}-${variant.name}.apk")
|
||||
src.copyTo(localVersioned, overwrite = true)
|
||||
println(">> Copied APK to ${localVersioned.absolutePath}")
|
||||
|
||||
// agent-ui.apk = only the universal signed build
|
||||
val isUniversal = variant.name.contains("universal", ignoreCase = true)
|
||||
val isSigned = variant.signingConfig != null
|
||||
if (isUniversal && isSigned) {
|
||||
val localFixed = File(localDir, "agent-ui.apk")
|
||||
src.copyTo(localFixed, overwrite = true)
|
||||
println(">> Copied universal signed APK to ${localFixed.absolutePath}")
|
||||
|
||||
// Network backup
|
||||
val networkDir = File("\\\\Memoria-1\\ActiveBackupforBusiness\\Nucleo v3\\agent-ui-apk")
|
||||
try {
|
||||
if (networkDir.exists() || networkDir.mkdirs()) {
|
||||
val networkFile = File(networkDir, "agent-ui.apk")
|
||||
src.copyTo(networkFile, overwrite = true)
|
||||
println(">> Copied APK to network: ${networkFile.absolutePath}")
|
||||
} else {
|
||||
println(">> WARNING: Network path not available: $networkDir")
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
println(">> WARNING: Failed to copy to network: ${e.message}")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
|
||||
<uses-permission android:name="android.permission.INTERNET" />
|
||||
<uses-permission android:name="android.permission.RECORD_AUDIO" />
|
||||
<uses-permission android:name="com.samsung.systemui.permission.FACE_WIDGET" />
|
||||
|
||||
<!-- AndroidTV support -->
|
||||
<uses-feature android:name="android.software.leanback" android:required="false" />
|
||||
@@ -16,7 +17,8 @@
|
||||
android:launchMode="singleTask"
|
||||
android:label="@string/main_activity_title"
|
||||
android:name=".MainActivity"
|
||||
android:exported="true">
|
||||
android:exported="true"
|
||||
android:supportsPictureInPicture="true">
|
||||
<intent-filter>
|
||||
<action android:name="android.intent.action.MAIN" />
|
||||
<category android:name="android.intent.category.LAUNCHER" />
|
||||
@@ -37,7 +39,7 @@
|
||||
android:resource="@xml/transcript_widget_info" />
|
||||
</receiver>
|
||||
|
||||
<!-- Voice Command / Share Activity -->
|
||||
<!-- Voice Command / Share / Assist Activity -->
|
||||
<activity
|
||||
android:name=".VoiceCommandActivity"
|
||||
android:label="Agent UI Voice"
|
||||
@@ -48,8 +50,70 @@
|
||||
<category android:name="android.intent.category.DEFAULT" />
|
||||
<data android:mimeType="text/plain" />
|
||||
</intent-filter>
|
||||
<intent-filter>
|
||||
<action android:name="android.intent.action.ASSIST" />
|
||||
<category android:name="android.intent.category.DEFAULT" />
|
||||
</intent-filter>
|
||||
<intent-filter>
|
||||
<action android:name="android.intent.action.VOICE_COMMAND" />
|
||||
<category android:name="android.intent.category.DEFAULT" />
|
||||
</intent-filter>
|
||||
</activity>
|
||||
|
||||
<!-- Digital Assistant Service (enables Samsung side-key, long-press home, etc.) -->
|
||||
<service
|
||||
android:name=".AgentVoiceInteractionService"
|
||||
android:label="@string/app_name"
|
||||
android:permission="android.permission.BIND_VOICE_INTERACTION"
|
||||
android:exported="true">
|
||||
<meta-data
|
||||
android:name="android.voice_interaction"
|
||||
android:resource="@xml/voice_interaction_service" />
|
||||
<intent-filter>
|
||||
<action android:name="android.service.voice.VoiceInteractionService" />
|
||||
</intent-filter>
|
||||
</service>
|
||||
|
||||
<service
|
||||
android:name=".AgentVoiceInteractionSessionService"
|
||||
android:permission="android.permission.BIND_VOICE_INTERACTION"
|
||||
android:exported="false" />
|
||||
|
||||
<!-- RecognitionService: required by Android to fully register as digital assistant -->
|
||||
<service
|
||||
android:name=".AgentRecognitionService"
|
||||
android:label="@string/app_name"
|
||||
android:permission="android.permission.BIND_RECOGNITION_SERVICE"
|
||||
android:exported="true">
|
||||
<intent-filter>
|
||||
<action android:name="android.speech.RecognitionService" />
|
||||
<category android:name="android.intent.category.DEFAULT" />
|
||||
</intent-filter>
|
||||
<meta-data
|
||||
android:name="android.speech"
|
||||
android:resource="@xml/recognition_service" />
|
||||
</service>
|
||||
|
||||
<!-- Widget ListView adapter service -->
|
||||
<service
|
||||
android:name=".TerminalListWidgetService"
|
||||
android:permission="android.permission.BIND_REMOTEVIEWS"
|
||||
android:exported="false" />
|
||||
|
||||
<!-- Samsung Lock Screen Face Widget -->
|
||||
<receiver
|
||||
android:name=".LockScreenWidgetReceiver"
|
||||
android:permission="com.samsung.systemui.permission.FACE_WIDGET"
|
||||
android:exported="true">
|
||||
<intent-filter>
|
||||
<action android:name="com.samsung.android.intent.action.REQUEST_SERVICEBOX_REMOTEVIEWS" />
|
||||
</intent-filter>
|
||||
</receiver>
|
||||
|
||||
<meta-data
|
||||
android:name="com.samsung.systemui.facewidget.executable"
|
||||
android:resource="@raw/facewidgets" />
|
||||
|
||||
<provider
|
||||
android:name="androidx.core.content.FileProvider"
|
||||
android:authorities="${applicationId}.fileprovider"
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
package com.agentui.desktop
|
||||
|
||||
import android.content.Intent
|
||||
import android.os.Bundle
|
||||
import android.speech.RecognitionService
|
||||
import android.speech.SpeechRecognizer
|
||||
import android.util.Log
|
||||
|
||||
/**
|
||||
* Stub RecognitionService required by Android to register the app
|
||||
* as a complete voice interaction service / digital assistant.
|
||||
*
|
||||
* Without this, the system won't fully recognize the app as an assistant
|
||||
* and settings will show "None" even when selected.
|
||||
*
|
||||
* We delegate actual speech recognition to the system's default
|
||||
* RecognizerIntent in VoiceCommandActivity.
|
||||
*/
|
||||
class AgentRecognitionService : RecognitionService() {
|
||||
|
||||
companion object {
|
||||
private const val TAG = "AgentUI.Recognition"
|
||||
}
|
||||
|
||||
override fun onStartListening(intent: Intent?, callback: Callback?) {
|
||||
Log.d(TAG, "onStartListening called")
|
||||
// We don't do real recognition here - VoiceCommandActivity uses
|
||||
// the system RecognizerIntent directly. This stub satisfies the
|
||||
// Android requirement that a VoiceInteractionService must have
|
||||
// an associated RecognitionService.
|
||||
callback?.error(SpeechRecognizer.ERROR_CLIENT)
|
||||
}
|
||||
|
||||
override fun onCancel(callback: Callback?) {
|
||||
Log.d(TAG, "onCancel called")
|
||||
}
|
||||
|
||||
override fun onStopListening(callback: Callback?) {
|
||||
Log.d(TAG, "onStopListening called")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package com.agentui.desktop
|
||||
|
||||
import android.service.voice.VoiceInteractionService
|
||||
import android.util.Log
|
||||
|
||||
class AgentVoiceInteractionService : VoiceInteractionService() {
|
||||
|
||||
override fun onReady() {
|
||||
super.onReady()
|
||||
Log.d(TAG, "VoiceInteractionService ready - registered as digital assistant")
|
||||
}
|
||||
|
||||
override fun onShutdown() {
|
||||
Log.d(TAG, "VoiceInteractionService shutting down")
|
||||
super.onShutdown()
|
||||
}
|
||||
|
||||
companion object {
|
||||
private const val TAG = "AgentUI.VIS"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package com.agentui.desktop
|
||||
|
||||
import android.content.Intent
|
||||
import android.os.Build
|
||||
import android.os.Bundle
|
||||
import android.service.voice.VoiceInteractionSession
|
||||
import android.service.voice.VoiceInteractionSessionService
|
||||
import android.util.Log
|
||||
|
||||
class AgentVoiceInteractionSessionService : VoiceInteractionSessionService() {
|
||||
|
||||
override fun onNewSession(args: Bundle?): VoiceInteractionSession {
|
||||
Log.d(TAG, "onNewSession called")
|
||||
return AgentSession(this)
|
||||
}
|
||||
|
||||
private class AgentSession(
|
||||
private val ctx: android.content.Context
|
||||
) : VoiceInteractionSession(ctx) {
|
||||
|
||||
override fun onPrepareShow(args: Bundle?, showFlags: Int) {
|
||||
super.onPrepareShow(args, showFlags)
|
||||
// Disable the default VoiceInteractionSession window/UI -
|
||||
// we launch our own activity instead
|
||||
setUiEnabled(false)
|
||||
Log.d(TAG, "onPrepareShow: UI disabled, will launch activity")
|
||||
}
|
||||
|
||||
override fun onShow(args: Bundle?, showFlags: Int) {
|
||||
super.onShow(args, showFlags)
|
||||
Log.d(TAG, "onShow called, flags=$showFlags")
|
||||
|
||||
val intent = Intent(ctx, VoiceCommandActivity::class.java).apply {
|
||||
action = Intent.ACTION_VOICE_COMMAND
|
||||
addFlags(Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_CLEAR_TOP)
|
||||
}
|
||||
|
||||
try {
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
|
||||
// API 30+: startAssistantActivity is the proper method
|
||||
// for launching activities from a VoiceInteractionSession
|
||||
startAssistantActivity(intent)
|
||||
Log.d(TAG, "startAssistantActivity dispatched (API 30+)")
|
||||
} else {
|
||||
// API 23-29: use startVoiceActivity
|
||||
startVoiceActivity(intent)
|
||||
Log.d(TAG, "startVoiceActivity dispatched (API <30)")
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.e(TAG, "Assistant/Voice activity launch failed, trying startActivity", e)
|
||||
try {
|
||||
ctx.startActivity(intent)
|
||||
Log.d(TAG, "Fallback startActivity dispatched")
|
||||
} catch (e2: Exception) {
|
||||
Log.e(TAG, "All launch methods failed", e2)
|
||||
}
|
||||
}
|
||||
|
||||
// Finish the session - the activity handles everything from here
|
||||
finish()
|
||||
}
|
||||
|
||||
override fun onHandleAssist(
|
||||
data: Bundle?,
|
||||
structure: android.app.assist.AssistStructure?,
|
||||
content: android.app.assist.AssistContent?
|
||||
) {
|
||||
Log.d(TAG, "onHandleAssist called")
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
private const val TAG = "AgentUI.VISS"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,238 @@
|
||||
package com.agentui.desktop
|
||||
|
||||
import android.content.BroadcastReceiver
|
||||
import android.content.Context
|
||||
import android.content.Intent
|
||||
import android.util.Log
|
||||
import android.view.View
|
||||
import android.widget.RemoteViews
|
||||
import okhttp3.OkHttpClient
|
||||
import okhttp3.Request
|
||||
import org.json.JSONObject
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
/**
|
||||
* Samsung Face Widget receiver for the lock screen / AOD.
|
||||
*
|
||||
* Samsung's proprietary system sends REQUEST_SERVICEBOX_REMOTEVIEWS
|
||||
* and expects RESPONSE_SERVICEBOX_REMOTEVIEWS back with RemoteViews
|
||||
* for both the lock screen ("origin") and AOD ("aod").
|
||||
*/
|
||||
class LockScreenWidgetReceiver : BroadcastReceiver() {
|
||||
|
||||
companion object {
|
||||
private const val TAG = "AgentUI.FaceWidget"
|
||||
private const val ACTION_REQUEST =
|
||||
"com.samsung.android.intent.action.REQUEST_SERVICEBOX_REMOTEVIEWS"
|
||||
private const val ACTION_RESPONSE =
|
||||
"com.samsung.android.intent.action.RESPONSE_SERVICEBOX_REMOTEVIEWS"
|
||||
private const val PAGE_ID = "agent_ui_transcript"
|
||||
|
||||
private val STATUS_COLORS = mapOf(
|
||||
"idle" to 0xFF6b7280.toInt(),
|
||||
"thinking" to 0xFF60a5fa.toInt(),
|
||||
"reading" to 0xFF22d3ee.toInt(),
|
||||
"writing" to 0xFF4ade80.toInt(),
|
||||
"toolUse" to 0xFFfbbf24.toInt(),
|
||||
"permissionRequest" to 0xFFfb923c.toInt(),
|
||||
"interrupted" to 0xFFf87171.toInt(),
|
||||
"error" to 0xFFf87171.toInt(),
|
||||
"sessionStart" to 0xFF60a5fa.toInt(),
|
||||
"sessionEnd" to 0xFF6b7280.toInt()
|
||||
)
|
||||
|
||||
// Dimmed versions for AOD
|
||||
private val AOD_STATUS_COLORS = mapOf(
|
||||
"idle" to 0xFF3b3f47.toInt(),
|
||||
"thinking" to 0xFF304f7a.toInt(),
|
||||
"reading" to 0xFF116670.toInt(),
|
||||
"writing" to 0xFF256b40.toInt(),
|
||||
"toolUse" to 0xFF7a5f12.toInt(),
|
||||
"permissionRequest" to 0xFF7a491e.toInt(),
|
||||
"interrupted" to 0xFF7a3838.toInt(),
|
||||
"error" to 0xFF7a3838.toInt(),
|
||||
"sessionStart" to 0xFF304f7a.toInt(),
|
||||
"sessionEnd" to 0xFF3b3f47.toInt()
|
||||
)
|
||||
|
||||
private val client = OkHttpClient.Builder()
|
||||
.connectTimeout(5, TimeUnit.SECONDS)
|
||||
.readTimeout(5, TimeUnit.SECONDS)
|
||||
.build()
|
||||
}
|
||||
|
||||
override fun onReceive(context: Context, intent: Intent) {
|
||||
if (intent.action != ACTION_REQUEST) return
|
||||
|
||||
val pageId = intent.getStringExtra("pageId")
|
||||
if (pageId != PAGE_ID) return
|
||||
|
||||
Log.d(TAG, "Face widget update requested for pageId=$pageId")
|
||||
|
||||
// Fetch data on a background thread to avoid ANR
|
||||
val pendingResult = goAsync()
|
||||
|
||||
Thread {
|
||||
try {
|
||||
val terminals = fetchTerminals(context)
|
||||
val lockViews = buildLockScreenViews(context, terminals)
|
||||
val aodViews = buildAodViews(context, terminals)
|
||||
|
||||
val response = Intent(ACTION_RESPONSE).apply {
|
||||
setPackage("com.android.systemui")
|
||||
putExtra("package", context.packageName)
|
||||
putExtra("pageId", PAGE_ID)
|
||||
putExtra("show", true)
|
||||
putExtra("origin", lockViews)
|
||||
putExtra("aod", aodViews)
|
||||
}
|
||||
|
||||
context.sendBroadcast(response)
|
||||
Log.d(TAG, "Face widget response sent with ${terminals.size} terminals")
|
||||
} catch (e: Exception) {
|
||||
Log.e(TAG, "Failed to build face widget", e)
|
||||
// Send empty response so widget doesn't get stuck
|
||||
val response = Intent(ACTION_RESPONSE).apply {
|
||||
setPackage("com.android.systemui")
|
||||
putExtra("package", context.packageName)
|
||||
putExtra("pageId", PAGE_ID)
|
||||
putExtra("show", true)
|
||||
putExtra("origin", buildEmptyLockScreenViews(context))
|
||||
putExtra("aod", buildEmptyAodViews(context))
|
||||
}
|
||||
context.sendBroadcast(response)
|
||||
} finally {
|
||||
pendingResult.finish()
|
||||
}
|
||||
}.start()
|
||||
}
|
||||
|
||||
private fun buildLockScreenViews(
|
||||
context: Context,
|
||||
terminals: List<FaceWidgetTerminal>
|
||||
): RemoteViews {
|
||||
val views = RemoteViews(context.packageName, R.layout.face_widget_lockscreen)
|
||||
|
||||
val statusText = if (terminals.isEmpty()) "offline"
|
||||
else "${terminals.size} agent${if (terminals.size > 1) "s" else ""}"
|
||||
views.setTextViewText(R.id.fw_status, statusText)
|
||||
|
||||
if (terminals.isEmpty()) {
|
||||
views.setViewVisibility(R.id.fw_empty, View.VISIBLE)
|
||||
} else {
|
||||
views.setViewVisibility(R.id.fw_empty, View.GONE)
|
||||
}
|
||||
|
||||
// Lockscreen terminal slot IDs
|
||||
val slotIds = listOf(
|
||||
Triple(R.id.fw_terminal_1, R.id.fw_dot_1, R.id.fw_name_1),
|
||||
Triple(R.id.fw_terminal_2, R.id.fw_dot_2, R.id.fw_name_2),
|
||||
Triple(R.id.fw_terminal_3, R.id.fw_dot_3, R.id.fw_name_3)
|
||||
)
|
||||
|
||||
for (i in slotIds.indices) {
|
||||
val (container, dot, name) = slotIds[i]
|
||||
if (i < terminals.size) {
|
||||
val t = terminals[i]
|
||||
views.setViewVisibility(container, View.VISIBLE)
|
||||
views.setTextColor(dot, t.statusColor)
|
||||
views.setTextViewText(name, "T${t.index} ${t.agent} ${t.status}")
|
||||
} else {
|
||||
views.setViewVisibility(container, View.GONE)
|
||||
}
|
||||
}
|
||||
|
||||
return views
|
||||
}
|
||||
|
||||
private fun buildAodViews(
|
||||
context: Context,
|
||||
terminals: List<FaceWidgetTerminal>
|
||||
): RemoteViews {
|
||||
val views = RemoteViews(context.packageName, R.layout.face_widget_aod)
|
||||
|
||||
if (terminals.isEmpty()) {
|
||||
views.setViewVisibility(R.id.fw_aod_empty, View.VISIBLE)
|
||||
} else {
|
||||
views.setViewVisibility(R.id.fw_aod_empty, View.GONE)
|
||||
}
|
||||
|
||||
val slotIds = listOf(
|
||||
Triple(R.id.fw_aod_terminal_1, R.id.fw_aod_dot_1, R.id.fw_aod_name_1),
|
||||
Triple(R.id.fw_aod_terminal_2, R.id.fw_aod_dot_2, R.id.fw_aod_name_2),
|
||||
Triple(R.id.fw_aod_terminal_3, R.id.fw_aod_dot_3, R.id.fw_aod_name_3)
|
||||
)
|
||||
|
||||
for (i in slotIds.indices) {
|
||||
val (container, dot, name) = slotIds[i]
|
||||
if (i < terminals.size) {
|
||||
val t = terminals[i]
|
||||
views.setViewVisibility(container, View.VISIBLE)
|
||||
val aodColor = AOD_STATUS_COLORS[t.status] ?: AOD_STATUS_COLORS["idle"]!!
|
||||
views.setTextColor(dot, aodColor)
|
||||
views.setTextViewText(name, "T${t.index} ${t.agent}")
|
||||
} else {
|
||||
views.setViewVisibility(container, View.GONE)
|
||||
}
|
||||
}
|
||||
|
||||
return views
|
||||
}
|
||||
|
||||
private fun buildEmptyLockScreenViews(context: Context): RemoteViews {
|
||||
return RemoteViews(context.packageName, R.layout.face_widget_lockscreen)
|
||||
}
|
||||
|
||||
private fun buildEmptyAodViews(context: Context): RemoteViews {
|
||||
return RemoteViews(context.packageName, R.layout.face_widget_aod)
|
||||
}
|
||||
|
||||
private fun fetchTerminals(context: Context): List<FaceWidgetTerminal> {
|
||||
val apiBase = ServerConfig.apiBaseUrl(context) ?: return emptyList()
|
||||
|
||||
try {
|
||||
val url = "$apiBase/session-state"
|
||||
val req = Request.Builder().url(url).build()
|
||||
val resp = client.newCall(req).execute()
|
||||
if (!resp.isSuccessful) return emptyList()
|
||||
|
||||
val json = JSONObject(resp.body?.string() ?: "{}")
|
||||
val registry = json.optJSONArray("registry") ?: return emptyList()
|
||||
val agents = json.optJSONObject("agents")
|
||||
|
||||
val result = mutableListOf<FaceWidgetTerminal>()
|
||||
|
||||
for (i in 0 until minOf(registry.length(), 3)) { // Max 3 for face widget
|
||||
val entry = registry.getJSONObject(i)
|
||||
val agentName = entry.optString("agent", "")
|
||||
val alive = entry.optBoolean("alive", false)
|
||||
|
||||
val agentState = agents?.optJSONObject(agentName)
|
||||
val status = agentState?.optString("status", if (alive) "idle" else "closed")
|
||||
?: if (alive) "idle" else "closed"
|
||||
val statusColor = STATUS_COLORS[status] ?: STATUS_COLORS["idle"]!!
|
||||
|
||||
result.add(
|
||||
FaceWidgetTerminal(
|
||||
index = i + 1,
|
||||
agent = agentName,
|
||||
status = status,
|
||||
statusColor = statusColor
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
return result
|
||||
} catch (e: Exception) {
|
||||
Log.w(TAG, "Failed to fetch terminals for face widget", e)
|
||||
return emptyList()
|
||||
}
|
||||
}
|
||||
|
||||
private data class FaceWidgetTerminal(
|
||||
val index: Int,
|
||||
val agent: String,
|
||||
val status: String,
|
||||
val statusColor: Int
|
||||
)
|
||||
}
|
||||
@@ -1,40 +1,279 @@
|
||||
package com.agentui.desktop
|
||||
|
||||
import android.app.PendingIntent
|
||||
import android.app.PictureInPictureParams
|
||||
import android.app.RemoteAction
|
||||
import android.content.BroadcastReceiver
|
||||
import android.content.Context
|
||||
import android.content.Intent
|
||||
import android.content.IntentFilter
|
||||
import android.content.res.Configuration
|
||||
import android.graphics.drawable.Icon
|
||||
import android.os.Build
|
||||
import android.os.Bundle
|
||||
import android.util.Log
|
||||
import android.util.Rational
|
||||
import android.webkit.WebView
|
||||
import androidx.activity.enableEdgeToEdge
|
||||
import org.json.JSONObject
|
||||
import java.io.File
|
||||
import androidx.core.view.ViewCompat
|
||||
import androidx.core.view.WindowInsetsCompat
|
||||
|
||||
class MainActivity : TauriActivity() {
|
||||
|
||||
companion object {
|
||||
private const val ACTION_PIP_MIC = "com.agentui.desktop.PIP_MIC"
|
||||
private const val ACTION_PIP_EXPAND = "com.agentui.desktop.PIP_EXPAND"
|
||||
private const val PIP_REQUEST_CODE_MIC = 2001
|
||||
private const val PIP_REQUEST_CODE_EXPAND = 2002
|
||||
}
|
||||
|
||||
private var pendingRoute: String? = null
|
||||
private var pendingVoiceTerminal: String? = null
|
||||
|
||||
private val pipReceiver = object : BroadcastReceiver() {
|
||||
override fun onReceive(context: Context, intent: Intent) {
|
||||
when (intent.action) {
|
||||
ACTION_PIP_MIC -> {
|
||||
Log.d("AgentUI", "PiP mic button pressed")
|
||||
// Launch voice command from PiP
|
||||
val voiceIntent = Intent(context, VoiceCommandActivity::class.java).apply {
|
||||
action = Intent.ACTION_VOICE_COMMAND
|
||||
addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
|
||||
}
|
||||
startActivity(voiceIntent)
|
||||
}
|
||||
ACTION_PIP_EXPAND -> {
|
||||
Log.d("AgentUI", "PiP expand button pressed")
|
||||
// Bring app to foreground full-screen
|
||||
val expandIntent = Intent(context, MainActivity::class.java).apply {
|
||||
addFlags(Intent.FLAG_ACTIVITY_REORDER_TO_FRONT or Intent.FLAG_ACTIVITY_SINGLE_TOP)
|
||||
}
|
||||
startActivity(expandIntent)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun onCreate(savedInstanceState: Bundle?) {
|
||||
enableEdgeToEdge()
|
||||
super.onCreate(savedInstanceState)
|
||||
syncServerUrlToPrefs()
|
||||
injectSafeAreaInsets()
|
||||
handleWidgetIntent(intent)
|
||||
handleVoiceIntent(intent)
|
||||
registerPipReceiver()
|
||||
}
|
||||
|
||||
override fun onDestroy() {
|
||||
super.onDestroy()
|
||||
try { unregisterReceiver(pipReceiver) } catch (_: Exception) {}
|
||||
}
|
||||
|
||||
private fun registerPipReceiver() {
|
||||
val filter = IntentFilter().apply {
|
||||
addAction(ACTION_PIP_MIC)
|
||||
addAction(ACTION_PIP_EXPAND)
|
||||
}
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
|
||||
registerReceiver(pipReceiver, filter, RECEIVER_NOT_EXPORTED)
|
||||
} else {
|
||||
registerReceiver(pipReceiver, filter)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Enter PiP when user leaves the app (home button / swipe).
|
||||
*/
|
||||
override fun onUserLeaveHint() {
|
||||
super.onUserLeaveHint()
|
||||
// Only enter PiP via voice command flow, not on regular minimize
|
||||
}
|
||||
|
||||
override fun onPictureInPictureModeChanged(
|
||||
isInPictureInPictureMode: Boolean,
|
||||
newConfig: Configuration
|
||||
) {
|
||||
super.onPictureInPictureModeChanged(isInPictureInPictureMode, newConfig)
|
||||
Log.d("AgentUI", "PiP mode changed: $isInPictureInPictureMode")
|
||||
}
|
||||
|
||||
private fun enterPipIfSupported() {
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
|
||||
try {
|
||||
val actions = buildPipActions()
|
||||
val builder = PictureInPictureParams.Builder()
|
||||
.setAspectRatio(Rational(9, 16))
|
||||
.setActions(actions)
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
|
||||
builder.setAutoEnterEnabled(false)
|
||||
builder.setSeamlessResizeEnabled(true)
|
||||
}
|
||||
enterPictureInPictureMode(builder.build())
|
||||
} catch (e: Exception) {
|
||||
Log.w("AgentUI", "Failed to enter PiP: $e")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun buildPipActions(): List<RemoteAction> {
|
||||
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) return emptyList()
|
||||
|
||||
val actions = mutableListOf<RemoteAction>()
|
||||
|
||||
// Mic button — launch voice command
|
||||
val micIntent = PendingIntent.getBroadcast(
|
||||
this, PIP_REQUEST_CODE_MIC,
|
||||
Intent(ACTION_PIP_MIC).setPackage(packageName),
|
||||
PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE
|
||||
)
|
||||
actions.add(RemoteAction(
|
||||
Icon.createWithResource(this, android.R.drawable.ic_btn_speak_now),
|
||||
"Voice", "Send voice command",
|
||||
micIntent
|
||||
))
|
||||
|
||||
// Expand button — bring app to foreground
|
||||
val expandIntent = PendingIntent.getBroadcast(
|
||||
this, PIP_REQUEST_CODE_EXPAND,
|
||||
Intent(ACTION_PIP_EXPAND).setPackage(packageName),
|
||||
PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE
|
||||
)
|
||||
actions.add(RemoteAction(
|
||||
Icon.createWithResource(this, android.R.drawable.ic_menu_view),
|
||||
"Open", "Open full app",
|
||||
expandIntent
|
||||
))
|
||||
|
||||
return actions
|
||||
}
|
||||
|
||||
override fun onNewIntent(intent: android.content.Intent) {
|
||||
super.onNewIntent(intent)
|
||||
handleWidgetIntent(intent)
|
||||
handleVoiceIntent(intent)
|
||||
}
|
||||
|
||||
override fun onResume() {
|
||||
super.onResume()
|
||||
syncServerUrlToPrefs()
|
||||
pendingRoute?.let { route ->
|
||||
pendingRoute = null
|
||||
navigateWebView(route)
|
||||
}
|
||||
}
|
||||
|
||||
private fun handleWidgetIntent(intent: android.content.Intent?) {
|
||||
val terminalIndex = intent?.getIntExtra("terminalIndex", -1) ?: -1
|
||||
if (terminalIndex > 0) {
|
||||
val route = "/transcript-debug/$terminalIndex"
|
||||
Log.d("AgentUI", "Widget click → navigate to $route")
|
||||
pendingRoute = route
|
||||
navigateWebView(route)
|
||||
}
|
||||
}
|
||||
|
||||
private fun handleVoiceIntent(intent: android.content.Intent?) {
|
||||
if (intent?.action == "com.agentui.desktop.VOICE_TERMINAL") {
|
||||
val sessionId = intent.getStringExtra("ephemeralSessionId") ?: return
|
||||
if (sessionId.isNotEmpty()) {
|
||||
Log.d("AgentUI", "Voice intent → open terminal $sessionId")
|
||||
pendingVoiceTerminal = sessionId
|
||||
// Don't call openVoiceTerminal here — WebView may not exist yet.
|
||||
// Instead, poll until the WebView is ready.
|
||||
pollForWebViewAndOpenTerminal(sessionId, 0)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads the Tauri plugin-store settings.json and syncs serverUrl
|
||||
* to SharedPreferences so native components (widget, voice) can use it.
|
||||
* Retry finding the WebView up to ~3 seconds (15 attempts x 200ms).
|
||||
* Once found, inject JS to open floating transcript and enter PiP.
|
||||
*/
|
||||
private fun syncServerUrlToPrefs() {
|
||||
try {
|
||||
val storeFile = File(filesDir, "app_tauri-plugin-store/settings.json")
|
||||
if (!storeFile.exists()) return
|
||||
private fun pollForWebViewAndOpenTerminal(ephemeralSessionId: String, attempt: Int) {
|
||||
if (attempt > 15) {
|
||||
Log.w("AgentUI", "Gave up waiting for WebView after ${attempt} attempts")
|
||||
pendingVoiceTerminal = null
|
||||
return
|
||||
}
|
||||
|
||||
val json = JSONObject(storeFile.readText())
|
||||
val serverUrl = json.optString("serverUrl", "")
|
||||
if (serverUrl.isNotEmpty()) {
|
||||
ServerConfig.setServerUrl(this, serverUrl)
|
||||
Log.d("AgentUI", "Synced serverUrl to prefs: $serverUrl")
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.w("AgentUI", "Failed to sync server URL", e)
|
||||
val webView = try { findWebView(window.decorView) } catch (_: Exception) { null }
|
||||
|
||||
if (webView != null) {
|
||||
val js = "window.__VOICE_OPEN_TERMINAL__ && window.__VOICE_OPEN_TERMINAL__('$ephemeralSessionId')"
|
||||
webView.evaluateJavascript(js, null)
|
||||
pendingVoiceTerminal = null
|
||||
Log.d("AgentUI", "Voice terminal JS dispatched (attempt $attempt): $ephemeralSessionId")
|
||||
|
||||
// Enter PiP after the WebView has time to render
|
||||
webView.postDelayed({ enterPipIfSupported() }, 500)
|
||||
} else {
|
||||
Log.d("AgentUI", "WebView not ready (attempt $attempt), retrying in 200ms")
|
||||
window.decorView.postDelayed({
|
||||
pollForWebViewAndOpenTerminal(ephemeralSessionId, attempt + 1)
|
||||
}, 200)
|
||||
}
|
||||
}
|
||||
|
||||
private fun navigateWebView(route: String) {
|
||||
try {
|
||||
val decorView = window.decorView
|
||||
val webView = findWebView(decorView)
|
||||
if (webView != null) {
|
||||
val js = "window.__WIDGET_NAVIGATE__ && window.__WIDGET_NAVIGATE__('$route') || (window.location.href = '$route')"
|
||||
webView.evaluateJavascript(js, null)
|
||||
pendingRoute = null
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.w("AgentUI", "Failed to navigate WebView: $e")
|
||||
}
|
||||
}
|
||||
|
||||
private fun injectSafeAreaInsets() {
|
||||
val decorView = window.decorView
|
||||
ViewCompat.setOnApplyWindowInsetsListener(decorView) { view, insets ->
|
||||
val systemBars = insets.getInsets(WindowInsetsCompat.Type.systemBars())
|
||||
val density = resources.displayMetrics.density
|
||||
val topPx = systemBars.top
|
||||
val bottomPx = systemBars.bottom
|
||||
val leftPx = systemBars.left
|
||||
val rightPx = systemBars.right
|
||||
val topDp = topPx / density
|
||||
val bottomDp = bottomPx / density
|
||||
val leftDp = leftPx / density
|
||||
val rightDp = rightPx / density
|
||||
val js = """
|
||||
document.documentElement.style.setProperty('--sat', '${topDp}px');
|
||||
document.documentElement.style.setProperty('--sab', '${bottomDp}px');
|
||||
document.documentElement.style.setProperty('--sal', '${leftDp}px');
|
||||
document.documentElement.style.setProperty('--sar', '${rightDp}px');
|
||||
""".trimIndent()
|
||||
try {
|
||||
val webView = view.findViewWithTag<WebView>("tauri_webview")
|
||||
?: view.findViewById<WebView>(android.R.id.content)?.let {
|
||||
findWebView(it)
|
||||
}
|
||||
webView?.evaluateJavascript(js, null)
|
||||
Log.d("AgentUI", "Injected safe-area: top=${topDp}dp bottom=${bottomDp}dp")
|
||||
} catch (e: Exception) {
|
||||
Log.w("AgentUI", "Failed to inject safe-area insets: $e")
|
||||
}
|
||||
ViewCompat.onApplyWindowInsets(view, insets)
|
||||
}
|
||||
}
|
||||
|
||||
private fun findWebView(view: android.view.View): WebView? {
|
||||
if (view is WebView) return view
|
||||
if (view is android.view.ViewGroup) {
|
||||
for (i in 0 until view.childCount) {
|
||||
val found = findWebView(view.getChildAt(i))
|
||||
if (found != null) return found
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
private fun syncServerUrlToPrefs() {
|
||||
val url = ServerConfig.getServerUrl(this)
|
||||
Log.d("AgentUI", "syncServerUrlToPrefs: resolved url=$url")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,21 +1,149 @@
|
||||
package com.agentui.desktop
|
||||
|
||||
import android.content.Context
|
||||
import android.util.Log
|
||||
import org.json.JSONObject
|
||||
import java.io.File
|
||||
|
||||
object ServerConfig {
|
||||
private const val TAG = "ServerConfig"
|
||||
private const val PREFS_NAME = "agent_ui_config"
|
||||
private const val KEY_SERVER_URL = "server_url"
|
||||
|
||||
fun getServerUrl(context: Context): String? {
|
||||
return context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE)
|
||||
val stored = context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE)
|
||||
.getString(KEY_SERVER_URL, null)
|
||||
if (!stored.isNullOrEmpty()) {
|
||||
Log.d(TAG, "Found serverUrl in SharedPreferences: $stored")
|
||||
return stored
|
||||
}
|
||||
|
||||
// Fallback: search for Tauri store file in multiple possible locations
|
||||
return readFromTauriStore(context)
|
||||
}
|
||||
|
||||
fun setServerUrl(context: Context, url: String) {
|
||||
val normalized = url.trimEnd('/')
|
||||
context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE)
|
||||
.edit()
|
||||
.putString(KEY_SERVER_URL, url.trimEnd('/'))
|
||||
.putString(KEY_SERVER_URL, normalized)
|
||||
.apply()
|
||||
Log.d(TAG, "Saved serverUrl to SharedPreferences: $normalized")
|
||||
}
|
||||
|
||||
/**
|
||||
* Searches for serverUrl in the Tauri plugin-store settings.json.
|
||||
* Tries multiple possible file locations since the Tauri store path
|
||||
* can vary by plugin version and platform.
|
||||
*/
|
||||
private fun readFromTauriStore(context: Context): String? {
|
||||
Log.d(TAG, "SharedPreferences empty, searching Tauri store files...")
|
||||
|
||||
// Try multiple possible paths where Tauri plugin-store may save settings.json
|
||||
val possiblePaths = listOf(
|
||||
File(context.filesDir, "app_tauri-plugin-store/settings.json"),
|
||||
File(context.filesDir, "settings.json"),
|
||||
File(context.dataDir, "app_tauri-plugin-store/settings.json"),
|
||||
File(context.dataDir, "settings.json"),
|
||||
File(context.filesDir, ".tauri/settings.json"),
|
||||
File(context.filesDir, "tauri-plugin-store/settings.json")
|
||||
)
|
||||
|
||||
for (file in possiblePaths) {
|
||||
Log.d(TAG, " Checking: ${file.absolutePath} exists=${file.exists()}")
|
||||
if (file.exists()) {
|
||||
try {
|
||||
val content = file.readText()
|
||||
Log.d(TAG, " Found store file! Content: ${content.take(200)}")
|
||||
val url = extractServerUrl(content)
|
||||
if (url != null) {
|
||||
setServerUrl(context, url)
|
||||
Log.d(TAG, " Extracted and cached serverUrl: $url")
|
||||
return url
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.w(TAG, " Failed to parse: ${file.absolutePath}", e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Also search recursively for any settings.json in the data directory
|
||||
val found = findFile(context.filesDir, "settings.json")
|
||||
if (found != null) {
|
||||
Log.d(TAG, " Found via recursive search: ${found.absolutePath}")
|
||||
try {
|
||||
val content = found.readText()
|
||||
Log.d(TAG, " Content: ${content.take(200)}")
|
||||
val url = extractServerUrl(content)
|
||||
if (url != null) {
|
||||
setServerUrl(context, url)
|
||||
return url
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.w(TAG, " Failed to parse found file", e)
|
||||
}
|
||||
}
|
||||
|
||||
// Debug: dump directory tree so we can see where files actually are
|
||||
Log.w(TAG, " Could not find settings.json. Directory tree of filesDir:")
|
||||
dumpTree(context.filesDir, " ")
|
||||
Log.w(TAG, " Directory tree of dataDir:")
|
||||
dumpTree(context.dataDir, " ")
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts serverUrl from JSON content.
|
||||
* Handles both flat format {"serverUrl":"..."} and
|
||||
* possible wrapped formats.
|
||||
*/
|
||||
private fun extractServerUrl(content: String): String? {
|
||||
try {
|
||||
val json = JSONObject(content)
|
||||
|
||||
// Direct key
|
||||
val direct = json.optString("serverUrl", "")
|
||||
if (direct.isNotEmpty()) return direct.trimEnd('/')
|
||||
|
||||
// Maybe nested under a "store" or "data" key
|
||||
for (key in json.keys()) {
|
||||
val inner = json.optJSONObject(key)
|
||||
if (inner != null) {
|
||||
val nested = inner.optString("serverUrl", "")
|
||||
if (nested.isNotEmpty()) return nested.trimEnd('/')
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.w(TAG, "extractServerUrl parse error", e)
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/** Recursively find a file by name */
|
||||
private fun findFile(dir: File, name: String): File? {
|
||||
val files = dir.listFiles() ?: return null
|
||||
for (file in files) {
|
||||
if (file.isFile && file.name == name) return file
|
||||
if (file.isDirectory) {
|
||||
val found = findFile(file, name)
|
||||
if (found != null) return found
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/** Dump directory tree for debugging */
|
||||
private fun dumpTree(dir: File, indent: String) {
|
||||
val files = dir.listFiles() ?: return
|
||||
for (file in files) {
|
||||
if (file.isDirectory) {
|
||||
Log.d(TAG, "$indent${file.name}/")
|
||||
dumpTree(file, "$indent ")
|
||||
} else {
|
||||
Log.d(TAG, "$indent${file.name} (${file.length()} bytes)")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** e.g. "http://192.168.1.10:4103" */
|
||||
|
||||
@@ -0,0 +1,260 @@
|
||||
package com.agentui.desktop
|
||||
|
||||
import android.appwidget.AppWidgetManager
|
||||
import android.content.ComponentName
|
||||
import android.content.Context
|
||||
import android.content.Intent
|
||||
import android.os.Handler
|
||||
import android.os.Looper
|
||||
import android.util.Log
|
||||
import android.widget.RemoteViews
|
||||
import android.widget.RemoteViewsService
|
||||
import okhttp3.OkHttpClient
|
||||
import okhttp3.Request
|
||||
import org.json.JSONObject
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
class TerminalListWidgetService : RemoteViewsService() {
|
||||
override fun onGetViewFactory(intent: Intent): RemoteViewsFactory {
|
||||
return TerminalListFactory(applicationContext)
|
||||
}
|
||||
}
|
||||
|
||||
data class TerminalItem(
|
||||
val ephemeralSessionId: String,
|
||||
val agent: String,
|
||||
val label: String,
|
||||
val status: String,
|
||||
val alive: Boolean,
|
||||
val hookBadges: String,
|
||||
val lastUserPrompt: String,
|
||||
val statusColor: Int,
|
||||
val terminalIndex: Int // 1-based, maps to /transcript-debug/:terminalIndex
|
||||
)
|
||||
|
||||
class TerminalListFactory(private val context: Context) : RemoteViewsService.RemoteViewsFactory {
|
||||
|
||||
companion object {
|
||||
private const val TAG = "WidgetListFactory"
|
||||
|
||||
// Refresh button states
|
||||
private const val ICON_NORMAL = "\u21BB" // ↻
|
||||
private const val ICON_LOADING = "\u23F3" // ⏳
|
||||
private const val ICON_OK = "\u2713" // ✓
|
||||
private const val ICON_ERROR = "\u26A0" // ⚠
|
||||
|
||||
private const val COLOR_NORMAL = 0xFF8888FF.toInt()
|
||||
private const val COLOR_LOADING = 0xFF60a5fa.toInt()
|
||||
private const val COLOR_OK = 0xFF4ade80.toInt()
|
||||
private const val COLOR_ERROR = 0xFFf87171.toInt()
|
||||
|
||||
private val client = OkHttpClient.Builder()
|
||||
.connectTimeout(8, TimeUnit.SECONDS)
|
||||
.readTimeout(8, TimeUnit.SECONDS)
|
||||
.build()
|
||||
|
||||
private val STATUS_COLORS = mapOf(
|
||||
"idle" to 0xFF6b7280.toInt(),
|
||||
"thinking" to 0xFF60a5fa.toInt(),
|
||||
"reading" to 0xFF22d3ee.toInt(),
|
||||
"writing" to 0xFF4ade80.toInt(),
|
||||
"toolUse" to 0xFFfbbf24.toInt(),
|
||||
"permissionRequest" to 0xFFfb923c.toInt(),
|
||||
"interrupted" to 0xFFf87171.toInt(),
|
||||
"error" to 0xFFf87171.toInt(),
|
||||
"sessionStart" to 0xFF60a5fa.toInt(),
|
||||
"sessionEnd" to 0xFF6b7280.toInt()
|
||||
)
|
||||
|
||||
private val TOOL_EVENTS = setOf("PreToolUse", "PostToolUse", "PostToolUseFailure")
|
||||
private val PERM_EVENTS = setOf("PermissionRequest")
|
||||
private val SESSION_EVENTS = setOf("SessionStart", "UserPromptSubmit", "Stop", "SessionEnd")
|
||||
}
|
||||
|
||||
private var items = listOf<TerminalItem>()
|
||||
private val mainHandler = Handler(Looper.getMainLooper())
|
||||
|
||||
override fun onCreate() {}
|
||||
|
||||
override fun onDataSetChanged() {
|
||||
setRefreshButton(ICON_LOADING, COLOR_LOADING)
|
||||
|
||||
val result = fetchTerminals()
|
||||
|
||||
if (result == null) {
|
||||
// Network error — keep previous items, show error on button
|
||||
setRefreshButton(ICON_ERROR, COLOR_ERROR)
|
||||
scheduleResetButton(3000)
|
||||
} else {
|
||||
items = result
|
||||
// Brief success flash, then back to normal
|
||||
setRefreshButton(ICON_OK, COLOR_OK)
|
||||
scheduleResetButton(1500)
|
||||
}
|
||||
}
|
||||
|
||||
override fun onDestroy() {
|
||||
items = emptyList()
|
||||
}
|
||||
|
||||
override fun getCount(): Int = items.size
|
||||
|
||||
override fun getViewAt(position: Int): RemoteViews {
|
||||
val views = RemoteViews(context.packageName, R.layout.widget_terminal_item)
|
||||
|
||||
if (position >= items.size) return views
|
||||
val item = items[position]
|
||||
|
||||
views.setTextColor(R.id.item_dot, item.statusColor)
|
||||
|
||||
val statusLabel = if (item.alive) item.status else "closed"
|
||||
views.setTextViewText(R.id.item_name, "T${item.terminalIndex} ${item.agent} $statusLabel")
|
||||
|
||||
views.setTextViewText(R.id.item_badges, item.hookBadges)
|
||||
|
||||
// Always show the registry label (unique per terminal)
|
||||
views.setTextViewText(R.id.item_label, item.label)
|
||||
|
||||
val fillIntent = Intent().apply {
|
||||
putExtra("terminalIndex", item.terminalIndex)
|
||||
putExtra("agent", item.agent)
|
||||
}
|
||||
views.setOnClickFillInIntent(R.id.item_root, fillIntent)
|
||||
|
||||
return views
|
||||
}
|
||||
|
||||
override fun getLoadingView(): RemoteViews? = null
|
||||
override fun getViewTypeCount(): Int = 1
|
||||
override fun getItemId(position: Int): Long = position.toLong()
|
||||
override fun hasStableIds(): Boolean = false
|
||||
|
||||
// ── Refresh button state management ──
|
||||
|
||||
/**
|
||||
* Update just the refresh button via partiallyUpdateAppWidget.
|
||||
* This doesn't reset the ListView adapter.
|
||||
*/
|
||||
private fun setRefreshButton(icon: String, color: Int) {
|
||||
try {
|
||||
val views = RemoteViews(context.packageName, R.layout.widget_transcript)
|
||||
views.setTextViewText(R.id.btn_refresh, icon)
|
||||
views.setTextColor(R.id.btn_refresh, color)
|
||||
|
||||
val mgr = AppWidgetManager.getInstance(context)
|
||||
val ids = mgr.getAppWidgetIds(
|
||||
ComponentName(context, TranscriptWidgetProvider::class.java)
|
||||
)
|
||||
mgr.partiallyUpdateAppWidget(ids, views)
|
||||
} catch (e: Exception) {
|
||||
Log.w(TAG, "Failed to update refresh button", e)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Schedule resetting the button back to normal after a delay.
|
||||
*/
|
||||
private fun scheduleResetButton(delayMs: Long) {
|
||||
mainHandler.postDelayed({
|
||||
setRefreshButton(ICON_NORMAL, COLOR_NORMAL)
|
||||
}, delayMs)
|
||||
}
|
||||
|
||||
// ── Data fetching ──
|
||||
|
||||
/**
|
||||
* Returns list on success, null on error.
|
||||
* Items keep the same order as the terminal registry (T1, T2, T3...)
|
||||
* so the index maps directly to /transcript-debug/:terminalIndex
|
||||
*/
|
||||
private fun fetchTerminals(): List<TerminalItem>? {
|
||||
val apiBase = ServerConfig.apiBaseUrl(context) ?: return emptyList()
|
||||
|
||||
try {
|
||||
val url = "$apiBase/session-state"
|
||||
val req = Request.Builder().url(url).build()
|
||||
val resp = client.newCall(req).execute()
|
||||
if (!resp.isSuccessful) return null
|
||||
|
||||
val json = JSONObject(resp.body?.string() ?: "{}")
|
||||
val registry = json.optJSONArray("registry")
|
||||
val agents = json.optJSONObject("agents")
|
||||
|
||||
if (registry == null || registry.length() == 0) return emptyList()
|
||||
|
||||
val result = mutableListOf<TerminalItem>()
|
||||
|
||||
for (i in 0 until registry.length()) {
|
||||
val entry = registry.getJSONObject(i)
|
||||
val agentName = entry.optString("agent", "")
|
||||
val ephId = entry.optString("ephemeralSessionId", "")
|
||||
val label = entry.optString("label", "")
|
||||
val alive = entry.optBoolean("alive", false)
|
||||
val terminalIndex = i + 1 // 1-based, maps to /transcript-debug/:terminalIndex
|
||||
|
||||
val agentState = agents?.optJSONObject(agentName)
|
||||
|
||||
val status = agentState?.optString("status", if (alive) "idle" else "closed")
|
||||
?: if (alive) "idle" else "closed"
|
||||
val statusColor = STATUS_COLORS[status] ?: STATUS_COLORS["idle"]!!
|
||||
|
||||
val lastUserPrompt = if (agentState != null) extractLastUserPrompt(agentState) else ""
|
||||
val hookBadges = if (agentState != null) buildBadgeString(agentState) else ""
|
||||
|
||||
result.add(
|
||||
TerminalItem(
|
||||
ephemeralSessionId = ephId,
|
||||
agent = agentName,
|
||||
label = label,
|
||||
status = status,
|
||||
alive = alive,
|
||||
hookBadges = hookBadges,
|
||||
lastUserPrompt = lastUserPrompt,
|
||||
statusColor = statusColor,
|
||||
terminalIndex = terminalIndex
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
// Keep registry order (don't sort) — index must match T1, T2, T3...
|
||||
return result
|
||||
|
||||
} catch (e: Exception) {
|
||||
Log.w(TAG, "Failed to fetch terminals", e)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
private fun extractLastUserPrompt(state: JSONObject): String {
|
||||
val history = state.optJSONArray("hookHistory") ?: return ""
|
||||
for (i in history.length() - 1 downTo 0) {
|
||||
val entry = history.optJSONObject(i) ?: continue
|
||||
if (entry.optString("event") == "UserPromptSubmit") {
|
||||
val detail = entry.optString("detail", "")
|
||||
if (detail.isNotEmpty()) return detail.take(120)
|
||||
}
|
||||
}
|
||||
val stopResp = state.optString("lastStopResponse", "")
|
||||
if (stopResp.isNotEmpty()) return "< ${stopResp.take(100)}"
|
||||
return ""
|
||||
}
|
||||
|
||||
private fun buildBadgeString(state: JSONObject): String {
|
||||
val history = state.optJSONArray("hookHistory") ?: return ""
|
||||
var tools = 0; var perms = 0; var sessions = 0
|
||||
for (i in 0 until history.length()) {
|
||||
val entry = history.optJSONObject(i) ?: continue
|
||||
val event = entry.optString("event", "")
|
||||
when {
|
||||
event in TOOL_EVENTS -> tools++
|
||||
event in PERM_EVENTS -> perms++
|
||||
event in SESSION_EVENTS -> sessions++
|
||||
}
|
||||
}
|
||||
val parts = mutableListOf<String>()
|
||||
if (tools > 0) parts.add("T:$tools")
|
||||
if (perms > 0) parts.add("P:$perms")
|
||||
if (sessions > 0) parts.add("S:$sessions")
|
||||
return parts.joinToString(" ")
|
||||
}
|
||||
}
|
||||
@@ -3,12 +3,19 @@ package com.agentui.desktop
|
||||
import android.app.PendingIntent
|
||||
import android.appwidget.AppWidgetManager
|
||||
import android.appwidget.AppWidgetProvider
|
||||
import android.content.ComponentName
|
||||
import android.content.Context
|
||||
import android.content.Intent
|
||||
import android.net.Uri
|
||||
import android.widget.RemoteViews
|
||||
|
||||
class TranscriptWidgetProvider : AppWidgetProvider() {
|
||||
|
||||
companion object {
|
||||
const val ACTION_REFRESH = "com.agentui.desktop.WIDGET_REFRESH"
|
||||
const val ACTION_ITEM_CLICK = "com.agentui.desktop.WIDGET_ITEM_CLICK"
|
||||
}
|
||||
|
||||
override fun onUpdate(
|
||||
context: Context,
|
||||
appWidgetManager: AppWidgetManager,
|
||||
@@ -17,22 +24,61 @@ class TranscriptWidgetProvider : AppWidgetProvider() {
|
||||
for (appWidgetId in appWidgetIds) {
|
||||
val views = RemoteViews(context.packageName, R.layout.widget_transcript)
|
||||
|
||||
// Tap widget → open main app
|
||||
val intent = Intent(context, MainActivity::class.java)
|
||||
val pendingIntent = PendingIntent.getActivity(
|
||||
context, 0, intent,
|
||||
// ListView adapter
|
||||
val serviceIntent = Intent(context, TerminalListWidgetService::class.java).apply {
|
||||
putExtra(AppWidgetManager.EXTRA_APPWIDGET_ID, appWidgetId)
|
||||
data = Uri.parse(toUri(Intent.URI_INTENT_SCHEME))
|
||||
}
|
||||
views.setRemoteAdapter(R.id.terminal_list, serviceIntent)
|
||||
views.setEmptyView(R.id.terminal_list, R.id.empty_view)
|
||||
|
||||
// Item click template → opens app
|
||||
val itemClickIntent = Intent(context, MainActivity::class.java).apply {
|
||||
action = ACTION_ITEM_CLICK
|
||||
}
|
||||
val itemClickPending = PendingIntent.getActivity(
|
||||
context, 0, itemClickIntent,
|
||||
PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_MUTABLE
|
||||
)
|
||||
views.setPendingIntentTemplate(R.id.terminal_list, itemClickPending)
|
||||
|
||||
// Refresh button
|
||||
val refreshIntent = Intent(context, TranscriptWidgetProvider::class.java).apply {
|
||||
action = ACTION_REFRESH
|
||||
}
|
||||
val refreshPending = PendingIntent.getBroadcast(
|
||||
context, 1, refreshIntent,
|
||||
PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE
|
||||
)
|
||||
views.setOnClickPendingIntent(R.id.widget_root, pendingIntent)
|
||||
views.setOnClickPendingIntent(R.id.btn_refresh, refreshPending)
|
||||
|
||||
// Title tap → open app
|
||||
val appIntent = Intent(context, MainActivity::class.java)
|
||||
val appPending = PendingIntent.getActivity(
|
||||
context, 2, appIntent,
|
||||
PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE
|
||||
)
|
||||
views.setOnClickPendingIntent(R.id.widget_title, appPending)
|
||||
|
||||
appWidgetManager.updateAppWidget(appWidgetId, views)
|
||||
}
|
||||
}
|
||||
|
||||
// Start periodic refresh worker
|
||||
TranscriptWidgetWorker.enqueue(context)
|
||||
override fun onReceive(context: Context, intent: Intent) {
|
||||
super.onReceive(context, intent)
|
||||
|
||||
if (intent.action == ACTION_REFRESH) {
|
||||
// Notify the ListView adapter to refresh its data
|
||||
val appWidgetManager = AppWidgetManager.getInstance(context)
|
||||
val widgetIds = appWidgetManager.getAppWidgetIds(
|
||||
ComponentName(context, TranscriptWidgetProvider::class.java)
|
||||
)
|
||||
appWidgetManager.notifyAppWidgetViewDataChanged(widgetIds, R.id.terminal_list)
|
||||
}
|
||||
}
|
||||
|
||||
override fun onEnabled(context: Context) {
|
||||
// Periodic refresh via WorkManager for background updates
|
||||
TranscriptWidgetWorker.enqueue(context)
|
||||
}
|
||||
|
||||
|
||||
@@ -3,14 +3,13 @@ package com.agentui.desktop
|
||||
import android.appwidget.AppWidgetManager
|
||||
import android.content.ComponentName
|
||||
import android.content.Context
|
||||
import android.widget.RemoteViews
|
||||
import androidx.work.*
|
||||
import okhttp3.OkHttpClient
|
||||
import okhttp3.Request
|
||||
import org.json.JSONArray
|
||||
import org.json.JSONObject
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
/**
|
||||
* Periodic background worker that triggers the widget ListView to refresh.
|
||||
* The actual data fetching happens in TerminalListFactory.onDataSetChanged().
|
||||
*/
|
||||
class TranscriptWidgetWorker(
|
||||
private val context: Context,
|
||||
params: WorkerParameters
|
||||
@@ -18,13 +17,6 @@ class TranscriptWidgetWorker(
|
||||
|
||||
companion object {
|
||||
const val WORK_NAME = "transcript_widget_refresh"
|
||||
private val MSG_IDS = intArrayOf(
|
||||
R.id.msg1, R.id.msg2, R.id.msg3, R.id.msg4, R.id.msg5
|
||||
)
|
||||
private val client = OkHttpClient.Builder()
|
||||
.connectTimeout(10, TimeUnit.SECONDS)
|
||||
.readTimeout(10, TimeUnit.SECONDS)
|
||||
.build()
|
||||
|
||||
fun enqueue(context: Context) {
|
||||
val request = OneTimeWorkRequestBuilder<TranscriptWidgetWorker>()
|
||||
@@ -39,108 +31,20 @@ class TranscriptWidgetWorker(
|
||||
}
|
||||
|
||||
override suspend fun doWork(): Result {
|
||||
val messages = fetchMessages()
|
||||
updateWidget(messages)
|
||||
scheduleNext()
|
||||
return Result.success()
|
||||
}
|
||||
|
||||
private fun fetchMessages(): List<String> {
|
||||
val apiBase = ServerConfig.apiBaseUrl(context) ?: return listOf("No server configured")
|
||||
|
||||
try {
|
||||
// Get most recent session for 'ejecutor' agent
|
||||
val sessionsUrl = "$apiBase/transcript-debug/sessions?agent=ejecutor"
|
||||
val sessionsReq = Request.Builder().url(sessionsUrl).build()
|
||||
val sessionsResp = client.newCall(sessionsReq).execute()
|
||||
if (!sessionsResp.isSuccessful) return listOf("Failed to fetch sessions")
|
||||
|
||||
val sessionsJson = JSONArray(sessionsResp.body?.string() ?: "[]")
|
||||
if (sessionsJson.length() == 0) return listOf("No sessions found")
|
||||
|
||||
// Get the most recent session (sorted by mtime desc from server)
|
||||
val latestSession = sessionsJson.getJSONObject(0)
|
||||
val sessionId = latestSession.getString("id")
|
||||
|
||||
// Fetch raw JSONL
|
||||
val rawUrl = "$apiBase/transcript-debug/$sessionId/raw?agent=ejecutor"
|
||||
val rawReq = Request.Builder().url(rawUrl).build()
|
||||
val rawResp = client.newCall(rawReq).execute()
|
||||
if (!rawResp.isSuccessful) return listOf("Failed to fetch transcript")
|
||||
|
||||
val rawText = rawResp.body?.string() ?: return listOf("Empty transcript")
|
||||
return parseJsonlMessages(rawText)
|
||||
} catch (e: Exception) {
|
||||
return listOf("Error: ${e.message?.take(40)}")
|
||||
}
|
||||
}
|
||||
|
||||
private fun parseJsonlMessages(jsonl: String): List<String> {
|
||||
val messages = mutableListOf<String>()
|
||||
val lines = jsonl.trim().split("\n").filter { it.isNotBlank() }
|
||||
|
||||
for (line in lines) {
|
||||
try {
|
||||
val obj = JSONObject(line)
|
||||
val type = obj.optString("type", "")
|
||||
|
||||
when (type) {
|
||||
"human" -> {
|
||||
val content = obj.optString("message", "")
|
||||
.ifEmpty { extractMessageContent(obj) }
|
||||
if (content.isNotEmpty()) {
|
||||
messages.add("> ${content.take(80)}")
|
||||
}
|
||||
}
|
||||
"assistant" -> {
|
||||
val content = extractMessageContent(obj)
|
||||
if (content.isNotEmpty()) {
|
||||
messages.add("< ${content.take(80)}")
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (_: Exception) { }
|
||||
}
|
||||
|
||||
return messages.takeLast(5).ifEmpty { listOf("No messages yet") }
|
||||
}
|
||||
|
||||
private fun extractMessageContent(obj: JSONObject): String {
|
||||
// Try "message" field first (simple string)
|
||||
val msg = obj.optString("message", "")
|
||||
if (msg.isNotEmpty()) return msg
|
||||
|
||||
// Try "message" as array of content blocks
|
||||
val msgArray = obj.optJSONArray("message")
|
||||
if (msgArray != null) {
|
||||
for (i in 0 until msgArray.length()) {
|
||||
val block = msgArray.optJSONObject(i) ?: continue
|
||||
if (block.optString("type") == "text") {
|
||||
return block.optString("text", "")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
private fun updateWidget(messages: List<String>) {
|
||||
val views = RemoteViews(context.packageName, R.layout.widget_transcript)
|
||||
|
||||
for (i in MSG_IDS.indices) {
|
||||
views.setTextViewText(MSG_IDS[i], messages.getOrElse(i) { "" })
|
||||
}
|
||||
|
||||
// Tell the widget's ListView adapter to re-fetch data
|
||||
val appWidgetManager = AppWidgetManager.getInstance(context)
|
||||
val widgetComponent = ComponentName(context, TranscriptWidgetProvider::class.java)
|
||||
appWidgetManager.updateAppWidget(widgetComponent, views)
|
||||
}
|
||||
val widgetIds = appWidgetManager.getAppWidgetIds(
|
||||
ComponentName(context, TranscriptWidgetProvider::class.java)
|
||||
)
|
||||
appWidgetManager.notifyAppWidgetViewDataChanged(widgetIds, R.id.terminal_list)
|
||||
|
||||
private fun scheduleNext() {
|
||||
val request = OneTimeWorkRequestBuilder<TranscriptWidgetWorker>()
|
||||
// Schedule next refresh
|
||||
val next = OneTimeWorkRequestBuilder<TranscriptWidgetWorker>()
|
||||
.setInitialDelay(30, TimeUnit.SECONDS)
|
||||
.build()
|
||||
WorkManager.getInstance(context)
|
||||
.enqueueUniqueWork(WORK_NAME, ExistingWorkPolicy.REPLACE, request)
|
||||
.enqueueUniqueWork(WORK_NAME, ExistingWorkPolicy.REPLACE, next)
|
||||
|
||||
return Result.success()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,7 +10,9 @@ import okhttp3.*
|
||||
import okhttp3.MediaType.Companion.toMediaType
|
||||
import okhttp3.RequestBody.Companion.toRequestBody
|
||||
import org.json.JSONObject
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.text.SimpleDateFormat
|
||||
import java.util.Date
|
||||
import java.util.Locale
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
class VoiceCommandActivity : Activity() {
|
||||
@@ -27,28 +29,44 @@ class VoiceCommandActivity : Activity() {
|
||||
|
||||
override fun onCreate(savedInstanceState: Bundle?) {
|
||||
super.onCreate(savedInstanceState)
|
||||
Log.d(TAG, "VoiceCommandActivity created, action=${intent?.action}")
|
||||
|
||||
// Check if launched via share intent
|
||||
if (intent?.action == Intent.ACTION_SEND && intent.type == "text/plain") {
|
||||
val sharedText = intent.getStringExtra(Intent.EXTRA_TEXT)
|
||||
if (!sharedText.isNullOrBlank()) {
|
||||
sendToServer(sharedText)
|
||||
when (intent?.action) {
|
||||
// Launched via share intent
|
||||
Intent.ACTION_SEND -> {
|
||||
if (intent.type == "text/plain") {
|
||||
val sharedText = intent.getStringExtra(Intent.EXTRA_TEXT)
|
||||
if (!sharedText.isNullOrBlank()) {
|
||||
sendTranscript(sharedText, "share-intent")
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
// Launched as assist app or voice command
|
||||
Intent.ACTION_ASSIST,
|
||||
Intent.ACTION_VOICE_COMMAND,
|
||||
"android.intent.action.VOICE_COMMAND" -> {
|
||||
Log.d(TAG, "Launched as assistant, starting speech recognition")
|
||||
startSpeechRecognition()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Otherwise, start speech recognition
|
||||
// Default fallback: start speech recognition
|
||||
startSpeechRecognition()
|
||||
}
|
||||
|
||||
private fun startSpeechRecognition() {
|
||||
val intent = Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply {
|
||||
putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM)
|
||||
putExtra(RecognizerIntent.EXTRA_PROMPT, "Agent UI - Speak your command")
|
||||
putExtra(RecognizerIntent.EXTRA_LANGUAGE, "es-HN")
|
||||
putExtra(RecognizerIntent.EXTRA_LANGUAGE_PREFERENCE, "es-HN")
|
||||
putExtra(RecognizerIntent.EXTRA_PROMPT, "Agent UI - Decí tu comando")
|
||||
}
|
||||
try {
|
||||
startActivityForResult(intent, SPEECH_REQUEST_CODE)
|
||||
} catch (e: Exception) {
|
||||
Log.e(TAG, "Speech recognition not available", e)
|
||||
Toast.makeText(this, "Speech recognition not available", Toast.LENGTH_SHORT).show()
|
||||
finish()
|
||||
}
|
||||
@@ -62,95 +80,90 @@ class VoiceCommandActivity : Activity() {
|
||||
val results = data.getStringArrayListExtra(RecognizerIntent.EXTRA_RESULTS)
|
||||
val text = results?.firstOrNull()
|
||||
if (!text.isNullOrBlank()) {
|
||||
sendToServer(text)
|
||||
Log.d(TAG, "Speech recognized: $text")
|
||||
sendTranscript(text, "voice-assistant")
|
||||
return
|
||||
}
|
||||
}
|
||||
Log.d(TAG, "Speech recognition cancelled or no result")
|
||||
finish()
|
||||
}
|
||||
}
|
||||
|
||||
private fun sendToServer(text: String) {
|
||||
/**
|
||||
* Sends the transcribed text to the server's /api/voice-transcript endpoint,
|
||||
* then launches MainActivity with the target terminal so the floating
|
||||
* transcript debug opens on the correct session.
|
||||
*/
|
||||
private fun sendTranscript(text: String, source: String) {
|
||||
Thread {
|
||||
try {
|
||||
val terminalBase = ServerConfig.terminalBaseUrl(this)
|
||||
val terminalWs = ServerConfig.terminalWsUrl(this)
|
||||
val apiBase = ServerConfig.apiBaseUrl(this)
|
||||
|
||||
if (terminalBase == null || terminalWs == null) {
|
||||
if (apiBase == null) {
|
||||
Log.w(TAG, "No server configured, cannot send transcript")
|
||||
showToastAndFinish("No server configured")
|
||||
return@Thread
|
||||
}
|
||||
|
||||
// 1. Create a new terminal session
|
||||
val createBody = JSONObject().apply {
|
||||
put("agent", "ejecutor")
|
||||
put("transcriptSessionId", "__new__")
|
||||
put("label", text.take(60))
|
||||
put("command", "ejecutor")
|
||||
val timestamp = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US)
|
||||
.format(Date())
|
||||
|
||||
val body = JSONObject().apply {
|
||||
put("text", text)
|
||||
put("timestamp", timestamp)
|
||||
put("source", source)
|
||||
}.toString().toRequestBody("application/json".toMediaType())
|
||||
|
||||
val createReq = Request.Builder()
|
||||
.url("$terminalBase/create-terminal")
|
||||
.post(createBody)
|
||||
val request = Request.Builder()
|
||||
.url("$apiBase/voice-transcript")
|
||||
.post(body)
|
||||
.build()
|
||||
|
||||
val createResp = client.newCall(createReq).execute()
|
||||
if (!createResp.isSuccessful) {
|
||||
showToastAndFinish("Failed to create terminal")
|
||||
return@Thread
|
||||
Log.d(TAG, "Sending transcript to $apiBase/voice-transcript")
|
||||
|
||||
val response = client.newCall(request).execute()
|
||||
val responseBody = response.body?.string() ?: ""
|
||||
|
||||
if (response.isSuccessful) {
|
||||
Log.d(TAG, "Server response: $responseBody")
|
||||
|
||||
// Parse the ephemeralSessionId from the response
|
||||
val json = JSONObject(responseBody)
|
||||
val terminal = json.optString("ephemeralSessionId", "")
|
||||
|
||||
// Launch MainActivity with the terminal info to open floating transcript
|
||||
openMainWithTerminal(terminal, text)
|
||||
} else {
|
||||
Log.e(TAG, "Server error ${response.code}: $responseBody")
|
||||
showToastAndFinish("Server error: ${response.code}")
|
||||
}
|
||||
|
||||
val respJson = JSONObject(createResp.body?.string() ?: "{}")
|
||||
val sessionId = respJson.optString("ephemeralSessionId", "")
|
||||
if (sessionId.isEmpty()) {
|
||||
showToastAndFinish("No session ID returned")
|
||||
return@Thread
|
||||
}
|
||||
|
||||
// 2. Connect via WebSocket and send the input
|
||||
val wsUrl = "$terminalWs/ws/terminal?session=$sessionId"
|
||||
val wsReq = Request.Builder().url(wsUrl).build()
|
||||
val latch = CountDownLatch(1)
|
||||
|
||||
client.newWebSocket(wsReq, object : WebSocketListener() {
|
||||
override fun onOpen(webSocket: WebSocket, response: Response) {
|
||||
// Send the text as input
|
||||
val inputMsg = JSONObject().apply {
|
||||
put("type", "input")
|
||||
put("data", text)
|
||||
}.toString()
|
||||
webSocket.send(inputMsg)
|
||||
|
||||
// Send Enter key after a short delay
|
||||
Thread.sleep(80)
|
||||
val enterMsg = JSONObject().apply {
|
||||
put("type", "input")
|
||||
put("data", "\r")
|
||||
}.toString()
|
||||
webSocket.send(enterMsg)
|
||||
|
||||
// Close after sending
|
||||
Thread.sleep(200)
|
||||
webSocket.close(1000, "done")
|
||||
latch.countDown()
|
||||
}
|
||||
|
||||
override fun onFailure(webSocket: WebSocket, t: Throwable, response: Response?) {
|
||||
Log.e(TAG, "WebSocket failed", t)
|
||||
latch.countDown()
|
||||
}
|
||||
})
|
||||
|
||||
latch.await(5, TimeUnit.SECONDS)
|
||||
showToastAndFinish("Sent: ${text.take(40)}")
|
||||
|
||||
} catch (e: Exception) {
|
||||
Log.e(TAG, "Failed to send command", e)
|
||||
showToastAndFinish("Error: ${e.message?.take(40)}")
|
||||
Log.e(TAG, "Failed to send transcript", e)
|
||||
showToastAndFinish("Error: ${e.message?.take(50)}")
|
||||
}
|
||||
}.start()
|
||||
}
|
||||
|
||||
/**
|
||||
* Opens MainActivity and tells the WebView to open FloatingTranscriptDebug
|
||||
* on the terminal that received the voice command.
|
||||
*/
|
||||
private fun openMainWithTerminal(terminal: String, text: String) {
|
||||
runOnUiThread {
|
||||
Toast.makeText(this, "Sent: ${text.take(50)}", Toast.LENGTH_SHORT).show()
|
||||
|
||||
val intent = Intent(this, MainActivity::class.java).apply {
|
||||
action = "com.agentui.desktop.VOICE_TERMINAL"
|
||||
putExtra("ephemeralSessionId", terminal)
|
||||
addFlags(Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_CLEAR_TOP)
|
||||
}
|
||||
startActivity(intent)
|
||||
finish()
|
||||
}
|
||||
}
|
||||
|
||||
private fun showToastAndFinish(message: String) {
|
||||
runOnUiThread {
|
||||
Toast.makeText(this, message, Toast.LENGTH_SHORT).show()
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<shape xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
android:shape="rectangle">
|
||||
<solid android:color="#ff171717" />
|
||||
<corners android:radius="22dp" />
|
||||
</shape>
|
||||
@@ -0,0 +1,6 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<shape xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
android:shape="rectangle">
|
||||
<solid android:color="#99010101" />
|
||||
<corners android:radius="22dp" />
|
||||
</shape>
|
||||
@@ -0,0 +1,124 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="match_parent"
|
||||
android:orientation="vertical"
|
||||
android:padding="12dp"
|
||||
android:background="@drawable/face_widget_bg_aod">
|
||||
|
||||
<!-- Minimal AOD layout: title + terminal count -->
|
||||
<TextView
|
||||
android:id="@+id/fw_aod_title"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:text="Agent UI"
|
||||
android:textColor="#555588"
|
||||
android:textSize="11sp"
|
||||
android:fontFamily="monospace"
|
||||
android:paddingBottom="4dp" />
|
||||
|
||||
<!-- Terminal 1 -->
|
||||
<LinearLayout
|
||||
android:id="@+id/fw_aod_terminal_1"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:orientation="horizontal"
|
||||
android:gravity="center_vertical"
|
||||
android:paddingTop="1dp"
|
||||
android:paddingBottom="1dp"
|
||||
android:visibility="gone">
|
||||
|
||||
<TextView
|
||||
android:id="@+id/fw_aod_dot_1"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:text="\u25CF"
|
||||
android:textColor="#444466"
|
||||
android:textSize="7sp"
|
||||
android:paddingEnd="4dp" />
|
||||
|
||||
<TextView
|
||||
android:id="@+id/fw_aod_name_1"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:textColor="#777777"
|
||||
android:textSize="9sp"
|
||||
android:fontFamily="monospace"
|
||||
android:maxLines="1"
|
||||
android:ellipsize="end" />
|
||||
</LinearLayout>
|
||||
|
||||
<!-- Terminal 2 -->
|
||||
<LinearLayout
|
||||
android:id="@+id/fw_aod_terminal_2"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:orientation="horizontal"
|
||||
android:gravity="center_vertical"
|
||||
android:paddingTop="1dp"
|
||||
android:paddingBottom="1dp"
|
||||
android:visibility="gone">
|
||||
|
||||
<TextView
|
||||
android:id="@+id/fw_aod_dot_2"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:text="\u25CF"
|
||||
android:textColor="#444466"
|
||||
android:textSize="7sp"
|
||||
android:paddingEnd="4dp" />
|
||||
|
||||
<TextView
|
||||
android:id="@+id/fw_aod_name_2"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:textColor="#777777"
|
||||
android:textSize="9sp"
|
||||
android:fontFamily="monospace"
|
||||
android:maxLines="1"
|
||||
android:ellipsize="end" />
|
||||
</LinearLayout>
|
||||
|
||||
<!-- Terminal 3 -->
|
||||
<LinearLayout
|
||||
android:id="@+id/fw_aod_terminal_3"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:orientation="horizontal"
|
||||
android:gravity="center_vertical"
|
||||
android:paddingTop="1dp"
|
||||
android:paddingBottom="1dp"
|
||||
android:visibility="gone">
|
||||
|
||||
<TextView
|
||||
android:id="@+id/fw_aod_dot_3"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:text="\u25CF"
|
||||
android:textColor="#444466"
|
||||
android:textSize="7sp"
|
||||
android:paddingEnd="4dp" />
|
||||
|
||||
<TextView
|
||||
android:id="@+id/fw_aod_name_3"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:textColor="#777777"
|
||||
android:textSize="9sp"
|
||||
android:fontFamily="monospace"
|
||||
android:maxLines="1"
|
||||
android:ellipsize="end" />
|
||||
</LinearLayout>
|
||||
|
||||
<!-- Empty state -->
|
||||
<TextView
|
||||
android:id="@+id/fw_aod_empty"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="match_parent"
|
||||
android:text="No agents"
|
||||
android:textColor="#444444"
|
||||
android:textSize="9sp"
|
||||
android:fontFamily="monospace"
|
||||
android:gravity="center" />
|
||||
|
||||
</LinearLayout>
|
||||
@@ -0,0 +1,140 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="match_parent"
|
||||
android:orientation="vertical"
|
||||
android:padding="12dp"
|
||||
android:background="@drawable/face_widget_bg_dark">
|
||||
|
||||
<!-- Title bar -->
|
||||
<LinearLayout
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:orientation="horizontal"
|
||||
android:gravity="center_vertical"
|
||||
android:paddingBottom="6dp">
|
||||
|
||||
<TextView
|
||||
android:id="@+id/fw_title"
|
||||
android:layout_width="0dp"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_weight="1"
|
||||
android:text="Agent UI"
|
||||
android:textColor="#8888FF"
|
||||
android:textSize="12sp"
|
||||
android:fontFamily="monospace" />
|
||||
|
||||
<TextView
|
||||
android:id="@+id/fw_status"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:textColor="#888888"
|
||||
android:textSize="10sp"
|
||||
android:fontFamily="monospace" />
|
||||
</LinearLayout>
|
||||
|
||||
<!-- Terminal 1 -->
|
||||
<LinearLayout
|
||||
android:id="@+id/fw_terminal_1"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:orientation="horizontal"
|
||||
android:gravity="center_vertical"
|
||||
android:paddingTop="2dp"
|
||||
android:paddingBottom="2dp"
|
||||
android:visibility="gone">
|
||||
|
||||
<TextView
|
||||
android:id="@+id/fw_dot_1"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:text="\u25CF"
|
||||
android:textColor="#6b7280"
|
||||
android:textSize="8sp"
|
||||
android:paddingEnd="4dp" />
|
||||
|
||||
<TextView
|
||||
android:id="@+id/fw_name_1"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:textColor="#DDDDDD"
|
||||
android:textSize="10sp"
|
||||
android:fontFamily="monospace"
|
||||
android:maxLines="1"
|
||||
android:ellipsize="end" />
|
||||
</LinearLayout>
|
||||
|
||||
<!-- Terminal 2 -->
|
||||
<LinearLayout
|
||||
android:id="@+id/fw_terminal_2"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:orientation="horizontal"
|
||||
android:gravity="center_vertical"
|
||||
android:paddingTop="2dp"
|
||||
android:paddingBottom="2dp"
|
||||
android:visibility="gone">
|
||||
|
||||
<TextView
|
||||
android:id="@+id/fw_dot_2"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:text="\u25CF"
|
||||
android:textColor="#6b7280"
|
||||
android:textSize="8sp"
|
||||
android:paddingEnd="4dp" />
|
||||
|
||||
<TextView
|
||||
android:id="@+id/fw_name_2"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:textColor="#DDDDDD"
|
||||
android:textSize="10sp"
|
||||
android:fontFamily="monospace"
|
||||
android:maxLines="1"
|
||||
android:ellipsize="end" />
|
||||
</LinearLayout>
|
||||
|
||||
<!-- Terminal 3 -->
|
||||
<LinearLayout
|
||||
android:id="@+id/fw_terminal_3"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:orientation="horizontal"
|
||||
android:gravity="center_vertical"
|
||||
android:paddingTop="2dp"
|
||||
android:paddingBottom="2dp"
|
||||
android:visibility="gone">
|
||||
|
||||
<TextView
|
||||
android:id="@+id/fw_dot_3"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:text="\u25CF"
|
||||
android:textColor="#6b7280"
|
||||
android:textSize="8sp"
|
||||
android:paddingEnd="4dp" />
|
||||
|
||||
<TextView
|
||||
android:id="@+id/fw_name_3"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:textColor="#DDDDDD"
|
||||
android:textSize="10sp"
|
||||
android:fontFamily="monospace"
|
||||
android:maxLines="1"
|
||||
android:ellipsize="end" />
|
||||
</LinearLayout>
|
||||
|
||||
<!-- Empty state -->
|
||||
<TextView
|
||||
android:id="@+id/fw_empty"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="match_parent"
|
||||
android:text="No agents active"
|
||||
android:textColor="#666666"
|
||||
android:textSize="10sp"
|
||||
android:fontFamily="monospace"
|
||||
android:gravity="center" />
|
||||
|
||||
</LinearLayout>
|
||||
@@ -0,0 +1,64 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
android:id="@+id/item_root"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:orientation="vertical"
|
||||
android:paddingTop="3dp"
|
||||
android:paddingBottom="3dp"
|
||||
android:background="?android:attr/selectableItemBackground">
|
||||
|
||||
<!-- Header: dot + agent name + status + badges -->
|
||||
<LinearLayout
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:orientation="horizontal"
|
||||
android:gravity="center_vertical">
|
||||
|
||||
<!-- Status dot -->
|
||||
<TextView
|
||||
android:id="@+id/item_dot"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:text="\u25CF"
|
||||
android:textColor="#6b7280"
|
||||
android:textSize="8sp"
|
||||
android:paddingEnd="4dp" />
|
||||
|
||||
<!-- Agent name + status -->
|
||||
<TextView
|
||||
android:id="@+id/item_name"
|
||||
android:layout_width="0dp"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_weight="1"
|
||||
android:textColor="#DDDDDD"
|
||||
android:textSize="10sp"
|
||||
android:fontFamily="monospace"
|
||||
android:maxLines="1"
|
||||
android:ellipsize="end" />
|
||||
|
||||
<!-- Hook badges -->
|
||||
<TextView
|
||||
android:id="@+id/item_badges"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:textColor="#888888"
|
||||
android:textSize="9sp"
|
||||
android:fontFamily="monospace"
|
||||
android:paddingStart="4dp" />
|
||||
</LinearLayout>
|
||||
|
||||
<!-- Label / last user prompt -->
|
||||
<TextView
|
||||
android:id="@+id/item_label"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:textColor="#9999CC"
|
||||
android:textSize="10sp"
|
||||
android:fontFamily="monospace"
|
||||
android:maxLines="2"
|
||||
android:ellipsize="end"
|
||||
android:paddingStart="12dp"
|
||||
android:paddingTop="1dp" />
|
||||
|
||||
</LinearLayout>
|
||||
@@ -4,67 +4,57 @@
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="match_parent"
|
||||
android:orientation="vertical"
|
||||
android:padding="12dp"
|
||||
android:padding="10dp"
|
||||
android:background="#DD1A1A2E">
|
||||
|
||||
<TextView
|
||||
android:id="@+id/widget_title"
|
||||
<!-- Title bar -->
|
||||
<LinearLayout
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:text="Agent UI Transcript"
|
||||
android:textColor="#8888FF"
|
||||
android:textSize="12sp"
|
||||
android:fontFamily="monospace"
|
||||
android:paddingBottom="4dp" />
|
||||
android:orientation="horizontal"
|
||||
android:gravity="center_vertical"
|
||||
android:paddingBottom="4dp">
|
||||
|
||||
<TextView
|
||||
android:id="@+id/msg1"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:textColor="#CCCCCC"
|
||||
android:textSize="11sp"
|
||||
android:fontFamily="monospace"
|
||||
android:maxLines="1"
|
||||
android:ellipsize="end" />
|
||||
<TextView
|
||||
android:id="@+id/widget_title"
|
||||
android:layout_width="0dp"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_weight="1"
|
||||
android:text="Agent UI"
|
||||
android:textColor="#8888FF"
|
||||
android:textSize="11sp"
|
||||
android:fontFamily="monospace" />
|
||||
|
||||
<TextView
|
||||
android:id="@+id/msg2"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:textColor="#CCCCCC"
|
||||
android:textSize="11sp"
|
||||
android:fontFamily="monospace"
|
||||
android:maxLines="1"
|
||||
android:ellipsize="end" />
|
||||
<TextView
|
||||
android:id="@+id/btn_refresh"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:text="\u21BB"
|
||||
android:textColor="#8888FF"
|
||||
android:textSize="14sp"
|
||||
android:paddingStart="8dp"
|
||||
android:paddingEnd="4dp"
|
||||
android:background="?android:attr/selectableItemBackground" />
|
||||
</LinearLayout>
|
||||
|
||||
<TextView
|
||||
android:id="@+id/msg3"
|
||||
<!-- Terminal list (dynamic, scrollable) -->
|
||||
<ListView
|
||||
android:id="@+id/terminal_list"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:textColor="#CCCCCC"
|
||||
android:textSize="11sp"
|
||||
android:fontFamily="monospace"
|
||||
android:maxLines="1"
|
||||
android:ellipsize="end" />
|
||||
android:layout_height="match_parent"
|
||||
android:divider="@null"
|
||||
android:dividerHeight="0dp"
|
||||
android:scrollbars="none" />
|
||||
|
||||
<!-- Fallback when empty -->
|
||||
<TextView
|
||||
android:id="@+id/msg4"
|
||||
android:id="@+id/empty_view"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:textColor="#CCCCCC"
|
||||
android:textSize="11sp"
|
||||
android:layout_height="match_parent"
|
||||
android:text="No terminals open"
|
||||
android:textColor="#666666"
|
||||
android:textSize="10sp"
|
||||
android:fontFamily="monospace"
|
||||
android:maxLines="1"
|
||||
android:ellipsize="end" />
|
||||
|
||||
<TextView
|
||||
android:id="@+id/msg5"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:textColor="#CCCCCC"
|
||||
android:textSize="11sp"
|
||||
android:fontFamily="monospace"
|
||||
android:maxLines="1"
|
||||
android:ellipsize="end" />
|
||||
android:gravity="center" />
|
||||
|
||||
</LinearLayout>
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"agent_ui_transcript": {
|
||||
"menuInSetting": 1,
|
||||
"labelResNameInSetting": "face_widget_label",
|
||||
"actionDetailSetting": "com.agentui.desktop.FACE_WIDGET_SETTINGS"
|
||||
}
|
||||
}
|
||||
@@ -2,4 +2,5 @@
|
||||
<string name="app_name">Agent UI</string>
|
||||
<string name="main_activity_title">Agent UI</string>
|
||||
<string name="widget_description">Shows recent transcript messages from Agent UI</string>
|
||||
<string name="face_widget_label">Agent UI Terminals</string>
|
||||
</resources>
|
||||
@@ -0,0 +1,4 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<recognition-service
|
||||
xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
android:settingsActivity="com.agentui.desktop.MainActivity" />
|
||||
@@ -1,9 +1,9 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<appwidget-provider xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
android:minWidth="250dp"
|
||||
android:minHeight="110dp"
|
||||
android:minHeight="140dp"
|
||||
android:updatePeriodMillis="1800000"
|
||||
android:initialLayout="@layout/widget_transcript"
|
||||
android:resizeMode="horizontal|vertical"
|
||||
android:widgetCategory="home_screen"
|
||||
android:widgetCategory="home_screen|keyguard"
|
||||
android:description="@string/widget_description" />
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<voice-interaction-service xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
android:sessionService="com.agentui.desktop.AgentVoiceInteractionSessionService"
|
||||
android:recognitionService="com.agentui.desktop.AgentRecognitionService"
|
||||
android:supportsAssist="true"
|
||||
android:supportsLaunchVoiceAssistFromKeyguard="true"
|
||||
android:supportsLocalInteraction="true"
|
||||
android:settingsActivity="com.agentui.desktop.MainActivity" />
|
||||
Reference in New Issue
Block a user