feat: integrate Tauri v2 with Android widget and voice assistant

- Add Tauri v2 shell (Cargo, tauri.conf.json, capabilities, plugins) - Migrate all fetch() calls to apiFetch() for Tauri-aware HTTP - Migrate WebSocket endpoints to resolveEndpoints() for dynamic URLs - Add ServerConfigDialog for remote server URL configuration - Add tauri.ts lib with isTauri detection, apiFetch wrapper, plugin helpers - Add server-config Pinia store with persistence via plugin-store - Conditional PWA (disabled in Tauri builds) - Android: home screen transcript widget (last 5 messages, 30s refresh) - Android: voice command / share activity (SpeechRecognizer + WebSocket) - Android: signed release APK with auto-copy to installers/ - Remove stale frontend/src-tauri directory
2026-02-23 15:33:43 -06:00
parent 6dc0c5ff6f
commit e1aa8b1bdb
108 changed files with 8155 additions and 151 deletions
--- a/src-tauri/gen/android/app/src/main/AndroidManifest.xml
+++ b/src-tauri/gen/android/app/src/main/AndroidManifest.xml
@@ -0,0 +1,63 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android">
+    <uses-permission android:name="android.permission.INTERNET" />
+    <uses-permission android:name="android.permission.RECORD_AUDIO" />
+
+    <!-- AndroidTV support -->
+    <uses-feature android:name="android.software.leanback" android:required="false" />
+
+    <application
+        android:icon="@mipmap/ic_launcher"
+        android:label="@string/app_name"
+        android:theme="@style/Theme.agent_ui"
+        android:usesCleartextTraffic="${usesCleartextTraffic}">
+        <activity
+            android:configChanges="orientation|keyboardHidden|keyboard|screenSize|locale|smallestScreenSize|screenLayout|uiMode"
+            android:launchMode="singleTask"
+            android:label="@string/main_activity_title"
+            android:name=".MainActivity"
+            android:exported="true">
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN" />
+                <category android:name="android.intent.category.LAUNCHER" />
+                <!-- AndroidTV support -->
+                <category android:name="android.intent.category.LEANBACK_LAUNCHER" />
+            </intent-filter>
+        </activity>
+
+        <!-- Transcript Widget -->
+        <receiver
+            android:name=".TranscriptWidgetProvider"
+            android:exported="false">
+            <intent-filter>
+                <action android:name="android.appwidget.action.APPWIDGET_UPDATE" />
+            </intent-filter>
+            <meta-data
+                android:name="android.appwidget.provider"
+                android:resource="@xml/transcript_widget_info" />
+        </receiver>
+
+        <!-- Voice Command / Share Activity -->
+        <activity
+            android:name=".VoiceCommandActivity"
+            android:label="Agent UI Voice"
+            android:theme="@android:style/Theme.Translucent.NoTitleBar"
+            android:exported="true">
+            <intent-filter>
+                <action android:name="android.intent.action.SEND" />
+                <category android:name="android.intent.category.DEFAULT" />
+                <data android:mimeType="text/plain" />
+            </intent-filter>
+        </activity>
+
+        <provider
+          android:name="androidx.core.content.FileProvider"
+          android:authorities="${applicationId}.fileprovider"
+          android:exported="false"
+          android:grantUriPermissions="true">
+          <meta-data
+            android:name="android.support.FILE_PROVIDER_PATHS"
+            android:resource="@xml/file_paths" />
+        </provider>
+    </application>
+</manifest>
--- a/src-tauri/gen/android/app/src/main/java/com/agentui/desktop/MainActivity.kt
+++ b/src-tauri/gen/android/app/src/main/java/com/agentui/desktop/MainActivity.kt
@@ -0,0 +1,40 @@
+package com.agentui.desktop
+
+import android.os.Bundle
+import android.util.Log
+import androidx.activity.enableEdgeToEdge
+import org.json.JSONObject
+import java.io.File
+
+class MainActivity : TauriActivity() {
+    override fun onCreate(savedInstanceState: Bundle?) {
+        enableEdgeToEdge()
+        super.onCreate(savedInstanceState)
+        syncServerUrlToPrefs()
+    }
+
+    override fun onResume() {
+        super.onResume()
+        syncServerUrlToPrefs()
+    }
+
+    /**
+     * Reads the Tauri plugin-store settings.json and syncs serverUrl
+     * to SharedPreferences so native components (widget, voice) can use it.
+     */
+    private fun syncServerUrlToPrefs() {
+        try {
+            val storeFile = File(filesDir, "app_tauri-plugin-store/settings.json")
+            if (!storeFile.exists()) return
+
+            val json = JSONObject(storeFile.readText())
+            val serverUrl = json.optString("serverUrl", "")
+            if (serverUrl.isNotEmpty()) {
+                ServerConfig.setServerUrl(this, serverUrl)
+                Log.d("AgentUI", "Synced serverUrl to prefs: $serverUrl")
+            }
+        } catch (e: Exception) {
+            Log.w("AgentUI", "Failed to sync server URL", e)
+        }
+    }
+}
--- a/src-tauri/gen/android/app/src/main/java/com/agentui/desktop/ServerConfig.kt
+++ b/src-tauri/gen/android/app/src/main/java/com/agentui/desktop/ServerConfig.kt
@@ -0,0 +1,40 @@
+package com.agentui.desktop
+
+import android.content.Context
+
+object ServerConfig {
+    private const val PREFS_NAME = "agent_ui_config"
+    private const val KEY_SERVER_URL = "server_url"
+
+    fun getServerUrl(context: Context): String? {
+        return context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE)
+            .getString(KEY_SERVER_URL, null)
+    }
+
+    fun setServerUrl(context: Context, url: String) {
+        context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE)
+            .edit()
+            .putString(KEY_SERVER_URL, url.trimEnd('/'))
+            .apply()
+    }
+
+    /** e.g. "http://192.168.1.10:4103" */
+    fun terminalBaseUrl(context: Context): String? {
+        val base = getServerUrl(context) ?: return null
+        val uri = android.net.Uri.parse(base)
+        val host = uri.host ?: return null
+        return "${uri.scheme}://$host:4103"
+    }
+
+    /** e.g. "ws://192.168.1.10:4103" */
+    fun terminalWsUrl(context: Context): String? {
+        val base = terminalBaseUrl(context) ?: return null
+        return base.replace("http://", "ws://").replace("https://", "wss://")
+    }
+
+    /** e.g. "http://192.168.1.10:4100/api" */
+    fun apiBaseUrl(context: Context): String? {
+        val base = getServerUrl(context) ?: return null
+        return "${base.trimEnd('/')}/api"
+    }
+}
--- a/src-tauri/gen/android/app/src/main/java/com/agentui/desktop/TranscriptWidgetProvider.kt
+++ b/src-tauri/gen/android/app/src/main/java/com/agentui/desktop/TranscriptWidgetProvider.kt
@@ -0,0 +1,42 @@
+package com.agentui.desktop
+
+import android.app.PendingIntent
+import android.appwidget.AppWidgetManager
+import android.appwidget.AppWidgetProvider
+import android.content.Context
+import android.content.Intent
+import android.widget.RemoteViews
+
+class TranscriptWidgetProvider : AppWidgetProvider() {
+
+    override fun onUpdate(
+        context: Context,
+        appWidgetManager: AppWidgetManager,
+        appWidgetIds: IntArray
+    ) {
+        for (appWidgetId in appWidgetIds) {
+            val views = RemoteViews(context.packageName, R.layout.widget_transcript)
+
+            // Tap widget → open main app
+            val intent = Intent(context, MainActivity::class.java)
+            val pendingIntent = PendingIntent.getActivity(
+                context, 0, intent,
+                PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE
+            )
+            views.setOnClickPendingIntent(R.id.widget_root, pendingIntent)
+
+            appWidgetManager.updateAppWidget(appWidgetId, views)
+        }
+
+        // Start periodic refresh worker
+        TranscriptWidgetWorker.enqueue(context)
+    }
+
+    override fun onEnabled(context: Context) {
+        TranscriptWidgetWorker.enqueue(context)
+    }
+
+    override fun onDisabled(context: Context) {
+        TranscriptWidgetWorker.cancel(context)
+    }
+}
--- a/src-tauri/gen/android/app/src/main/java/com/agentui/desktop/TranscriptWidgetWorker.kt
+++ b/src-tauri/gen/android/app/src/main/java/com/agentui/desktop/TranscriptWidgetWorker.kt
@@ -0,0 +1,146 @@
+package com.agentui.desktop
+
+import android.appwidget.AppWidgetManager
+import android.content.ComponentName
+import android.content.Context
+import android.widget.RemoteViews
+import androidx.work.*
+import okhttp3.OkHttpClient
+import okhttp3.Request
+import org.json.JSONArray
+import org.json.JSONObject
+import java.util.concurrent.TimeUnit
+
+class TranscriptWidgetWorker(
+    private val context: Context,
+    params: WorkerParameters
+) : CoroutineWorker(context, params) {
+
+    companion object {
+        const val WORK_NAME = "transcript_widget_refresh"
+        private val MSG_IDS = intArrayOf(
+            R.id.msg1, R.id.msg2, R.id.msg3, R.id.msg4, R.id.msg5
+        )
+        private val client = OkHttpClient.Builder()
+            .connectTimeout(10, TimeUnit.SECONDS)
+            .readTimeout(10, TimeUnit.SECONDS)
+            .build()
+
+        fun enqueue(context: Context) {
+            val request = OneTimeWorkRequestBuilder<TranscriptWidgetWorker>()
+                .build()
+            WorkManager.getInstance(context)
+                .enqueueUniqueWork(WORK_NAME, ExistingWorkPolicy.REPLACE, request)
+        }
+
+        fun cancel(context: Context) {
+            WorkManager.getInstance(context).cancelUniqueWork(WORK_NAME)
+        }
+    }
+
+    override suspend fun doWork(): Result {
+        val messages = fetchMessages()
+        updateWidget(messages)
+        scheduleNext()
+        return Result.success()
+    }
+
+    private fun fetchMessages(): List<String> {
+        val apiBase = ServerConfig.apiBaseUrl(context) ?: return listOf("No server configured")
+
+        try {
+            // Get most recent session for 'ejecutor' agent
+            val sessionsUrl = "$apiBase/transcript-debug/sessions?agent=ejecutor"
+            val sessionsReq = Request.Builder().url(sessionsUrl).build()
+            val sessionsResp = client.newCall(sessionsReq).execute()
+            if (!sessionsResp.isSuccessful) return listOf("Failed to fetch sessions")
+
+            val sessionsJson = JSONArray(sessionsResp.body?.string() ?: "[]")
+            if (sessionsJson.length() == 0) return listOf("No sessions found")
+
+            // Get the most recent session (sorted by mtime desc from server)
+            val latestSession = sessionsJson.getJSONObject(0)
+            val sessionId = latestSession.getString("id")
+
+            // Fetch raw JSONL
+            val rawUrl = "$apiBase/transcript-debug/$sessionId/raw?agent=ejecutor"
+            val rawReq = Request.Builder().url(rawUrl).build()
+            val rawResp = client.newCall(rawReq).execute()
+            if (!rawResp.isSuccessful) return listOf("Failed to fetch transcript")
+
+            val rawText = rawResp.body?.string() ?: return listOf("Empty transcript")
+            return parseJsonlMessages(rawText)
+        } catch (e: Exception) {
+            return listOf("Error: ${e.message?.take(40)}")
+        }
+    }
+
+    private fun parseJsonlMessages(jsonl: String): List<String> {
+        val messages = mutableListOf<String>()
+        val lines = jsonl.trim().split("\n").filter { it.isNotBlank() }
+
+        for (line in lines) {
+            try {
+                val obj = JSONObject(line)
+                val type = obj.optString("type", "")
+
+                when (type) {
+                    "human" -> {
+                        val content = obj.optString("message", "")
+                            .ifEmpty { extractMessageContent(obj) }
+                        if (content.isNotEmpty()) {
+                            messages.add("> ${content.take(80)}")
+                        }
+                    }
+                    "assistant" -> {
+                        val content = extractMessageContent(obj)
+                        if (content.isNotEmpty()) {
+                            messages.add("< ${content.take(80)}")
+                        }
+                    }
+                }
+            } catch (_: Exception) { }
+        }
+
+        return messages.takeLast(5).ifEmpty { listOf("No messages yet") }
+    }
+
+    private fun extractMessageContent(obj: JSONObject): String {
+        // Try "message" field first (simple string)
+        val msg = obj.optString("message", "")
+        if (msg.isNotEmpty()) return msg
+
+        // Try "message" as array of content blocks
+        val msgArray = obj.optJSONArray("message")
+        if (msgArray != null) {
+            for (i in 0 until msgArray.length()) {
+                val block = msgArray.optJSONObject(i) ?: continue
+                if (block.optString("type") == "text") {
+                    return block.optString("text", "")
+                }
+            }
+        }
+
+        return ""
+    }
+
+    private fun updateWidget(messages: List<String>) {
+        val views = RemoteViews(context.packageName, R.layout.widget_transcript)
+
+        for (i in MSG_IDS.indices) {
+            views.setTextViewText(MSG_IDS[i], messages.getOrElse(i) { "" })
+        }
+
+        val appWidgetManager = AppWidgetManager.getInstance(context)
+        val widgetComponent = ComponentName(context, TranscriptWidgetProvider::class.java)
+        appWidgetManager.updateAppWidget(widgetComponent, views)
+    }
+
+    private fun scheduleNext() {
+        val request = OneTimeWorkRequestBuilder<TranscriptWidgetWorker>()
+            .setInitialDelay(30, TimeUnit.SECONDS)
+            .build()
+        WorkManager.getInstance(context)
+            .enqueueUniqueWork(WORK_NAME, ExistingWorkPolicy.REPLACE, request)
+    }
+}
--- a/src-tauri/gen/android/app/src/main/java/com/agentui/desktop/VoiceCommandActivity.kt
+++ b/src-tauri/gen/android/app/src/main/java/com/agentui/desktop/VoiceCommandActivity.kt
@@ -0,0 +1,160 @@
+package com.agentui.desktop
+
+import android.app.Activity
+import android.content.Intent
+import android.os.Bundle
+import android.speech.RecognizerIntent
+import android.util.Log
+import android.widget.Toast
+import okhttp3.*
+import okhttp3.MediaType.Companion.toMediaType
+import okhttp3.RequestBody.Companion.toRequestBody
+import org.json.JSONObject
+import java.util.concurrent.CountDownLatch
+import java.util.concurrent.TimeUnit
+
+class VoiceCommandActivity : Activity() {
+
+    companion object {
+        private const val TAG = "AgentUI.Voice"
+        private const val SPEECH_REQUEST_CODE = 1001
+    }
+
+    private val client = OkHttpClient.Builder()
+        .connectTimeout(10, TimeUnit.SECONDS)
+        .readTimeout(10, TimeUnit.SECONDS)
+        .build()
+
+    override fun onCreate(savedInstanceState: Bundle?) {
+        super.onCreate(savedInstanceState)
+
+        // Check if launched via share intent
+        if (intent?.action == Intent.ACTION_SEND && intent.type == "text/plain") {
+            val sharedText = intent.getStringExtra(Intent.EXTRA_TEXT)
+            if (!sharedText.isNullOrBlank()) {
+                sendToServer(sharedText)
+                return
+            }
+        }
+
+        // Otherwise, start speech recognition
+        startSpeechRecognition()
+    }
+
+    private fun startSpeechRecognition() {
+        val intent = Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply {
+            putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM)
+            putExtra(RecognizerIntent.EXTRA_PROMPT, "Agent UI - Speak your command")
+        }
+        try {
+            startActivityForResult(intent, SPEECH_REQUEST_CODE)
+        } catch (e: Exception) {
+            Toast.makeText(this, "Speech recognition not available", Toast.LENGTH_SHORT).show()
+            finish()
+        }
+    }
+
+    override fun onActivityResult(requestCode: Int, resultCode: Int, data: Intent?) {
+        super.onActivityResult(requestCode, resultCode, data)
+
+        if (requestCode == SPEECH_REQUEST_CODE) {
+            if (resultCode == RESULT_OK && data != null) {
+                val results = data.getStringArrayListExtra(RecognizerIntent.EXTRA_RESULTS)
+                val text = results?.firstOrNull()
+                if (!text.isNullOrBlank()) {
+                    sendToServer(text)
+                    return
+                }
+            }
+            finish()
+        }
+    }
+
+    private fun sendToServer(text: String) {
+        Thread {
+            try {
+                val terminalBase = ServerConfig.terminalBaseUrl(this)
+                val terminalWs = ServerConfig.terminalWsUrl(this)
+
+                if (terminalBase == null || terminalWs == null) {
+                    showToastAndFinish("No server configured")
+                    return@Thread
+                }
+
+                // 1. Create a new terminal session
+                val createBody = JSONObject().apply {
+                    put("agent", "ejecutor")
+                    put("transcriptSessionId", "__new__")
+                    put("label", text.take(60))
+                    put("command", "ejecutor")
+                }.toString().toRequestBody("application/json".toMediaType())
+
+                val createReq = Request.Builder()
+                    .url("$terminalBase/create-terminal")
+                    .post(createBody)
+                    .build()
+
+                val createResp = client.newCall(createReq).execute()
+                if (!createResp.isSuccessful) {
+                    showToastAndFinish("Failed to create terminal")
+                    return@Thread
+                }
+
+                val respJson = JSONObject(createResp.body?.string() ?: "{}")
+                val sessionId = respJson.optString("ephemeralSessionId", "")
+                if (sessionId.isEmpty()) {
+                    showToastAndFinish("No session ID returned")
+                    return@Thread
+                }
+
+                // 2. Connect via WebSocket and send the input
+                val wsUrl = "$terminalWs/ws/terminal?session=$sessionId"
+                val wsReq = Request.Builder().url(wsUrl).build()
+                val latch = CountDownLatch(1)
+
+                client.newWebSocket(wsReq, object : WebSocketListener() {
+                    override fun onOpen(webSocket: WebSocket, response: Response) {
+                        // Send the text as input
+                        val inputMsg = JSONObject().apply {
+                            put("type", "input")
+                            put("data", text)
+                        }.toString()
+                        webSocket.send(inputMsg)
+
+                        // Send Enter key after a short delay
+                        Thread.sleep(80)
+                        val enterMsg = JSONObject().apply {
+                            put("type", "input")
+                            put("data", "\r")
+                        }.toString()
+                        webSocket.send(enterMsg)
+
+                        // Close after sending
+                        Thread.sleep(200)
+                        webSocket.close(1000, "done")
+                        latch.countDown()
+                    }
+
+                    override fun onFailure(webSocket: WebSocket, t: Throwable, response: Response?) {
+                        Log.e(TAG, "WebSocket failed", t)
+                        latch.countDown()
+                    }
+                })
+
+                latch.await(5, TimeUnit.SECONDS)
+                showToastAndFinish("Sent: ${text.take(40)}")
+
+            } catch (e: Exception) {
+                Log.e(TAG, "Failed to send command", e)
+                showToastAndFinish("Error: ${e.message?.take(40)}")
+            }
+        }.start()
+    }
+
+    private fun showToastAndFinish(message: String) {
+        runOnUiThread {
+            Toast.makeText(this, message, Toast.LENGTH_SHORT).show()
+            finish()
+        }
+    }
+}
--- a/src-tauri/gen/android/app/src/main/res/layout/widget_transcript.xml
+++ b/src-tauri/gen/android/app/src/main/res/layout/widget_transcript.xml
@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="utf-8"?>
+<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
+    android:id="@+id/widget_root"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent"
+    android:orientation="vertical"
+    android:padding="12dp"
+    android:background="#DD1A1A2E">
+
+    <TextView
+        android:id="@+id/widget_title"
+        android:layout_width="match_parent"
+        android:layout_height="wrap_content"
+        android:text="Agent UI Transcript"
+        android:textColor="#8888FF"
+        android:textSize="12sp"
+        android:fontFamily="monospace"
+        android:paddingBottom="4dp" />
+
+    <TextView
+        android:id="@+id/msg1"
+        android:layout_width="match_parent"
+        android:layout_height="wrap_content"
+        android:textColor="#CCCCCC"
+        android:textSize="11sp"
+        android:fontFamily="monospace"
+        android:maxLines="1"
+        android:ellipsize="end" />
+
+    <TextView
+        android:id="@+id/msg2"
+        android:layout_width="match_parent"
+        android:layout_height="wrap_content"
+        android:textColor="#CCCCCC"
+        android:textSize="11sp"
+        android:fontFamily="monospace"
+        android:maxLines="1"
+        android:ellipsize="end" />
+
+    <TextView
+        android:id="@+id/msg3"
+        android:layout_width="match_parent"
+        android:layout_height="wrap_content"
+        android:textColor="#CCCCCC"
+        android:textSize="11sp"
+        android:fontFamily="monospace"
+        android:maxLines="1"
+        android:ellipsize="end" />
+
+    <TextView
+        android:id="@+id/msg4"
+        android:layout_width="match_parent"
+        android:layout_height="wrap_content"
+        android:textColor="#CCCCCC"
+        android:textSize="11sp"
+        android:fontFamily="monospace"
+        android:maxLines="1"
+        android:ellipsize="end" />
+
+    <TextView
+        android:id="@+id/msg5"
+        android:layout_width="match_parent"
+        android:layout_height="wrap_content"
+        android:textColor="#CCCCCC"
+        android:textSize="11sp"
+        android:fontFamily="monospace"
+        android:maxLines="1"
+        android:ellipsize="end" />
+
+</LinearLayout>
--- a/src-tauri/gen/android/app/src/main/res/values/strings.xml
+++ b/src-tauri/gen/android/app/src/main/res/values/strings.xml
@@ -0,0 +1,5 @@
+<resources>
+    <string name="app_name">Agent UI</string>
+    <string name="main_activity_title">Agent UI</string>
+    <string name="widget_description">Shows recent transcript messages from Agent UI</string>
+</resources>
--- a/src-tauri/gen/android/app/src/main/res/xml/transcript_widget_info.xml
+++ b/src-tauri/gen/android/app/src/main/res/xml/transcript_widget_info.xml
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<appwidget-provider xmlns:android="http://schemas.android.com/apk/res/android"
+    android:minWidth="250dp"
+    android:minHeight="110dp"
+    android:updatePeriodMillis="1800000"
+    android:initialLayout="@layout/widget_transcript"
+    android:resizeMode="horizontal|vertical"
+    android:widgetCategory="home_screen"
+    android:description="@string/widget_description" />