fix: Mobile terminal improvements

- Add auto-reconnect when WebSocket disconnects (up to 10 attempts) - Add disconnected/connecting overlay with visual feedback - Add scroll buttons (up/down/end) for mobile - Add refresh button to reconnect or redraw terminal - Make all virtual keys bigger and more clickable - Enable touch scrolling in terminal viewport - Fix Whisper connection: don't connect when server not ready
2026-02-14 04:35:46 -06:00
parent a2a4806c47
commit 12a95c6206
2 changed files with 299 additions and 20 deletions
--- a/frontend/src/components/FloatingTerminal.vue
+++ b/frontend/src/components/FloatingTerminal.vue
@@ -50,6 +50,10 @@ let terminal: Terminal | null = null
 let fitAddon: FitAddon | null = null
 let socket: WebSocket | null = null
 let resizeObserver: ResizeObserver | null = null
+let reconnectTimeout: number | null = null
+let reconnectAttempts = 0
+const MAX_RECONNECT_ATTEMPTS = 10
+const RECONNECT_DELAY_MS = 2000

 // Buffer for detecting WebMCP token
 let tokenBuffer = ''
@@ -423,12 +427,29 @@ async function connect() {
  if (connecting.value || connected.value) return
  connecting.value = true

+  // Connection timeout - if not connected in 10s, retry
+  const connectionTimeout = window.setTimeout(() => {
+    if (connecting.value && !connected.value) {
+      console.log('[Terminal] Connection timeout, retrying...')
+      connecting.value = false
+      socket?.close()
+      socket = null
+      if (isOpen.value && reconnectAttempts < MAX_RECONNECT_ATTEMPTS) {
+        scheduleReconnect()
+      }
+    }
+  }, 10000)
+
  try {
    socket = new WebSocket(WS_URL)

+    // Clear timeout on successful connection
+    socket.addEventListener('open', () => clearTimeout(connectionTimeout), { once: true })
+
    socket.onopen = () => {
      connected.value = true
      connecting.value = false
+      reconnectAttempts = 0 // Reset on successful connection
      terminal?.focus()
      if (terminal) {
        socket?.send(JSON.stringify({
@@ -493,6 +514,13 @@ async function connect() {
    socket.onclose = () => {
      connected.value = false
      connecting.value = false
+      socket = null
+
+      // Auto-reconnect if terminal is still open
+      if (isOpen.value && reconnectAttempts < MAX_RECONNECT_ATTEMPTS) {
+        terminal?.write('\r\n\x1b[33m[Disconnected - reconnecting...]\x1b[0m\r\n')
+        scheduleReconnect()
+      }
    }

    socket.onerror = () => {
@@ -500,9 +528,36 @@ async function connect() {
    }
  } catch (e) {
    connecting.value = false
+    // Try to reconnect on connection error
+    if (isOpen.value && reconnectAttempts < MAX_RECONNECT_ATTEMPTS) {
+      scheduleReconnect()
+    }
  }
 }

+function scheduleReconnect() {
+  if (reconnectTimeout) clearTimeout(reconnectTimeout)
+
+  reconnectAttempts++
+  const delay = RECONNECT_DELAY_MS * Math.min(reconnectAttempts, 5) // Max 10s delay
+
+  console.log(`[Terminal] Reconnecting in ${delay}ms (attempt ${reconnectAttempts}/${MAX_RECONNECT_ATTEMPTS})`)
+
+  reconnectTimeout = window.setTimeout(() => {
+    if (isOpen.value && !connected.value && !connecting.value) {
+      connect()
+    }
+  }, delay)
+}
+
+function cancelReconnect() {
+  if (reconnectTimeout) {
+    clearTimeout(reconnectTimeout)
+    reconnectTimeout = null
+  }
+  reconnectAttempts = 0
+}
+
 function close() {
  isOpen.value = false
 }
@@ -554,6 +609,36 @@ function sendKey(key: string) {
  }
 }

+// Mobile scroll controls
+function scrollTerminal(direction: 'up' | 'down' | 'end') {
+  if (!terminal) return
+
+  if (direction === 'up') {
+    terminal.scrollLines(-10)
+  } else if (direction === 'down') {
+    terminal.scrollLines(10)
+  } else if (direction === 'end') {
+    terminal.scrollToBottom()
+  }
+}
+
+// Refresh terminal display
+function refreshTerminal() {
+  // If disconnected, try to reconnect
+  if (!connected.value && !connecting.value) {
+    reconnectAttempts = 0
+    connect()
+    return
+  }
+
+  if (!terminal || !fitAddon) return
+
+  // Clear and refit terminal
+  fitAddon.fit()
+  terminal.scrollToBottom()
+  terminal.focus()
+}
+
 watch(isOpen, async (open) => {
  if (open) {
    await nextTick()
@@ -565,6 +650,7 @@ watch(isOpen, async (open) => {
    })
  } else {
    // Cleanup when closing
+    cancelReconnect() // Stop any pending reconnection
    resizeObserver?.disconnect()
    resizeObserver = null
    terminal?.dispose()
@@ -603,6 +689,7 @@ onMounted(async () => {
 })

 onBeforeUnmount(() => {
+  cancelReconnect()
  resizeObserver?.disconnect()
  socket?.close()
  terminal?.dispose()
@@ -739,6 +826,24 @@ defineExpose({
          <!-- Content -->
          <div class="content">
            <div ref="terminalContainer" class="term"></div>
+            <!-- Disconnected overlay -->
+            <div v-if="!connected && !connecting" class="disconnect-overlay" @click="connect">
+              <div class="disconnect-msg">
+                <svg width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
+                  <path d="M18.36 6.64a9 9 0 1 1-12.73 0"/>
+                  <line x1="12" y1="2" x2="12" y2="12"/>
+                </svg>
+                <span>Desconectado</span>
+                <small>Toca para reconectar</small>
+              </div>
+            </div>
+            <!-- Connecting overlay -->
+            <div v-else-if="connecting" class="disconnect-overlay connecting">
+              <div class="disconnect-msg">
+                <div class="spinner"></div>
+                <span>Conectando...</span>
+              </div>
+            </div>
          </div>
          <!-- Mobile virtual keys -->
          <div v-if="isMobile" class="virtual-keys">
@@ -746,6 +851,14 @@ defineExpose({
            <button @click="sendKey('tab')" class="vk">Tab</button>
            <button @click="sendKey('ctrl-c')" class="vk ctrl">^C</button>
            <button @click="sendKey('alt-m')" class="vk alt">Alt+M</button>
+            <button @click="refreshTerminal" class="vk refresh" title="Refrescar">↻</button>
+            <!-- Scroll controls -->
+            <div class="vk-scroll">
+              <button @click="scrollTerminal('up')" class="vk scroll">⬆</button>
+              <button @click="scrollTerminal('end')" class="vk scroll end">⤓</button>
+              <button @click="scrollTerminal('down')" class="vk scroll">⬇</button>
+            </div>
+            <!-- Arrow keys for input -->
            <div class="vk-arrows">
              <button @click="sendKey('up')" class="vk arrow">▲</button>
              <div class="vk-row">
@@ -880,6 +993,60 @@ defineExpose({
  border-radius: 2px;
  overflow: hidden;
  background: rgba(0,0,0,0.92);
+  position: relative;
+}
+
+/* Disconnected/Connecting overlay */
+.disconnect-overlay {
+  position: absolute;
+  inset: 0;
+  background: rgba(0, 0, 0, 0.85);
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  z-index: 10;
+  cursor: pointer;
+}
+
+.disconnect-overlay.connecting {
+  cursor: wait;
+}
+
+.disconnect-msg {
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+  gap: 8px;
+  color: #fff;
+  text-align: center;
+}
+
+.disconnect-msg svg {
+  color: #ef4444;
+  opacity: 0.8;
+}
+
+.disconnect-msg span {
+  font-size: 14px;
+  font-weight: 500;
+}
+
+.disconnect-msg small {
+  font-size: 11px;
+  color: #888;
+}
+
+.spinner {
+  width: 24px;
+  height: 24px;
+  border: 2px solid rgba(255, 255, 255, 0.2);
+  border-top-color: #fff;
+  border-radius: 50%;
+  animation: spin 0.8s linear infinite;
+}
+
+@keyframes spin {
+  to { transform: rotate(360deg); }
 }

 .resize-handle {
@@ -913,6 +1080,13 @@ defineExpose({
 }
 .term :deep(.xterm-viewport) {
  overflow-y: auto !important;
+  /* Enable touch scrolling on mobile */
+  -webkit-overflow-scrolling: touch;
+  touch-action: pan-y;
+}
+.term :deep(.xterm-screen) {
+  /* Allow touch events to pass through for scrolling */
+  touch-action: pan-y;
 }
 .term :deep(.xterm-viewport::-webkit-scrollbar) {
  width: 8px;
@@ -981,6 +1155,9 @@ defineExpose({
 .aero-win.mobile .content {
  flex: 1;
  min-height: 100px;
+  /* Allow touch scrolling in terminal area */
+  touch-action: pan-y;
+  overflow: hidden;
 }

 /* Mobile animations */
@@ -1003,14 +1180,14 @@ defineExpose({
 }

 .vk {
-  min-width: 40px;
-  height: 32px;
-  padding: 0 8px;
+  min-width: 48px;
+  height: 40px;
+  padding: 0 10px;
  background: linear-gradient(180deg, #444 0%, #333 100%);
  border: 1px solid rgba(255, 255, 255, 0.15);
-  border-radius: 5px;
+  border-radius: 6px;
  color: #fff;
-  font-size: 11px;
+  font-size: 13px;
  font-weight: 500;
  font-family: system-ui, sans-serif;
  cursor: pointer;
@@ -1033,7 +1210,13 @@ defineExpose({
  border-color: rgba(100, 150, 255, 0.3);
 }

-.vk-arrows {
+.vk.refresh {
+  background: linear-gradient(180deg, #f59e0b 0%, #d97706 100%);
+  border-color: rgba(245, 158, 11, 0.3);
+  font-size: 18px;
+}
+
+.vk-scroll {
  display: flex;
  flex-direction: column;
  align-items: center;
@@ -1041,16 +1224,39 @@ defineExpose({
  margin-left: auto;
 }

+.vk.scroll {
+  min-width: 44px;
+  width: 44px;
+  height: 36px;
+  padding: 0;
+  font-size: 16px;
+  background: linear-gradient(180deg, #065f46 0%, #047857 100%);
+  border-color: rgba(16, 185, 129, 0.3);
+}
+
+.vk.scroll.end {
+  background: linear-gradient(180deg, #7c3aed 0%, #6d28d9 100%);
+  border-color: rgba(139, 92, 246, 0.3);
+  font-size: 18px;
+}
+
+.vk-arrows {
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+  gap: 2px;
+}
+
 .vk-row {
  display: flex;
  gap: 2px;
 }

 .vk.arrow {
-  min-width: 32px;
-  width: 32px;
-  height: 26px;
+  min-width: 38px;
+  width: 38px;
+  height: 32px;
  padding: 0;
-  font-size: 10px;
+  font-size: 12px;
 }
 </style>
--- a/frontend/src/components/FloatingVoice.vue
+++ b/frontend/src/components/FloatingVoice.vue
@@ -46,6 +46,7 @@ const isRecording = ref(false)
 const transcript = ref('')
 const interimTranscript = ref('')
 const error = ref('')
+let lastProcessedResult = '' // Track last result to avoid duplicates on Android

 // Typing animation state
 const animatedTranscript = ref('')
@@ -93,6 +94,7 @@ const showMicSelector = ref(false)

 // ============ MOBILE DETECTION & AUDIO FORMAT ============
 const isMobile = ref(false)
+const isAndroid = ref(false)
 const supportedMimeType = ref('audio/webm;codecs=opus')
 const sheetHeight = ref(45) // percentage of viewport for mobile
 const isDraggingSheet = ref(false)
@@ -101,7 +103,9 @@ const keyboardHeight = ref(0)
 const snapPoints = [25, 45, 70] // collapsed, default, expanded

 function checkMobile() {
-  isMobile.value = window.innerWidth <= 640 || /iPhone|iPad|iPod|Android/i.test(navigator.userAgent)
+  const ua = navigator.userAgent
+  isMobile.value = window.innerWidth <= 640 || /iPhone|iPad|iPod|Android/i.test(ua)
+  isAndroid.value = /Android/i.test(ua)
 }

 // Virtual keyboard detection
@@ -338,10 +342,16 @@ function initRecognition() {
  }

  const rec = new SpeechRecognition()
-  rec.continuous = true
+  // Android Chrome doesn't support continuous mode properly - it stops after first result
+  // Using non-continuous mode on Android prevents constant restarts and audio loss
+  rec.continuous = !isAndroid.value
  rec.interimResults = true
  rec.lang = 'es-419' // Latin American Spanish (better for accents)

+  if (isAndroid.value) {
+    console.log('[Voice] Android detected - using non-continuous mode for stability')
+  }
+
  rec.onresult = (event: SpeechRecognitionEvent) => {
    let interim = ''
    let final = ''
@@ -358,7 +368,19 @@ function initRecognition() {
    }

    if (final) {
-      transcript.value += final
+      // Android: Check for duplicates (same text being processed again)
+      const trimmedFinal = final.trim()
+      if (isAndroid.value && lastProcessedResult && trimmedFinal.startsWith(lastProcessedResult.trim())) {
+        // Only add the new part
+        const newPart = trimmedFinal.slice(lastProcessedResult.trim().length).trim()
+        if (newPart) {
+          transcript.value += newPart + ' '
+          lastProcessedResult = trimmedFinal
+        }
+      } else {
+        transcript.value += final
+        lastProcessedResult = trimmedFinal
+      }
    }
    interimTranscript.value = interim
  }
@@ -375,8 +397,15 @@ function initRecognition() {

  rec.onend = () => {
    if (isRecording.value && !useWhisper.value) {
-      // Restart if still recording (browser stops after silence)
-      rec.start()
+      if (isAndroid.value) {
+        // Android: Don't auto-restart - causes duplicate text
+        // User should use push-to-talk or tap mic button for each phrase
+        isRecording.value = false
+        console.log('[Voice] Android session ended - tap mic to continue')
+      } else {
+        // Desktop: Restart immediately (browser stops after silence)
+        rec.start()
+      }
    }
  }

@@ -501,6 +530,11 @@ async function pollWhisperStatus() {
 }

 function connectWhisperSocket() {
+  // Don't connect if Whisper isn't ready
+  if (!whisperReady.value) {
+    console.log('[Voice] Whisper not ready, skipping connection')
+    return
+  }
  if (whisperSocket?.readyState === WebSocket.OPEN) return

  console.log('[Voice] Connecting to Whisper server...')
@@ -581,6 +615,20 @@ function disconnectWhisperSocket() {
 }

 async function startWhisperRecording() {
+  // Ensure WebSocket is connected before recording
+  if (!whisperSocket || whisperSocket.readyState !== WebSocket.OPEN) {
+    console.warn('[Voice] Whisper socket not connected, attempting to connect...')
+    connectWhisperSocket()
+    // Wait a moment for connection
+    await new Promise(resolve => setTimeout(resolve, 500))
+
+    if (!whisperSocket || whisperSocket.readyState !== WebSocket.OPEN) {
+      error.value = 'Whisper server not connected'
+      canvasStore.showNotification('Whisper not connected. Try toggling GPU mode.', 'error')
+      return
+    }
+  }
+
  try {
    // Mobile-optimized audio constraints
    const audioConstraints: MediaTrackConstraints = {
@@ -616,9 +664,11 @@ async function startWhisperRecording() {
    mediaRecorder.start(100) // Collect data every 100ms
    isRecording.value = true
    recordingStartTime = Date.now()
+    console.log(`[Voice] Whisper recording started, socket ready: ${whisperSocket?.readyState === WebSocket.OPEN}`)

    // Send chunks periodically for progressive transcription
    chunkInterval = window.setInterval(() => {
+      console.log(`[Voice] Chunk interval: ${audioChunks.length} chunks, socket: ${whisperSocket?.readyState}`)
      if (audioChunks.length > 0 && whisperSocket?.readyState === WebSocket.OPEN) {
        sendAudioChunk(false) // false = partial, don't clear
      }
@@ -633,8 +683,10 @@ async function startWhisperRecording() {
 function sendAudioChunk(isFinal: boolean) {
  if (audioChunks.length === 0) return

-  // Always send ALL accumulated audio (webm needs header from first chunk)
-  const audioBlob = new Blob(audioChunks, { type: 'audio/webm' })
+  // Always send ALL accumulated audio (needs header from first chunk)
+  const mimeType = mediaRecorder?.mimeType || supportedMimeType.value || 'audio/webm'
+  const audioBlob = new Blob(audioChunks, { type: mimeType })
+  console.log(`[Voice] Sending chunk: ${audioChunks.length} chunks, ${audioBlob.size} bytes, ${mimeType}`)
  const chunkCount = audioChunks.length

  // Skip if audio is too small (< 5KB) - WebM header alone is ~1-2KB
@@ -719,6 +771,15 @@ function startRecording() {
      try {
        recognition.start()
        isRecording.value = true
+
+        // Notify Android users about limitations
+        if (isAndroid.value && !isPushToTalk.value) {
+          canvasStore.showNotification(
+            'Android: Tap mic again to continue recording',
+            'info',
+            3000
+          )
+        }
      } catch (e) {
        console.error('[Voice] Failed to start:', e)
      }
@@ -743,6 +804,7 @@ function clearTranscript() {
  interimTranscript.value = ''
  animatedTranscript.value = ''
  lastAnimatedLength = 0
+  lastProcessedResult = ''
  if (typingTimeout) {
    clearTimeout(typingTimeout)
    typingTimeout = null
@@ -1025,8 +1087,13 @@ onMounted(async () => {
  if (status?.starting) {
    console.log('[Voice] Server is starting, resuming polling...')
    pollWhisperStatus()
-  } else if (useWhisper.value) {
+  } else if (useWhisper.value && whisperReady.value) {
+    // Only connect if both enabled AND actually running
    connectWhisperSocket()
+  } else if (useWhisper.value && !whisperReady.value) {
+    // User had Whisper enabled but server isn't running - disable it
+    console.log('[Voice] Whisper was enabled but server not running, disabling')
+    useWhisper.value = false
  }
 })

@@ -1124,8 +1191,8 @@ defineExpose({
              </svg>
              <span>Voice</span>
              <i class="dot" :class="{ recording: isRecording, ptt: isPushToTalk }"></i>
-              <span class="mode-badge" :class="{ gpu: useWhisper }">
-                {{ useWhisper ? 'GPU' : 'Web' }}
+              <span class="mode-badge" :class="{ gpu: useWhisper, android: isAndroid && !useWhisper }">
+                {{ useWhisper ? 'GPU' : (isAndroid ? 'Android' : 'Web') }}
              </span>
            </div>
            <div class="window-controls">
@@ -1350,6 +1417,12 @@ defineExpose({
  box-shadow: 0 0 4px rgba(16, 185, 129, 0.5);
 }

+.mode-badge.android {
+  background: linear-gradient(135deg, #f59e0b, #d97706);
+  color: #fff;
+  box-shadow: 0 0 4px rgba(245, 158, 11, 0.5);
+}
+
 .whisper-toggle {
  width: 20px;
  height: 18px;