fix: Mobile terminal improvements

- Add auto-reconnect when WebSocket disconnects (up to 10 attempts)
- Add disconnected/connecting overlay with visual feedback
- Add scroll buttons (up/down/end) for mobile
- Add refresh button to reconnect or redraw terminal
- Make all virtual keys bigger and more clickable
- Enable touch scrolling in terminal viewport
- Fix Whisper connection: don't connect when server not ready
This commit is contained in:
2026-02-14 04:35:46 -06:00
parent a2a4806c47
commit 12a95c6206
2 changed files with 299 additions and 20 deletions

View File

@@ -50,6 +50,10 @@ let terminal: Terminal | null = null
let fitAddon: FitAddon | null = null let fitAddon: FitAddon | null = null
let socket: WebSocket | null = null let socket: WebSocket | null = null
let resizeObserver: ResizeObserver | null = null let resizeObserver: ResizeObserver | null = null
let reconnectTimeout: number | null = null
let reconnectAttempts = 0
const MAX_RECONNECT_ATTEMPTS = 10
const RECONNECT_DELAY_MS = 2000
// Buffer for detecting WebMCP token // Buffer for detecting WebMCP token
let tokenBuffer = '' let tokenBuffer = ''
@@ -423,12 +427,29 @@ async function connect() {
if (connecting.value || connected.value) return if (connecting.value || connected.value) return
connecting.value = true connecting.value = true
// Connection timeout - if not connected in 10s, retry
const connectionTimeout = window.setTimeout(() => {
if (connecting.value && !connected.value) {
console.log('[Terminal] Connection timeout, retrying...')
connecting.value = false
socket?.close()
socket = null
if (isOpen.value && reconnectAttempts < MAX_RECONNECT_ATTEMPTS) {
scheduleReconnect()
}
}
}, 10000)
try { try {
socket = new WebSocket(WS_URL) socket = new WebSocket(WS_URL)
// Clear timeout on successful connection
socket.addEventListener('open', () => clearTimeout(connectionTimeout), { once: true })
socket.onopen = () => { socket.onopen = () => {
connected.value = true connected.value = true
connecting.value = false connecting.value = false
reconnectAttempts = 0 // Reset on successful connection
terminal?.focus() terminal?.focus()
if (terminal) { if (terminal) {
socket?.send(JSON.stringify({ socket?.send(JSON.stringify({
@@ -493,6 +514,13 @@ async function connect() {
socket.onclose = () => { socket.onclose = () => {
connected.value = false connected.value = false
connecting.value = false connecting.value = false
socket = null
// Auto-reconnect if terminal is still open
if (isOpen.value && reconnectAttempts < MAX_RECONNECT_ATTEMPTS) {
terminal?.write('\r\n\x1b[33m[Disconnected - reconnecting...]\x1b[0m\r\n')
scheduleReconnect()
}
} }
socket.onerror = () => { socket.onerror = () => {
@@ -500,9 +528,36 @@ async function connect() {
} }
} catch (e) { } catch (e) {
connecting.value = false connecting.value = false
// Try to reconnect on connection error
if (isOpen.value && reconnectAttempts < MAX_RECONNECT_ATTEMPTS) {
scheduleReconnect()
}
} }
} }
function scheduleReconnect() {
if (reconnectTimeout) clearTimeout(reconnectTimeout)
reconnectAttempts++
const delay = RECONNECT_DELAY_MS * Math.min(reconnectAttempts, 5) // Max 10s delay
console.log(`[Terminal] Reconnecting in ${delay}ms (attempt ${reconnectAttempts}/${MAX_RECONNECT_ATTEMPTS})`)
reconnectTimeout = window.setTimeout(() => {
if (isOpen.value && !connected.value && !connecting.value) {
connect()
}
}, delay)
}
function cancelReconnect() {
if (reconnectTimeout) {
clearTimeout(reconnectTimeout)
reconnectTimeout = null
}
reconnectAttempts = 0
}
function close() { function close() {
isOpen.value = false isOpen.value = false
} }
@@ -554,6 +609,36 @@ function sendKey(key: string) {
} }
} }
// Mobile scroll controls
function scrollTerminal(direction: 'up' | 'down' | 'end') {
if (!terminal) return
if (direction === 'up') {
terminal.scrollLines(-10)
} else if (direction === 'down') {
terminal.scrollLines(10)
} else if (direction === 'end') {
terminal.scrollToBottom()
}
}
// Refresh terminal display
function refreshTerminal() {
// If disconnected, try to reconnect
if (!connected.value && !connecting.value) {
reconnectAttempts = 0
connect()
return
}
if (!terminal || !fitAddon) return
// Clear and refit terminal
fitAddon.fit()
terminal.scrollToBottom()
terminal.focus()
}
watch(isOpen, async (open) => { watch(isOpen, async (open) => {
if (open) { if (open) {
await nextTick() await nextTick()
@@ -565,6 +650,7 @@ watch(isOpen, async (open) => {
}) })
} else { } else {
// Cleanup when closing // Cleanup when closing
cancelReconnect() // Stop any pending reconnection
resizeObserver?.disconnect() resizeObserver?.disconnect()
resizeObserver = null resizeObserver = null
terminal?.dispose() terminal?.dispose()
@@ -603,6 +689,7 @@ onMounted(async () => {
}) })
onBeforeUnmount(() => { onBeforeUnmount(() => {
cancelReconnect()
resizeObserver?.disconnect() resizeObserver?.disconnect()
socket?.close() socket?.close()
terminal?.dispose() terminal?.dispose()
@@ -739,6 +826,24 @@ defineExpose({
<!-- Content --> <!-- Content -->
<div class="content"> <div class="content">
<div ref="terminalContainer" class="term"></div> <div ref="terminalContainer" class="term"></div>
<!-- Disconnected overlay -->
<div v-if="!connected && !connecting" class="disconnect-overlay" @click="connect">
<div class="disconnect-msg">
<svg width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
<path d="M18.36 6.64a9 9 0 1 1-12.73 0"/>
<line x1="12" y1="2" x2="12" y2="12"/>
</svg>
<span>Desconectado</span>
<small>Toca para reconectar</small>
</div>
</div>
<!-- Connecting overlay -->
<div v-else-if="connecting" class="disconnect-overlay connecting">
<div class="disconnect-msg">
<div class="spinner"></div>
<span>Conectando...</span>
</div>
</div>
</div> </div>
<!-- Mobile virtual keys --> <!-- Mobile virtual keys -->
<div v-if="isMobile" class="virtual-keys"> <div v-if="isMobile" class="virtual-keys">
@@ -746,6 +851,14 @@ defineExpose({
<button @click="sendKey('tab')" class="vk">Tab</button> <button @click="sendKey('tab')" class="vk">Tab</button>
<button @click="sendKey('ctrl-c')" class="vk ctrl">^C</button> <button @click="sendKey('ctrl-c')" class="vk ctrl">^C</button>
<button @click="sendKey('alt-m')" class="vk alt">Alt+M</button> <button @click="sendKey('alt-m')" class="vk alt">Alt+M</button>
<button @click="refreshTerminal" class="vk refresh" title="Refrescar"></button>
<!-- Scroll controls -->
<div class="vk-scroll">
<button @click="scrollTerminal('up')" class="vk scroll"></button>
<button @click="scrollTerminal('end')" class="vk scroll end"></button>
<button @click="scrollTerminal('down')" class="vk scroll"></button>
</div>
<!-- Arrow keys for input -->
<div class="vk-arrows"> <div class="vk-arrows">
<button @click="sendKey('up')" class="vk arrow"></button> <button @click="sendKey('up')" class="vk arrow"></button>
<div class="vk-row"> <div class="vk-row">
@@ -880,6 +993,60 @@ defineExpose({
border-radius: 2px; border-radius: 2px;
overflow: hidden; overflow: hidden;
background: rgba(0,0,0,0.92); background: rgba(0,0,0,0.92);
position: relative;
}
/* Disconnected/Connecting overlay */
.disconnect-overlay {
position: absolute;
inset: 0;
background: rgba(0, 0, 0, 0.85);
display: flex;
align-items: center;
justify-content: center;
z-index: 10;
cursor: pointer;
}
.disconnect-overlay.connecting {
cursor: wait;
}
.disconnect-msg {
display: flex;
flex-direction: column;
align-items: center;
gap: 8px;
color: #fff;
text-align: center;
}
.disconnect-msg svg {
color: #ef4444;
opacity: 0.8;
}
.disconnect-msg span {
font-size: 14px;
font-weight: 500;
}
.disconnect-msg small {
font-size: 11px;
color: #888;
}
.spinner {
width: 24px;
height: 24px;
border: 2px solid rgba(255, 255, 255, 0.2);
border-top-color: #fff;
border-radius: 50%;
animation: spin 0.8s linear infinite;
}
@keyframes spin {
to { transform: rotate(360deg); }
} }
.resize-handle { .resize-handle {
@@ -913,6 +1080,13 @@ defineExpose({
} }
.term :deep(.xterm-viewport) { .term :deep(.xterm-viewport) {
overflow-y: auto !important; overflow-y: auto !important;
/* Enable touch scrolling on mobile */
-webkit-overflow-scrolling: touch;
touch-action: pan-y;
}
.term :deep(.xterm-screen) {
/* Allow touch events to pass through for scrolling */
touch-action: pan-y;
} }
.term :deep(.xterm-viewport::-webkit-scrollbar) { .term :deep(.xterm-viewport::-webkit-scrollbar) {
width: 8px; width: 8px;
@@ -981,6 +1155,9 @@ defineExpose({
.aero-win.mobile .content { .aero-win.mobile .content {
flex: 1; flex: 1;
min-height: 100px; min-height: 100px;
/* Allow touch scrolling in terminal area */
touch-action: pan-y;
overflow: hidden;
} }
/* Mobile animations */ /* Mobile animations */
@@ -1003,14 +1180,14 @@ defineExpose({
} }
.vk { .vk {
min-width: 40px; min-width: 48px;
height: 32px; height: 40px;
padding: 0 8px; padding: 0 10px;
background: linear-gradient(180deg, #444 0%, #333 100%); background: linear-gradient(180deg, #444 0%, #333 100%);
border: 1px solid rgba(255, 255, 255, 0.15); border: 1px solid rgba(255, 255, 255, 0.15);
border-radius: 5px; border-radius: 6px;
color: #fff; color: #fff;
font-size: 11px; font-size: 13px;
font-weight: 500; font-weight: 500;
font-family: system-ui, sans-serif; font-family: system-ui, sans-serif;
cursor: pointer; cursor: pointer;
@@ -1033,7 +1210,13 @@ defineExpose({
border-color: rgba(100, 150, 255, 0.3); border-color: rgba(100, 150, 255, 0.3);
} }
.vk-arrows { .vk.refresh {
background: linear-gradient(180deg, #f59e0b 0%, #d97706 100%);
border-color: rgba(245, 158, 11, 0.3);
font-size: 18px;
}
.vk-scroll {
display: flex; display: flex;
flex-direction: column; flex-direction: column;
align-items: center; align-items: center;
@@ -1041,16 +1224,39 @@ defineExpose({
margin-left: auto; margin-left: auto;
} }
.vk.scroll {
min-width: 44px;
width: 44px;
height: 36px;
padding: 0;
font-size: 16px;
background: linear-gradient(180deg, #065f46 0%, #047857 100%);
border-color: rgba(16, 185, 129, 0.3);
}
.vk.scroll.end {
background: linear-gradient(180deg, #7c3aed 0%, #6d28d9 100%);
border-color: rgba(139, 92, 246, 0.3);
font-size: 18px;
}
.vk-arrows {
display: flex;
flex-direction: column;
align-items: center;
gap: 2px;
}
.vk-row { .vk-row {
display: flex; display: flex;
gap: 2px; gap: 2px;
} }
.vk.arrow { .vk.arrow {
min-width: 32px; min-width: 38px;
width: 32px; width: 38px;
height: 26px; height: 32px;
padding: 0; padding: 0;
font-size: 10px; font-size: 12px;
} }
</style> </style>

View File

@@ -46,6 +46,7 @@ const isRecording = ref(false)
const transcript = ref('') const transcript = ref('')
const interimTranscript = ref('') const interimTranscript = ref('')
const error = ref('') const error = ref('')
let lastProcessedResult = '' // Track last result to avoid duplicates on Android
// Typing animation state // Typing animation state
const animatedTranscript = ref('') const animatedTranscript = ref('')
@@ -93,6 +94,7 @@ const showMicSelector = ref(false)
// ============ MOBILE DETECTION & AUDIO FORMAT ============ // ============ MOBILE DETECTION & AUDIO FORMAT ============
const isMobile = ref(false) const isMobile = ref(false)
const isAndroid = ref(false)
const supportedMimeType = ref('audio/webm;codecs=opus') const supportedMimeType = ref('audio/webm;codecs=opus')
const sheetHeight = ref(45) // percentage of viewport for mobile const sheetHeight = ref(45) // percentage of viewport for mobile
const isDraggingSheet = ref(false) const isDraggingSheet = ref(false)
@@ -101,7 +103,9 @@ const keyboardHeight = ref(0)
const snapPoints = [25, 45, 70] // collapsed, default, expanded const snapPoints = [25, 45, 70] // collapsed, default, expanded
function checkMobile() { function checkMobile() {
isMobile.value = window.innerWidth <= 640 || /iPhone|iPad|iPod|Android/i.test(navigator.userAgent) const ua = navigator.userAgent
isMobile.value = window.innerWidth <= 640 || /iPhone|iPad|iPod|Android/i.test(ua)
isAndroid.value = /Android/i.test(ua)
} }
// Virtual keyboard detection // Virtual keyboard detection
@@ -338,10 +342,16 @@ function initRecognition() {
} }
const rec = new SpeechRecognition() const rec = new SpeechRecognition()
rec.continuous = true // Android Chrome doesn't support continuous mode properly - it stops after first result
// Using non-continuous mode on Android prevents constant restarts and audio loss
rec.continuous = !isAndroid.value
rec.interimResults = true rec.interimResults = true
rec.lang = 'es-419' // Latin American Spanish (better for accents) rec.lang = 'es-419' // Latin American Spanish (better for accents)
if (isAndroid.value) {
console.log('[Voice] Android detected - using non-continuous mode for stability')
}
rec.onresult = (event: SpeechRecognitionEvent) => { rec.onresult = (event: SpeechRecognitionEvent) => {
let interim = '' let interim = ''
let final = '' let final = ''
@@ -358,7 +368,19 @@ function initRecognition() {
} }
if (final) { if (final) {
transcript.value += final // Android: Check for duplicates (same text being processed again)
const trimmedFinal = final.trim()
if (isAndroid.value && lastProcessedResult && trimmedFinal.startsWith(lastProcessedResult.trim())) {
// Only add the new part
const newPart = trimmedFinal.slice(lastProcessedResult.trim().length).trim()
if (newPart) {
transcript.value += newPart + ' '
lastProcessedResult = trimmedFinal
}
} else {
transcript.value += final
lastProcessedResult = trimmedFinal
}
} }
interimTranscript.value = interim interimTranscript.value = interim
} }
@@ -375,8 +397,15 @@ function initRecognition() {
rec.onend = () => { rec.onend = () => {
if (isRecording.value && !useWhisper.value) { if (isRecording.value && !useWhisper.value) {
// Restart if still recording (browser stops after silence) if (isAndroid.value) {
rec.start() // Android: Don't auto-restart - causes duplicate text
// User should use push-to-talk or tap mic button for each phrase
isRecording.value = false
console.log('[Voice] Android session ended - tap mic to continue')
} else {
// Desktop: Restart immediately (browser stops after silence)
rec.start()
}
} }
} }
@@ -501,6 +530,11 @@ async function pollWhisperStatus() {
} }
function connectWhisperSocket() { function connectWhisperSocket() {
// Don't connect if Whisper isn't ready
if (!whisperReady.value) {
console.log('[Voice] Whisper not ready, skipping connection')
return
}
if (whisperSocket?.readyState === WebSocket.OPEN) return if (whisperSocket?.readyState === WebSocket.OPEN) return
console.log('[Voice] Connecting to Whisper server...') console.log('[Voice] Connecting to Whisper server...')
@@ -581,6 +615,20 @@ function disconnectWhisperSocket() {
} }
async function startWhisperRecording() { async function startWhisperRecording() {
// Ensure WebSocket is connected before recording
if (!whisperSocket || whisperSocket.readyState !== WebSocket.OPEN) {
console.warn('[Voice] Whisper socket not connected, attempting to connect...')
connectWhisperSocket()
// Wait a moment for connection
await new Promise(resolve => setTimeout(resolve, 500))
if (!whisperSocket || whisperSocket.readyState !== WebSocket.OPEN) {
error.value = 'Whisper server not connected'
canvasStore.showNotification('Whisper not connected. Try toggling GPU mode.', 'error')
return
}
}
try { try {
// Mobile-optimized audio constraints // Mobile-optimized audio constraints
const audioConstraints: MediaTrackConstraints = { const audioConstraints: MediaTrackConstraints = {
@@ -616,9 +664,11 @@ async function startWhisperRecording() {
mediaRecorder.start(100) // Collect data every 100ms mediaRecorder.start(100) // Collect data every 100ms
isRecording.value = true isRecording.value = true
recordingStartTime = Date.now() recordingStartTime = Date.now()
console.log(`[Voice] Whisper recording started, socket ready: ${whisperSocket?.readyState === WebSocket.OPEN}`)
// Send chunks periodically for progressive transcription // Send chunks periodically for progressive transcription
chunkInterval = window.setInterval(() => { chunkInterval = window.setInterval(() => {
console.log(`[Voice] Chunk interval: ${audioChunks.length} chunks, socket: ${whisperSocket?.readyState}`)
if (audioChunks.length > 0 && whisperSocket?.readyState === WebSocket.OPEN) { if (audioChunks.length > 0 && whisperSocket?.readyState === WebSocket.OPEN) {
sendAudioChunk(false) // false = partial, don't clear sendAudioChunk(false) // false = partial, don't clear
} }
@@ -633,8 +683,10 @@ async function startWhisperRecording() {
function sendAudioChunk(isFinal: boolean) { function sendAudioChunk(isFinal: boolean) {
if (audioChunks.length === 0) return if (audioChunks.length === 0) return
// Always send ALL accumulated audio (webm needs header from first chunk) // Always send ALL accumulated audio (needs header from first chunk)
const audioBlob = new Blob(audioChunks, { type: 'audio/webm' }) const mimeType = mediaRecorder?.mimeType || supportedMimeType.value || 'audio/webm'
const audioBlob = new Blob(audioChunks, { type: mimeType })
console.log(`[Voice] Sending chunk: ${audioChunks.length} chunks, ${audioBlob.size} bytes, ${mimeType}`)
const chunkCount = audioChunks.length const chunkCount = audioChunks.length
// Skip if audio is too small (< 5KB) - WebM header alone is ~1-2KB // Skip if audio is too small (< 5KB) - WebM header alone is ~1-2KB
@@ -719,6 +771,15 @@ function startRecording() {
try { try {
recognition.start() recognition.start()
isRecording.value = true isRecording.value = true
// Notify Android users about limitations
if (isAndroid.value && !isPushToTalk.value) {
canvasStore.showNotification(
'Android: Tap mic again to continue recording',
'info',
3000
)
}
} catch (e) { } catch (e) {
console.error('[Voice] Failed to start:', e) console.error('[Voice] Failed to start:', e)
} }
@@ -743,6 +804,7 @@ function clearTranscript() {
interimTranscript.value = '' interimTranscript.value = ''
animatedTranscript.value = '' animatedTranscript.value = ''
lastAnimatedLength = 0 lastAnimatedLength = 0
lastProcessedResult = ''
if (typingTimeout) { if (typingTimeout) {
clearTimeout(typingTimeout) clearTimeout(typingTimeout)
typingTimeout = null typingTimeout = null
@@ -1025,8 +1087,13 @@ onMounted(async () => {
if (status?.starting) { if (status?.starting) {
console.log('[Voice] Server is starting, resuming polling...') console.log('[Voice] Server is starting, resuming polling...')
pollWhisperStatus() pollWhisperStatus()
} else if (useWhisper.value) { } else if (useWhisper.value && whisperReady.value) {
// Only connect if both enabled AND actually running
connectWhisperSocket() connectWhisperSocket()
} else if (useWhisper.value && !whisperReady.value) {
// User had Whisper enabled but server isn't running - disable it
console.log('[Voice] Whisper was enabled but server not running, disabling')
useWhisper.value = false
} }
}) })
@@ -1124,8 +1191,8 @@ defineExpose({
</svg> </svg>
<span>Voice</span> <span>Voice</span>
<i class="dot" :class="{ recording: isRecording, ptt: isPushToTalk }"></i> <i class="dot" :class="{ recording: isRecording, ptt: isPushToTalk }"></i>
<span class="mode-badge" :class="{ gpu: useWhisper }"> <span class="mode-badge" :class="{ gpu: useWhisper, android: isAndroid && !useWhisper }">
{{ useWhisper ? 'GPU' : 'Web' }} {{ useWhisper ? 'GPU' : (isAndroid ? 'Android' : 'Web') }}
</span> </span>
</div> </div>
<div class="window-controls"> <div class="window-controls">
@@ -1350,6 +1417,12 @@ defineExpose({
box-shadow: 0 0 4px rgba(16, 185, 129, 0.5); box-shadow: 0 0 4px rgba(16, 185, 129, 0.5);
} }
.mode-badge.android {
background: linear-gradient(135deg, #f59e0b, #d97706);
color: #fff;
box-shadow: 0 0 4px rgba(245, 158, 11, 0.5);
}
.whisper-toggle { .whisper-toggle {
width: 20px; width: 20px;
height: 18px; height: 18px;