feat: Add mobile support for terminal and voice components

- Terminal: bottom sheet with drag handle and snap points (20%, 55%, 85%)
- Terminal: virtual keys bar (arrows, Esc, Tab, Ctrl+C, Alt+M)
- Terminal: keyboard detection adjusts position above virtual keyboard
- Terminal: touch drag support for mobile/tablet devices
- Voice: bottom sheet behavior matching terminal
- Voice: auto-detect supported audio format (webm/mp4/aac)
- Voice: improved audio constraints for mobile quality
- Both: detect touch devices up to 1024px width
This commit is contained in:
2026-02-14 03:52:54 -06:00
parent e51eb6749d
commit 759de1e010
2 changed files with 589 additions and 36 deletions

View File

@@ -38,6 +38,14 @@ const dragOffset = ref({ x: 0, y: 0 })
const isResizing = ref(false)
const size = ref({ w: 580, h: 360 })
// Mobile bottom sheet state
const isMobile = ref(false)
const sheetHeight = ref(55) // percentage of viewport
const isDraggingSheet = ref(false)
const sheetDragStart = ref({ y: 0, height: 0 })
const keyboardHeight = ref(0)
const snapPoints = [20, 55, 85] // collapsed, half, full (percentages)
let terminal: Terminal | null = null
let fitAddon: FitAddon | null = null
let socket: WebSocket | null = null
@@ -58,6 +66,84 @@ function trackMouse(e: MouseEvent) {
mousePos.value = { x: e.clientX, y: e.clientY }
}
// Mobile/tablet detection - show virtual keys on touch devices
function checkMobile() {
const isTouchDevice = 'ontouchstart' in window || navigator.maxTouchPoints > 0
const isSmallScreen = window.innerWidth <= 1024
isMobile.value = isTouchDevice && isSmallScreen
}
// Virtual keyboard detection using visualViewport API
function setupKeyboardDetection() {
if (window.visualViewport) {
window.visualViewport.addEventListener('resize', handleViewportResize)
}
}
function handleViewportResize() {
if (!window.visualViewport || !isMobile.value) return
const viewportHeight = window.visualViewport.height
const windowHeight = window.innerHeight
const diff = windowHeight - viewportHeight
// If difference > 100px, keyboard is probably open
if (diff > 100) {
keyboardHeight.value = diff
// Auto-expand sheet when keyboard opens if it's too small
if (sheetHeight.value < 55 && isOpen.value) {
sheetHeight.value = 55
}
} else {
keyboardHeight.value = 0
}
}
// Find nearest snap point
function findNearestSnap(height: number): number {
return snapPoints.reduce((prev, curr) =>
Math.abs(curr - height) < Math.abs(prev - height) ? curr : prev
)
}
// Mobile sheet touch handlers
function startSheetDrag(e: TouchEvent) {
if (!isMobile.value) return
const touch = e.touches[0]
if (!touch) return
isDraggingSheet.value = true
sheetDragStart.value = {
y: touch.clientY,
height: sheetHeight.value
}
}
function onSheetDrag(e: TouchEvent) {
if (!isDraggingSheet.value || !isMobile.value) return
const touch = e.touches[0]
if (!touch) return
const deltaY = sheetDragStart.value.y - touch.clientY
const deltaPercent = (deltaY / window.innerHeight) * 100
const newHeight = sheetDragStart.value.height + deltaPercent
// Clamp between 15% and 90%
sheetHeight.value = Math.max(15, Math.min(90, newHeight))
}
function stopSheetDrag() {
if (!isDraggingSheet.value) return
isDraggingSheet.value = false
// Snap to nearest point
sheetHeight.value = findNearestSnap(sheetHeight.value)
nextTick(() => fitAddon?.fit())
}
function toggleTerminal() {
const now = Date.now()
if (now - lastToggle < 150) return // Debounce 150ms
@@ -89,31 +175,53 @@ function handleKeydown(e: KeyboardEvent) {
}
}
function startDrag(e: MouseEvent) {
function startDrag(e: MouseEvent | TouchEvent) {
if ((e.target as HTMLElement).closest('.window-controls')) return
// On mobile, use sheet drag instead
if (isMobile.value) {
if (e instanceof TouchEvent) {
startSheetDrag(e)
}
return
}
isDragging.value = true
const rect = terminalRef.value?.getBoundingClientRect()
const touch = e instanceof TouchEvent ? e.touches[0] : null
const clientX = e instanceof MouseEvent ? e.clientX : (touch?.clientX ?? 0)
const clientY = e instanceof MouseEvent ? e.clientY : (touch?.clientY ?? 0)
if (rect) {
// Capture actual position if using default bottom/right
if (!hasCustomPosition.value) {
position.value = { x: rect.left, y: rect.top }
}
dragOffset.value = {
x: e.clientX - rect.left,
y: e.clientY - rect.top
x: clientX - rect.left,
y: clientY - rect.top
}
}
document.addEventListener('mousemove', onDrag)
document.addEventListener('mouseup', stopDrag)
document.addEventListener('touchmove', onDrag, { passive: false })
document.addEventListener('touchend', stopDrag)
}
function onDrag(e: MouseEvent) {
function onDrag(e: MouseEvent | TouchEvent) {
if (!isDragging.value) return
const newX = e.clientX - dragOffset.value.x
const newY = e.clientY - dragOffset.value.y
if (e instanceof TouchEvent) {
e.preventDefault()
}
const touch = e instanceof TouchEvent ? e.touches[0] : null
const clientX = e instanceof MouseEvent ? e.clientX : (touch?.clientX ?? 0)
const clientY = e instanceof MouseEvent ? e.clientY : (touch?.clientY ?? 0)
const newX = clientX - dragOffset.value.x
const newY = clientY - dragOffset.value.y
const w = terminalRef.value?.offsetWidth || 580
const h = terminalRef.value?.offsetHeight || 360
@@ -135,6 +243,8 @@ function stopDrag() {
hasCustomPosition.value = true
document.removeEventListener('mousemove', onDrag)
document.removeEventListener('mouseup', stopDrag)
document.removeEventListener('touchmove', onDrag)
document.removeEventListener('touchend', stopDrag)
}
// Resize functions
@@ -174,6 +284,26 @@ function stopResize() {
}
const terminalStyle = computed(() => {
// Mobile: bottom sheet with dynamic height
if (isMobile.value) {
// When keyboard is open, position above it
const bottomOffset = keyboardHeight.value > 0 ? `${keyboardHeight.value}px` : '0'
const maxH = keyboardHeight.value > 0
? `calc(100vh - ${keyboardHeight.value}px)`
: '90vh'
return {
position: 'fixed',
left: '0',
right: '0',
bottom: bottomOffset,
width: '100%',
height: `${sheetHeight.value}vh`,
maxHeight: maxH
}
}
// Desktop: floating window
const base = {
width: `${size.value.w}px`,
height: `${size.value.h}px`
@@ -401,6 +531,27 @@ function requestToken() {
}
}
// Mobile virtual keys - send special key sequences
function sendKey(key: string) {
if (!socket || socket.readyState !== WebSocket.OPEN) return
const keyMap: Record<string, string> = {
'up': '\x1b[A',
'down': '\x1b[B',
'left': '\x1b[D',
'right': '\x1b[C',
'alt-m': '\x1bm',
'ctrl-c': '\x03',
'tab': '\t',
'esc': '\x1b'
}
const data = keyMap[key]
if (data) {
socket.send(JSON.stringify({ type: 'input', data }))
}
}
watch(isOpen, async (open) => {
if (open) {
await nextTick()
@@ -423,11 +574,25 @@ watch(isOpen, async (open) => {
}
})
// Refit terminal when mobile sheet height or keyboard changes
watch([sheetHeight, keyboardHeight], () => {
if (isMobile.value && isOpen.value) {
nextTick(() => fitAddon?.fit())
}
})
onMounted(async () => {
// Global listeners for Ctrl+E
document.addEventListener('mousemove', trackMouse)
document.addEventListener('keydown', handleKeydown)
// Mobile detection
checkMobile()
window.addEventListener('resize', checkMobile)
// Virtual keyboard detection
setupKeyboardDetection()
if (isOpen.value) {
await nextTick()
initTerminal()
@@ -441,10 +606,16 @@ onBeforeUnmount(() => {
terminal?.dispose()
document.removeEventListener('mousemove', onDrag)
document.removeEventListener('mouseup', stopDrag)
document.removeEventListener('touchmove', onDrag)
document.removeEventListener('touchend', stopDrag)
document.removeEventListener('mousemove', onResize)
document.removeEventListener('mouseup', stopResize)
document.removeEventListener('mousemove', trackMouse)
document.removeEventListener('keydown', handleKeydown)
window.removeEventListener('resize', checkMobile)
if (window.visualViewport) {
window.visualViewport.removeEventListener('resize', handleViewportResize)
}
})
// Expose controls for MCP tools
@@ -492,12 +663,33 @@ defineExpose({
v-if="isOpen"
ref="terminalRef"
class="aero-win"
:class="{ dragging: isDragging, resizing: isResizing }"
:class="{
dragging: isDragging,
resizing: isResizing,
mobile: isMobile,
'sheet-dragging': isDraggingSheet
}"
:style="terminalStyle"
>
<div class="glass">
<!-- Mobile drag handle -->
<div
v-if="isMobile"
class="sheet-handle"
@touchstart.passive="startSheetDrag"
@touchmove.prevent="onSheetDrag"
@touchend="stopSheetDrag"
>
<div class="handle-bar"></div>
</div>
<!-- Titlebar -->
<div class="titlebar" @mousedown="startDrag">
<div
class="titlebar"
@mousedown="startDrag"
@touchstart.passive="startDrag"
@touchmove="onSheetDrag"
@touchend="stopSheetDrag"
>
<div class="left">
<!-- Nucleo Logo -->
<div class="nucleo-logo">
@@ -546,6 +738,21 @@ defineExpose({
<div class="content">
<div ref="terminalContainer" class="term"></div>
</div>
<!-- Mobile virtual keys -->
<div v-if="isMobile" class="virtual-keys">
<button @click="sendKey('esc')" class="vk">Esc</button>
<button @click="sendKey('tab')" class="vk">Tab</button>
<button @click="sendKey('ctrl-c')" class="vk ctrl">^C</button>
<button @click="sendKey('alt-m')" class="vk alt">Alt+M</button>
<div class="vk-arrows">
<button @click="sendKey('up')" class="vk arrow"></button>
<div class="vk-row">
<button @click="sendKey('left')" class="vk arrow"></button>
<button @click="sendKey('down')" class="vk arrow"></button>
<button @click="sendKey('right')" class="vk arrow"></button>
</div>
</div>
</div>
<!-- Resize handle -->
<div class="resize-handle" @mousedown="startResize"></div>
</div>
@@ -722,12 +929,126 @@ defineExpose({
@keyframes pulse { 0%,100% { opacity:1; } 50% { opacity:.4; } }
@media (max-width: 640px) {
.aero-win {
inset: auto 0 0 0 !important;
width: 100% !important;
height: 55% !important;
}
.glass { border-radius: 6px 6px 0 0; }
/* Mobile/tablet bottom sheet styles */
.aero-win.mobile {
min-width: unset;
min-height: unset;
max-width: 100%;
transition: height 0.25s cubic-bezier(0.4, 0, 0.2, 1);
}
.aero-win.mobile.sheet-dragging {
transition: none;
}
.aero-win.mobile .glass {
border-radius: 16px 16px 0 0;
padding-bottom: env(safe-area-inset-bottom, 0);
}
.sheet-handle {
display: flex;
justify-content: center;
align-items: center;
height: 24px;
cursor: grab;
touch-action: none;
background: rgba(255,255,255,0.15);
}
.sheet-handle:active {
cursor: grabbing;
}
.handle-bar {
width: 36px;
height: 4px;
background: rgba(0,0,0,0.3);
border-radius: 2px;
}
.aero-win.mobile .titlebar {
cursor: grab;
touch-action: none;
}
.aero-win.mobile .resize-handle {
display: none;
}
.aero-win.mobile .content {
flex: 1;
min-height: 100px;
}
/* Mobile animations */
.aero-win.mobile.win-slide-enter-from,
.aero-win.mobile.win-slide-leave-to {
transform: translateY(100%);
opacity: 1;
}
/* Virtual keys for mobile */
.virtual-keys {
display: flex;
align-items: center;
gap: 6px;
padding: 8px 10px;
background: rgba(30, 30, 30, 0.95);
border-top: 1px solid rgba(255, 255, 255, 0.1);
flex-shrink: 0;
flex-wrap: wrap;
}
.vk {
min-width: 40px;
height: 32px;
padding: 0 8px;
background: linear-gradient(180deg, #444 0%, #333 100%);
border: 1px solid rgba(255, 255, 255, 0.15);
border-radius: 5px;
color: #fff;
font-size: 11px;
font-weight: 500;
font-family: system-ui, sans-serif;
cursor: pointer;
touch-action: manipulation;
transition: all 0.1s;
}
.vk:active {
background: linear-gradient(180deg, #555 0%, #444 100%);
transform: scale(0.95);
}
.vk.ctrl {
background: linear-gradient(180deg, #c53030 0%, #9b2c2c 100%);
border-color: rgba(255, 100, 100, 0.3);
}
.vk.alt {
background: linear-gradient(180deg, #2563eb 0%, #1d4ed8 100%);
border-color: rgba(100, 150, 255, 0.3);
}
.vk-arrows {
display: flex;
flex-direction: column;
align-items: center;
gap: 2px;
margin-left: auto;
}
.vk-row {
display: flex;
gap: 2px;
}
.vk.arrow {
min-width: 32px;
width: 32px;
height: 26px;
padding: 0;
font-size: 10px;
}
</style>

View File

@@ -3,6 +3,29 @@ import { ref, computed, onMounted, onBeforeUnmount, watch } from 'vue'
import { useCanvasStore } from '../stores/canvas'
import { endpoints } from '../config/endpoints'
// Web Speech API types (not in default TS lib)
interface SpeechRecognitionEvent extends Event {
resultIndex: number
results: SpeechRecognitionResultList
}
interface SpeechRecognitionErrorEvent extends Event {
error: string
message?: string
}
interface SpeechRecognition extends EventTarget {
continuous: boolean
interimResults: boolean
lang: string
onresult: ((event: SpeechRecognitionEvent) => void) | null
onerror: ((event: SpeechRecognitionErrorEvent) => void) | null
onend: (() => void) | null
start(): void
stop(): void
abort(): void
}
const props = defineProps<{
modelValue: boolean
}>()
@@ -37,7 +60,7 @@ const dragOffset = ref({ x: 0, y: 0 })
const containerRef = ref<HTMLElement | null>(null)
// Speech recognition (Web Speech API)
let recognition: SpeechRecognition | null = null
let recognition: SpeechRecognition | null = null as SpeechRecognition | null
// WebSocket connection to terminal
const WS_URL = endpoints.terminal
@@ -58,7 +81,7 @@ const WHISPER_WS_URL = endpoints.whisper
let whisperSocket: WebSocket | null = null
let mediaRecorder: MediaRecorder | null = null
let audioChunks: Blob[] = []
let lastTranscriptLength = 0 // Track length of last transcription to show only new text
// Note: transcript length tracking removed - using full transcription now
let chunkInterval: number | null = null
const CHUNK_INTERVAL_MS = 3000 // Send audio every 3 seconds
let mediaStream: MediaStream | null = null
@@ -68,6 +91,103 @@ const audioDevices = ref<MediaDeviceInfo[]>([])
const selectedDeviceId = ref<string>('')
const showMicSelector = ref(false)
// ============ MOBILE DETECTION & AUDIO FORMAT ============
const isMobile = ref(false)
const supportedMimeType = ref('audio/webm;codecs=opus')
const sheetHeight = ref(45) // percentage of viewport for mobile
const isDraggingSheet = ref(false)
const sheetDragStart = ref({ y: 0, height: 0 })
const keyboardHeight = ref(0)
const snapPoints = [25, 45, 70] // collapsed, default, expanded
function checkMobile() {
isMobile.value = window.innerWidth <= 640 || /iPhone|iPad|iPod|Android/i.test(navigator.userAgent)
}
// Virtual keyboard detection
function setupKeyboardDetection() {
if (window.visualViewport) {
window.visualViewport.addEventListener('resize', handleViewportResize)
}
}
function handleViewportResize() {
if (!window.visualViewport || !isMobile.value) return
const viewportHeight = window.visualViewport.height
const windowHeight = window.innerHeight
const diff = windowHeight - viewportHeight
if (diff > 100) {
keyboardHeight.value = diff
// Auto-expand when keyboard opens
if (sheetHeight.value < 45 && isOpen.value) {
sheetHeight.value = 45
}
} else {
keyboardHeight.value = 0
}
}
// Find nearest snap point
function findNearestSnap(height: number): number {
return snapPoints.reduce((prev, curr) =>
Math.abs(curr - height) < Math.abs(prev - height) ? curr : prev
)
}
// Sheet touch handlers
function startSheetDrag(e: TouchEvent) {
if (!isMobile.value) return
const touch = e.touches[0]
if (!touch) return
isDraggingSheet.value = true
sheetDragStart.value = { y: touch.clientY, height: sheetHeight.value }
}
function onSheetDrag(e: TouchEvent) {
if (!isDraggingSheet.value || !isMobile.value) return
const touch = e.touches[0]
if (!touch) return
const deltaY = sheetDragStart.value.y - touch.clientY
const deltaPercent = (deltaY / window.innerHeight) * 100
const newHeight = sheetDragStart.value.height + deltaPercent
sheetHeight.value = Math.max(20, Math.min(80, newHeight))
}
function stopSheetDrag() {
if (!isDraggingSheet.value) return
isDraggingSheet.value = false
sheetHeight.value = findNearestSnap(sheetHeight.value)
}
function detectAudioFormat(): string {
// Test formats in order of preference
const formats = [
'audio/webm;codecs=opus',
'audio/webm',
'audio/mp4',
'audio/mp4;codecs=mp4a.40.2',
'audio/aac',
'audio/ogg;codecs=opus',
'audio/wav'
]
for (const format of formats) {
if (MediaRecorder.isTypeSupported(format)) {
console.log(`[Voice] Using audio format: ${format}`)
return format
}
}
// Fallback - let browser choose
console.warn('[Voice] No preferred format supported, using default')
return ''
}
// ============ AUDIO PLAYBACK (DEBUG) ============
const lastAudioUrl = ref<string>('')
const isPlayingAudio = ref(false)
@@ -180,14 +300,24 @@ function closeMicSelector(e: MouseEvent) {
}
}
const displayText = computed(() => {
if (interimTranscript.value) {
return transcript.value + ' ' + interimTranscript.value
}
return transcript.value || 'Presiona el micrófono o mantén Ctrl+Space...'
})
const containerStyle = computed(() => {
// Mobile: bottom sheet
if (isMobile.value) {
const heightPx = keyboardHeight.value > 0
? `calc(${sheetHeight.value}vh - ${keyboardHeight.value}px)`
: `${sheetHeight.value}vh`
return {
inset: 'auto 0 0 0',
width: '100%',
height: heightPx,
maxHeight: keyboardHeight.value > 0
? `calc(100vh - ${keyboardHeight.value}px)`
: '80vh'
}
}
// Desktop: floating window
if (!hasCustomPosition.value) {
return { bottom: '80px', left: '16px' }
}
@@ -218,6 +348,8 @@ function initRecognition() {
for (let i = event.resultIndex; i < event.results.length; i++) {
const result = event.results[i]
if (!result || !result[0]) continue
if (result.isFinal) {
final += result[0].transcript + ' '
} else {
@@ -414,8 +546,6 @@ function connectWhisperSocket() {
}
}
// Update last transcript length for next partial
lastTranscriptLength = fullText.length
} else if (msg.error) {
error.value = msg.error
console.error('[Voice] Whisper error:', msg.error)
@@ -452,14 +582,24 @@ function disconnectWhisperSocket() {
async function startWhisperRecording() {
try {
const audioConstraints: MediaTrackConstraints = selectedDeviceId.value
? { deviceId: { exact: selectedDeviceId.value } }
: {}
// Mobile-optimized audio constraints
const audioConstraints: MediaTrackConstraints = {
echoCancellation: true,
noiseSuppression: true,
autoGainControl: true,
...(selectedDeviceId.value ? { deviceId: { exact: selectedDeviceId.value } } : {})
}
mediaStream = await navigator.mediaDevices.getUserMedia({ audio: audioConstraints })
mediaRecorder = new MediaRecorder(mediaStream, {
mimeType: 'audio/webm;codecs=opus'
})
// Use detected supported format
const recorderOptions: MediaRecorderOptions = {}
if (supportedMimeType.value) {
recorderOptions.mimeType = supportedMimeType.value
}
mediaRecorder = new MediaRecorder(mediaStream, recorderOptions)
console.log(`[Voice] MediaRecorder using: ${mediaRecorder.mimeType}`)
audioChunks = []
@@ -471,7 +611,6 @@ async function startWhisperRecording() {
// Reset state for new recording
audioChunks = []
lastTranscriptLength = 0
// Start recording
mediaRecorder.start(100) // Collect data every 100ms
@@ -510,7 +649,6 @@ function sendAudioChunk(isFinal: boolean) {
// Clear chunks only if final
if (isFinal) {
audioChunks = []
lastTranscriptLength = 0
// Save audio for playback debugging
saveAudioForPlayback(audioBlob)
}
@@ -867,6 +1005,16 @@ onMounted(async () => {
document.addEventListener('keydown', handleKeyDown, { capture: true })
document.addEventListener('keyup', handleKeyUp, { capture: true })
// Mobile detection
checkMobile()
window.addEventListener('resize', checkMobile)
// Virtual keyboard detection for mobile
setupKeyboardDetection()
// Detect supported audio format
supportedMimeType.value = detectAudioFormat()
// Load available audio devices
await loadAudioDevices()
@@ -905,6 +1053,10 @@ onBeforeUnmount(() => {
document.removeEventListener('mousemove', onDrag)
document.removeEventListener('mouseup', stopDrag)
document.removeEventListener('click', closeMicSelector)
window.removeEventListener('resize', checkMobile)
if (window.visualViewport) {
window.visualViewport.removeEventListener('resize', handleViewportResize)
}
if (holdTimeout) clearTimeout(holdTimeout)
})
@@ -937,12 +1089,32 @@ defineExpose({
v-if="isOpen"
ref="containerRef"
class="voice-window"
:class="{ dragging: isDragging }"
:class="{
dragging: isDragging,
mobile: isMobile,
'sheet-dragging': isDraggingSheet
}"
:style="containerStyle"
>
<div class="glass">
<!-- Mobile drag handle -->
<div
v-if="isMobile"
class="sheet-handle"
@touchstart.passive="startSheetDrag"
@touchmove.prevent="onSheetDrag"
@touchend="stopSheetDrag"
>
<div class="handle-bar"></div>
</div>
<!-- Titlebar -->
<div class="titlebar" @mousedown="startDrag">
<div
class="titlebar"
@mousedown="startDrag"
@touchstart.passive="startSheetDrag"
@touchmove.prevent="onSheetDrag"
@touchend="stopSheetDrag"
>
<div class="left">
<svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
<path d="M12 1a3 3 0 0 0-3 3v8a3 3 0 0 0 6 0V4a3 3 0 0 0-3-3z"/>
@@ -1533,4 +1705,64 @@ defineExpose({
opacity: 0;
transform: translateY(16px) scale(0.95);
}
/* Mobile bottom sheet styles */
.voice-window.mobile {
width: 100% !important;
transition: height 0.25s cubic-bezier(0.4, 0, 0.2, 1);
}
.voice-window.mobile.sheet-dragging {
transition: none;
}
.voice-window.mobile .glass {
height: 100%;
border-radius: 16px 16px 0 0;
padding-bottom: env(safe-area-inset-bottom, 0);
}
.sheet-handle {
display: flex;
justify-content: center;
align-items: center;
height: 20px;
cursor: grab;
touch-action: none;
background: rgba(255, 255, 255, 0.15);
flex-shrink: 0;
}
.sheet-handle:active {
cursor: grabbing;
}
.handle-bar {
width: 36px;
height: 4px;
background: rgba(0, 0, 0, 0.3);
border-radius: 2px;
}
.voice-window.mobile .titlebar {
cursor: grab;
touch-action: none;
}
.voice-window.mobile .content {
flex: 1;
min-height: 60px;
max-height: none;
}
.voice-window.mobile .controls {
flex-shrink: 0;
}
/* Mobile animations */
.voice-window.mobile.voice-slide-enter-from,
.voice-window.mobile.voice-slide-leave-to {
transform: translateY(100%);
opacity: 1;
}
</style>