diff --git a/frontend/src/components/FloatingTerminal.vue b/frontend/src/components/FloatingTerminal.vue index 33a59b7..878fe61 100644 --- a/frontend/src/components/FloatingTerminal.vue +++ b/frontend/src/components/FloatingTerminal.vue @@ -38,6 +38,14 @@ const dragOffset = ref({ x: 0, y: 0 }) const isResizing = ref(false) const size = ref({ w: 580, h: 360 }) +// Mobile bottom sheet state +const isMobile = ref(false) +const sheetHeight = ref(55) // percentage of viewport +const isDraggingSheet = ref(false) +const sheetDragStart = ref({ y: 0, height: 0 }) +const keyboardHeight = ref(0) +const snapPoints = [20, 55, 85] // collapsed, half, full (percentages) + let terminal: Terminal | null = null let fitAddon: FitAddon | null = null let socket: WebSocket | null = null @@ -58,6 +66,84 @@ function trackMouse(e: MouseEvent) { mousePos.value = { x: e.clientX, y: e.clientY } } +// Mobile/tablet detection - show virtual keys on touch devices +function checkMobile() { + const isTouchDevice = 'ontouchstart' in window || navigator.maxTouchPoints > 0 + const isSmallScreen = window.innerWidth <= 1024 + isMobile.value = isTouchDevice && isSmallScreen +} + +// Virtual keyboard detection using visualViewport API +function setupKeyboardDetection() { + if (window.visualViewport) { + window.visualViewport.addEventListener('resize', handleViewportResize) + } +} + +function handleViewportResize() { + if (!window.visualViewport || !isMobile.value) return + + const viewportHeight = window.visualViewport.height + const windowHeight = window.innerHeight + const diff = windowHeight - viewportHeight + + // If difference > 100px, keyboard is probably open + if (diff > 100) { + keyboardHeight.value = diff + // Auto-expand sheet when keyboard opens if it's too small + if (sheetHeight.value < 55 && isOpen.value) { + sheetHeight.value = 55 + } + } else { + keyboardHeight.value = 0 + } +} + +// Find nearest snap point +function findNearestSnap(height: number): number { + return snapPoints.reduce((prev, curr) => + Math.abs(curr - height) < Math.abs(prev - height) ? curr : prev + ) +} + +// Mobile sheet touch handlers +function startSheetDrag(e: TouchEvent) { + if (!isMobile.value) return + + const touch = e.touches[0] + if (!touch) return + + isDraggingSheet.value = true + sheetDragStart.value = { + y: touch.clientY, + height: sheetHeight.value + } +} + +function onSheetDrag(e: TouchEvent) { + if (!isDraggingSheet.value || !isMobile.value) return + + const touch = e.touches[0] + if (!touch) return + + const deltaY = sheetDragStart.value.y - touch.clientY + const deltaPercent = (deltaY / window.innerHeight) * 100 + const newHeight = sheetDragStart.value.height + deltaPercent + + // Clamp between 15% and 90% + sheetHeight.value = Math.max(15, Math.min(90, newHeight)) +} + +function stopSheetDrag() { + if (!isDraggingSheet.value) return + + isDraggingSheet.value = false + // Snap to nearest point + sheetHeight.value = findNearestSnap(sheetHeight.value) + + nextTick(() => fitAddon?.fit()) +} + function toggleTerminal() { const now = Date.now() if (now - lastToggle < 150) return // Debounce 150ms @@ -89,31 +175,53 @@ function handleKeydown(e: KeyboardEvent) { } } -function startDrag(e: MouseEvent) { +function startDrag(e: MouseEvent | TouchEvent) { if ((e.target as HTMLElement).closest('.window-controls')) return + // On mobile, use sheet drag instead + if (isMobile.value) { + if (e instanceof TouchEvent) { + startSheetDrag(e) + } + return + } + isDragging.value = true const rect = terminalRef.value?.getBoundingClientRect() + const touch = e instanceof TouchEvent ? e.touches[0] : null + const clientX = e instanceof MouseEvent ? e.clientX : (touch?.clientX ?? 0) + const clientY = e instanceof MouseEvent ? e.clientY : (touch?.clientY ?? 0) + if (rect) { // Capture actual position if using default bottom/right if (!hasCustomPosition.value) { position.value = { x: rect.left, y: rect.top } } dragOffset.value = { - x: e.clientX - rect.left, - y: e.clientY - rect.top + x: clientX - rect.left, + y: clientY - rect.top } } document.addEventListener('mousemove', onDrag) document.addEventListener('mouseup', stopDrag) + document.addEventListener('touchmove', onDrag, { passive: false }) + document.addEventListener('touchend', stopDrag) } -function onDrag(e: MouseEvent) { +function onDrag(e: MouseEvent | TouchEvent) { if (!isDragging.value) return - const newX = e.clientX - dragOffset.value.x - const newY = e.clientY - dragOffset.value.y + if (e instanceof TouchEvent) { + e.preventDefault() + } + + const touch = e instanceof TouchEvent ? e.touches[0] : null + const clientX = e instanceof MouseEvent ? e.clientX : (touch?.clientX ?? 0) + const clientY = e instanceof MouseEvent ? e.clientY : (touch?.clientY ?? 0) + + const newX = clientX - dragOffset.value.x + const newY = clientY - dragOffset.value.y const w = terminalRef.value?.offsetWidth || 580 const h = terminalRef.value?.offsetHeight || 360 @@ -135,6 +243,8 @@ function stopDrag() { hasCustomPosition.value = true document.removeEventListener('mousemove', onDrag) document.removeEventListener('mouseup', stopDrag) + document.removeEventListener('touchmove', onDrag) + document.removeEventListener('touchend', stopDrag) } // Resize functions @@ -174,6 +284,26 @@ function stopResize() { } const terminalStyle = computed(() => { + // Mobile: bottom sheet with dynamic height + if (isMobile.value) { + // When keyboard is open, position above it + const bottomOffset = keyboardHeight.value > 0 ? `${keyboardHeight.value}px` : '0' + const maxH = keyboardHeight.value > 0 + ? `calc(100vh - ${keyboardHeight.value}px)` + : '90vh' + + return { + position: 'fixed', + left: '0', + right: '0', + bottom: bottomOffset, + width: '100%', + height: `${sheetHeight.value}vh`, + maxHeight: maxH + } + } + + // Desktop: floating window const base = { width: `${size.value.w}px`, height: `${size.value.h}px` @@ -401,6 +531,27 @@ function requestToken() { } } +// Mobile virtual keys - send special key sequences +function sendKey(key: string) { + if (!socket || socket.readyState !== WebSocket.OPEN) return + + const keyMap: Record = { + 'up': '\x1b[A', + 'down': '\x1b[B', + 'left': '\x1b[D', + 'right': '\x1b[C', + 'alt-m': '\x1bm', + 'ctrl-c': '\x03', + 'tab': '\t', + 'esc': '\x1b' + } + + const data = keyMap[key] + if (data) { + socket.send(JSON.stringify({ type: 'input', data })) + } +} + watch(isOpen, async (open) => { if (open) { await nextTick() @@ -423,11 +574,25 @@ watch(isOpen, async (open) => { } }) +// Refit terminal when mobile sheet height or keyboard changes +watch([sheetHeight, keyboardHeight], () => { + if (isMobile.value && isOpen.value) { + nextTick(() => fitAddon?.fit()) + } +}) + onMounted(async () => { // Global listeners for Ctrl+E document.addEventListener('mousemove', trackMouse) document.addEventListener('keydown', handleKeydown) + // Mobile detection + checkMobile() + window.addEventListener('resize', checkMobile) + + // Virtual keyboard detection + setupKeyboardDetection() + if (isOpen.value) { await nextTick() initTerminal() @@ -441,10 +606,16 @@ onBeforeUnmount(() => { terminal?.dispose() document.removeEventListener('mousemove', onDrag) document.removeEventListener('mouseup', stopDrag) + document.removeEventListener('touchmove', onDrag) + document.removeEventListener('touchend', stopDrag) document.removeEventListener('mousemove', onResize) document.removeEventListener('mouseup', stopResize) document.removeEventListener('mousemove', trackMouse) document.removeEventListener('keydown', handleKeydown) + window.removeEventListener('resize', checkMobile) + if (window.visualViewport) { + window.visualViewport.removeEventListener('resize', handleViewportResize) + } }) // Expose controls for MCP tools @@ -492,12 +663,33 @@ defineExpose({ v-if="isOpen" ref="terminalRef" class="aero-win" - :class="{ dragging: isDragging, resizing: isResizing }" + :class="{ + dragging: isDragging, + resizing: isResizing, + mobile: isMobile, + 'sheet-dragging': isDraggingSheet + }" :style="terminalStyle" >
+ +
+
+
-
+
@@ -722,12 +929,126 @@ defineExpose({ @keyframes pulse { 0%,100% { opacity:1; } 50% { opacity:.4; } } -@media (max-width: 640px) { - .aero-win { - inset: auto 0 0 0 !important; - width: 100% !important; - height: 55% !important; - } - .glass { border-radius: 6px 6px 0 0; } +/* Mobile/tablet bottom sheet styles */ +.aero-win.mobile { + min-width: unset; + min-height: unset; + max-width: 100%; + transition: height 0.25s cubic-bezier(0.4, 0, 0.2, 1); +} + +.aero-win.mobile.sheet-dragging { + transition: none; +} + +.aero-win.mobile .glass { + border-radius: 16px 16px 0 0; + padding-bottom: env(safe-area-inset-bottom, 0); +} + +.sheet-handle { + display: flex; + justify-content: center; + align-items: center; + height: 24px; + cursor: grab; + touch-action: none; + background: rgba(255,255,255,0.15); +} + +.sheet-handle:active { + cursor: grabbing; +} + +.handle-bar { + width: 36px; + height: 4px; + background: rgba(0,0,0,0.3); + border-radius: 2px; +} + +.aero-win.mobile .titlebar { + cursor: grab; + touch-action: none; +} + +.aero-win.mobile .resize-handle { + display: none; +} + +.aero-win.mobile .content { + flex: 1; + min-height: 100px; +} + +/* Mobile animations */ +.aero-win.mobile.win-slide-enter-from, +.aero-win.mobile.win-slide-leave-to { + transform: translateY(100%); + opacity: 1; +} + +/* Virtual keys for mobile */ +.virtual-keys { + display: flex; + align-items: center; + gap: 6px; + padding: 8px 10px; + background: rgba(30, 30, 30, 0.95); + border-top: 1px solid rgba(255, 255, 255, 0.1); + flex-shrink: 0; + flex-wrap: wrap; +} + +.vk { + min-width: 40px; + height: 32px; + padding: 0 8px; + background: linear-gradient(180deg, #444 0%, #333 100%); + border: 1px solid rgba(255, 255, 255, 0.15); + border-radius: 5px; + color: #fff; + font-size: 11px; + font-weight: 500; + font-family: system-ui, sans-serif; + cursor: pointer; + touch-action: manipulation; + transition: all 0.1s; +} + +.vk:active { + background: linear-gradient(180deg, #555 0%, #444 100%); + transform: scale(0.95); +} + +.vk.ctrl { + background: linear-gradient(180deg, #c53030 0%, #9b2c2c 100%); + border-color: rgba(255, 100, 100, 0.3); +} + +.vk.alt { + background: linear-gradient(180deg, #2563eb 0%, #1d4ed8 100%); + border-color: rgba(100, 150, 255, 0.3); +} + +.vk-arrows { + display: flex; + flex-direction: column; + align-items: center; + gap: 2px; + margin-left: auto; +} + +.vk-row { + display: flex; + gap: 2px; +} + +.vk.arrow { + min-width: 32px; + width: 32px; + height: 26px; + padding: 0; + font-size: 10px; } diff --git a/frontend/src/components/FloatingVoice.vue b/frontend/src/components/FloatingVoice.vue index 9d9495f..be4d940 100644 --- a/frontend/src/components/FloatingVoice.vue +++ b/frontend/src/components/FloatingVoice.vue @@ -3,6 +3,29 @@ import { ref, computed, onMounted, onBeforeUnmount, watch } from 'vue' import { useCanvasStore } from '../stores/canvas' import { endpoints } from '../config/endpoints' +// Web Speech API types (not in default TS lib) +interface SpeechRecognitionEvent extends Event { + resultIndex: number + results: SpeechRecognitionResultList +} + +interface SpeechRecognitionErrorEvent extends Event { + error: string + message?: string +} + +interface SpeechRecognition extends EventTarget { + continuous: boolean + interimResults: boolean + lang: string + onresult: ((event: SpeechRecognitionEvent) => void) | null + onerror: ((event: SpeechRecognitionErrorEvent) => void) | null + onend: (() => void) | null + start(): void + stop(): void + abort(): void +} + const props = defineProps<{ modelValue: boolean }>() @@ -37,7 +60,7 @@ const dragOffset = ref({ x: 0, y: 0 }) const containerRef = ref(null) // Speech recognition (Web Speech API) -let recognition: SpeechRecognition | null = null +let recognition: SpeechRecognition | null = null as SpeechRecognition | null // WebSocket connection to terminal const WS_URL = endpoints.terminal @@ -58,7 +81,7 @@ const WHISPER_WS_URL = endpoints.whisper let whisperSocket: WebSocket | null = null let mediaRecorder: MediaRecorder | null = null let audioChunks: Blob[] = [] -let lastTranscriptLength = 0 // Track length of last transcription to show only new text +// Note: transcript length tracking removed - using full transcription now let chunkInterval: number | null = null const CHUNK_INTERVAL_MS = 3000 // Send audio every 3 seconds let mediaStream: MediaStream | null = null @@ -68,6 +91,103 @@ const audioDevices = ref([]) const selectedDeviceId = ref('') const showMicSelector = ref(false) +// ============ MOBILE DETECTION & AUDIO FORMAT ============ +const isMobile = ref(false) +const supportedMimeType = ref('audio/webm;codecs=opus') +const sheetHeight = ref(45) // percentage of viewport for mobile +const isDraggingSheet = ref(false) +const sheetDragStart = ref({ y: 0, height: 0 }) +const keyboardHeight = ref(0) +const snapPoints = [25, 45, 70] // collapsed, default, expanded + +function checkMobile() { + isMobile.value = window.innerWidth <= 640 || /iPhone|iPad|iPod|Android/i.test(navigator.userAgent) +} + +// Virtual keyboard detection +function setupKeyboardDetection() { + if (window.visualViewport) { + window.visualViewport.addEventListener('resize', handleViewportResize) + } +} + +function handleViewportResize() { + if (!window.visualViewport || !isMobile.value) return + + const viewportHeight = window.visualViewport.height + const windowHeight = window.innerHeight + const diff = windowHeight - viewportHeight + + if (diff > 100) { + keyboardHeight.value = diff + // Auto-expand when keyboard opens + if (sheetHeight.value < 45 && isOpen.value) { + sheetHeight.value = 45 + } + } else { + keyboardHeight.value = 0 + } +} + +// Find nearest snap point +function findNearestSnap(height: number): number { + return snapPoints.reduce((prev, curr) => + Math.abs(curr - height) < Math.abs(prev - height) ? curr : prev + ) +} + +// Sheet touch handlers +function startSheetDrag(e: TouchEvent) { + if (!isMobile.value) return + const touch = e.touches[0] + if (!touch) return + + isDraggingSheet.value = true + sheetDragStart.value = { y: touch.clientY, height: sheetHeight.value } +} + +function onSheetDrag(e: TouchEvent) { + if (!isDraggingSheet.value || !isMobile.value) return + const touch = e.touches[0] + if (!touch) return + + const deltaY = sheetDragStart.value.y - touch.clientY + const deltaPercent = (deltaY / window.innerHeight) * 100 + const newHeight = sheetDragStart.value.height + deltaPercent + + sheetHeight.value = Math.max(20, Math.min(80, newHeight)) +} + +function stopSheetDrag() { + if (!isDraggingSheet.value) return + isDraggingSheet.value = false + sheetHeight.value = findNearestSnap(sheetHeight.value) +} + +function detectAudioFormat(): string { + // Test formats in order of preference + const formats = [ + 'audio/webm;codecs=opus', + 'audio/webm', + 'audio/mp4', + 'audio/mp4;codecs=mp4a.40.2', + 'audio/aac', + 'audio/ogg;codecs=opus', + 'audio/wav' + ] + + for (const format of formats) { + if (MediaRecorder.isTypeSupported(format)) { + console.log(`[Voice] Using audio format: ${format}`) + return format + } + } + + // Fallback - let browser choose + console.warn('[Voice] No preferred format supported, using default') + return '' +} + // ============ AUDIO PLAYBACK (DEBUG) ============ const lastAudioUrl = ref('') const isPlayingAudio = ref(false) @@ -180,14 +300,24 @@ function closeMicSelector(e: MouseEvent) { } } -const displayText = computed(() => { - if (interimTranscript.value) { - return transcript.value + ' ' + interimTranscript.value - } - return transcript.value || 'Presiona el micrófono o mantén Ctrl+Space...' -}) - const containerStyle = computed(() => { + // Mobile: bottom sheet + if (isMobile.value) { + const heightPx = keyboardHeight.value > 0 + ? `calc(${sheetHeight.value}vh - ${keyboardHeight.value}px)` + : `${sheetHeight.value}vh` + + return { + inset: 'auto 0 0 0', + width: '100%', + height: heightPx, + maxHeight: keyboardHeight.value > 0 + ? `calc(100vh - ${keyboardHeight.value}px)` + : '80vh' + } + } + + // Desktop: floating window if (!hasCustomPosition.value) { return { bottom: '80px', left: '16px' } } @@ -218,6 +348,8 @@ function initRecognition() { for (let i = event.resultIndex; i < event.results.length; i++) { const result = event.results[i] + if (!result || !result[0]) continue + if (result.isFinal) { final += result[0].transcript + ' ' } else { @@ -414,8 +546,6 @@ function connectWhisperSocket() { } } - // Update last transcript length for next partial - lastTranscriptLength = fullText.length } else if (msg.error) { error.value = msg.error console.error('[Voice] Whisper error:', msg.error) @@ -452,14 +582,24 @@ function disconnectWhisperSocket() { async function startWhisperRecording() { try { - const audioConstraints: MediaTrackConstraints = selectedDeviceId.value - ? { deviceId: { exact: selectedDeviceId.value } } - : {} + // Mobile-optimized audio constraints + const audioConstraints: MediaTrackConstraints = { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + ...(selectedDeviceId.value ? { deviceId: { exact: selectedDeviceId.value } } : {}) + } + mediaStream = await navigator.mediaDevices.getUserMedia({ audio: audioConstraints }) - mediaRecorder = new MediaRecorder(mediaStream, { - mimeType: 'audio/webm;codecs=opus' - }) + // Use detected supported format + const recorderOptions: MediaRecorderOptions = {} + if (supportedMimeType.value) { + recorderOptions.mimeType = supportedMimeType.value + } + + mediaRecorder = new MediaRecorder(mediaStream, recorderOptions) + console.log(`[Voice] MediaRecorder using: ${mediaRecorder.mimeType}`) audioChunks = [] @@ -471,7 +611,6 @@ async function startWhisperRecording() { // Reset state for new recording audioChunks = [] - lastTranscriptLength = 0 // Start recording mediaRecorder.start(100) // Collect data every 100ms @@ -510,7 +649,6 @@ function sendAudioChunk(isFinal: boolean) { // Clear chunks only if final if (isFinal) { audioChunks = [] - lastTranscriptLength = 0 // Save audio for playback debugging saveAudioForPlayback(audioBlob) } @@ -867,6 +1005,16 @@ onMounted(async () => { document.addEventListener('keydown', handleKeyDown, { capture: true }) document.addEventListener('keyup', handleKeyUp, { capture: true }) + // Mobile detection + checkMobile() + window.addEventListener('resize', checkMobile) + + // Virtual keyboard detection for mobile + setupKeyboardDetection() + + // Detect supported audio format + supportedMimeType.value = detectAudioFormat() + // Load available audio devices await loadAudioDevices() @@ -905,6 +1053,10 @@ onBeforeUnmount(() => { document.removeEventListener('mousemove', onDrag) document.removeEventListener('mouseup', stopDrag) document.removeEventListener('click', closeMicSelector) + window.removeEventListener('resize', checkMobile) + if (window.visualViewport) { + window.visualViewport.removeEventListener('resize', handleViewportResize) + } if (holdTimeout) clearTimeout(holdTimeout) }) @@ -937,12 +1089,32 @@ defineExpose({ v-if="isOpen" ref="containerRef" class="voice-window" - :class="{ dragging: isDragging }" + :class="{ + dragging: isDragging, + mobile: isMobile, + 'sheet-dragging': isDraggingSheet + }" :style="containerStyle" >
+ +
+
+
-
+
@@ -1533,4 +1705,64 @@ defineExpose({ opacity: 0; transform: translateY(16px) scale(0.95); } + +/* Mobile bottom sheet styles */ +.voice-window.mobile { + width: 100% !important; + transition: height 0.25s cubic-bezier(0.4, 0, 0.2, 1); +} + +.voice-window.mobile.sheet-dragging { + transition: none; +} + +.voice-window.mobile .glass { + height: 100%; + border-radius: 16px 16px 0 0; + padding-bottom: env(safe-area-inset-bottom, 0); +} + +.sheet-handle { + display: flex; + justify-content: center; + align-items: center; + height: 20px; + cursor: grab; + touch-action: none; + background: rgba(255, 255, 255, 0.15); + flex-shrink: 0; +} + +.sheet-handle:active { + cursor: grabbing; +} + +.handle-bar { + width: 36px; + height: 4px; + background: rgba(0, 0, 0, 0.3); + border-radius: 2px; +} + +.voice-window.mobile .titlebar { + cursor: grab; + touch-action: none; +} + +.voice-window.mobile .content { + flex: 1; + min-height: 60px; + max-height: none; +} + +.voice-window.mobile .controls { + flex-shrink: 0; +} + +/* Mobile animations */ +.voice-window.mobile.voice-slide-enter-from, +.voice-window.mobile.voice-slide-leave-to { + transform: translateY(100%); + opacity: 1; +}