nucleo000/agent-ui
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Add mobile support for terminal and voice components

Browse Source 
- Terminal: bottom sheet with drag handle and snap points (20%, 55%, 85%)
- Terminal: virtual keys bar (arrows, Esc, Tab, Ctrl+C, Alt+M)
- Terminal: keyboard detection adjusts position above virtual keyboard
- Terminal: touch drag support for mobile/tablet devices
- Voice: bottom sheet behavior matching terminal
- Voice: auto-detect supported audio format (webm/mp4/aac)
- Voice: improved audio constraints for mobile quality
- Both: detect touch devices up to 1024px width
This commit is contained in:
josedario87
2026-02-14 03:52:54 -06:00
parent e51eb6749d
commit 759de1e010
2 changed files with 589 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

351
frontend/src/components/FloatingTerminal.vue
View File

@@ -38,6 +38,14 @@ const dragOffset = ref({ x: 0, y: 0 })
const isResizing = ref(false)
const size = ref({ w: 580, h: 360 })
// Mobile bottom sheet state
const isMobile = ref(false)
const sheetHeight = ref(55) // percentage of viewport
const isDraggingSheet = ref(false)
const sheetDragStart = ref({ y: 0, height: 0 })
const keyboardHeight = ref(0)
const snapPoints = [20, 55, 85] // collapsed, half, full (percentages)
let terminal: Terminal | null = null
let fitAddon: FitAddon | null = null
let socket: WebSocket | null = null
@@ -58,6 +66,84 @@ function trackMouse(e: MouseEvent) {
mousePos.value = { x: e.clientX, y: e.clientY }
}
// Mobile/tablet detection - show virtual keys on touch devices
function checkMobile() {
const isTouchDevice = 'ontouchstart' in window || navigator.maxTouchPoints > 0
const isSmallScreen = window.innerWidth <= 1024
isMobile.value = isTouchDevice && isSmallScreen
}
// Virtual keyboard detection using visualViewport API
function setupKeyboardDetection() {
if (window.visualViewport) {
window.visualViewport.addEventListener('resize', handleViewportResize)
}
}
function handleViewportResize() {
if (!window.visualViewport || !isMobile.value) return
const viewportHeight = window.visualViewport.height
const windowHeight = window.innerHeight
const diff = windowHeight - viewportHeight
// If difference > 100px, keyboard is probably open
if (diff > 100) {
keyboardHeight.value = diff
// Auto-expand sheet when keyboard opens if it's too small
if (sheetHeight.value < 55 && isOpen.value) {
sheetHeight.value = 55
}
} else {
keyboardHeight.value = 0
}
}
// Find nearest snap point
function findNearestSnap(height: number): number {
return snapPoints.reduce((prev, curr) =>
Math.abs(curr - height) < Math.abs(prev - height) ? curr : prev
)
}
// Mobile sheet touch handlers
function startSheetDrag(e: TouchEvent) {
if (!isMobile.value) return
const touch = e.touches[0]
if (!touch) return
isDraggingSheet.value = true
sheetDragStart.value = {
y: touch.clientY,
height: sheetHeight.value
}
}
function onSheetDrag(e: TouchEvent) {
if (!isDraggingSheet.value || !isMobile.value) return
const touch = e.touches[0]
if (!touch) return
const deltaY = sheetDragStart.value.y - touch.clientY
const deltaPercent = (deltaY / window.innerHeight) * 100
const newHeight = sheetDragStart.value.height + deltaPercent
// Clamp between 15% and 90%
sheetHeight.value = Math.max(15, Math.min(90, newHeight))
}
function stopSheetDrag() {
if (!isDraggingSheet.value) return
isDraggingSheet.value = false
// Snap to nearest point
sheetHeight.value = findNearestSnap(sheetHeight.value)
nextTick(() => fitAddon?.fit())
}
function toggleTerminal() {
const now = Date.now()
if (now - lastToggle < 150) return // Debounce 150ms
@@ -89,31 +175,53 @@ function handleKeydown(e: KeyboardEvent) {
}
}
function startDrag(e: MouseEvent) {
function startDrag(e: MouseEvent | TouchEvent) {
if ((e.target as HTMLElement).closest('.window-controls')) return
// On mobile, use sheet drag instead
if (isMobile.value) {
if (e instanceof TouchEvent) {
startSheetDrag(e)
}
return
}
isDragging.value = true
const rect = terminalRef.value?.getBoundingClientRect()
const touch = e instanceof TouchEvent ? e.touches[0] : null
const clientX = e instanceof MouseEvent ? e.clientX : (touch?.clientX ?? 0)
const clientY = e instanceof MouseEvent ? e.clientY : (touch?.clientY ?? 0)
if (rect) {
// Capture actual position if using default bottom/right
if (!hasCustomPosition.value) {
position.value = { x: rect.left, y: rect.top }
}
dragOffset.value = {
x: e.clientX - rect.left,
y: e.clientY - rect.top
x: clientX - rect.left,
y: clientY - rect.top
}
}
document.addEventListener('mousemove', onDrag)
document.addEventListener('mouseup', stopDrag)
document.addEventListener('touchmove', onDrag, { passive: false })
document.addEventListener('touchend', stopDrag)
}
function onDrag(e: MouseEvent) {
function onDrag(e: MouseEvent | TouchEvent) {
if (!isDragging.value) return
const newX = e.clientX - dragOffset.value.x
const newY = e.clientY - dragOffset.value.y
if (e instanceof TouchEvent) {
e.preventDefault()
}
const touch = e instanceof TouchEvent ? e.touches[0] : null
const clientX = e instanceof MouseEvent ? e.clientX : (touch?.clientX ?? 0)
const clientY = e instanceof MouseEvent ? e.clientY : (touch?.clientY ?? 0)
const newX = clientX - dragOffset.value.x
const newY = clientY - dragOffset.value.y
const w = terminalRef.value?.offsetWidth || 580
const h = terminalRef.value?.offsetHeight || 360
@@ -135,6 +243,8 @@ function stopDrag() {
hasCustomPosition.value = true
document.removeEventListener('mousemove', onDrag)
document.removeEventListener('mouseup', stopDrag)
document.removeEventListener('touchmove', onDrag)
document.removeEventListener('touchend', stopDrag)
}
// Resize functions
@@ -174,6 +284,26 @@ function stopResize() {
}
const terminalStyle = computed(() => {
// Mobile: bottom sheet with dynamic height
if (isMobile.value) {
// When keyboard is open, position above it
const bottomOffset = keyboardHeight.value > 0 ? `${keyboardHeight.value}px` : '0'
const maxH = keyboardHeight.value > 0
? `calc(100vh - ${keyboardHeight.value}px)`
: '90vh'
return {
position: 'fixed',
left: '0',
right: '0',
bottom: bottomOffset,
width: '100%',
height: `${sheetHeight.value}vh`,
maxHeight: maxH
}
}
// Desktop: floating window
const base = {
width: `${size.value.w}px`,
height: `${size.value.h}px`
@@ -401,6 +531,27 @@ function requestToken() {
}
}
// Mobile virtual keys - send special key sequences
function sendKey(key: string) {
if (!socket || socket.readyState !== WebSocket.OPEN) return
const keyMap: Record<string, string> = {
'up': '\x1b[A',
'down': '\x1b[B',
'left': '\x1b[D',
'right': '\x1b[C',
'alt-m': '\x1bm',
'ctrl-c': '\x03',
'tab': '\t',
'esc': '\x1b'
}
const data = keyMap[key]
if (data) {
socket.send(JSON.stringify({ type: 'input', data }))
}
}
watch(isOpen, async (open) => {
if (open) {
await nextTick()
@@ -423,11 +574,25 @@ watch(isOpen, async (open) => {
}
})
// Refit terminal when mobile sheet height or keyboard changes
watch([sheetHeight, keyboardHeight], () => {
if (isMobile.value && isOpen.value) {
nextTick(() => fitAddon?.fit())
}
})
onMounted(async () => {
// Global listeners for Ctrl+E
document.addEventListener('mousemove', trackMouse)
document.addEventListener('keydown', handleKeydown)
// Mobile detection
checkMobile()
window.addEventListener('resize', checkMobile)
// Virtual keyboard detection
setupKeyboardDetection()
if (isOpen.value) {
await nextTick()
initTerminal()
@@ -441,10 +606,16 @@ onBeforeUnmount(() => {
terminal?.dispose()
document.removeEventListener('mousemove', onDrag)
document.removeEventListener('mouseup', stopDrag)
document.removeEventListener('touchmove', onDrag)
document.removeEventListener('touchend', stopDrag)
document.removeEventListener('mousemove', onResize)
document.removeEventListener('mouseup', stopResize)
document.removeEventListener('mousemove', trackMouse)
document.removeEventListener('keydown', handleKeydown)
window.removeEventListener('resize', checkMobile)
if (window.visualViewport) {
window.visualViewport.removeEventListener('resize', handleViewportResize)
}
})
// Expose controls for MCP tools
@@ -492,12 +663,33 @@ defineExpose({
v-if="isOpen"
ref="terminalRef"
class="aero-win"
:class="{ dragging: isDragging, resizing: isResizing }"
:class="{
dragging: isDragging,
resizing: isResizing,
mobile: isMobile,
'sheet-dragging': isDraggingSheet
}"
:style="terminalStyle"
>
<div class="glass">
<!-- Mobile drag handle -->
<div
v-if="isMobile"
class="sheet-handle"
@touchstart.passive="startSheetDrag"
@touchmove.prevent="onSheetDrag"
@touchend="stopSheetDrag"
>
<div class="handle-bar"></div>
</div>
<!-- Titlebar -->
<div class="titlebar" @mousedown="startDrag">
<div
class="titlebar"
@mousedown="startDrag"
@touchstart.passive="startDrag"
@touchmove="onSheetDrag"
@touchend="stopSheetDrag"
>
<div class="left">
<!-- Nucleo Logo -->
<div class="nucleo-logo">
@@ -546,6 +738,21 @@ defineExpose({
<div class="content">
<div ref="terminalContainer" class="term"></div>
</div>
<!-- Mobile virtual keys -->
<div v-if="isMobile" class="virtual-keys">
<button @click="sendKey('esc')" class="vk">Esc</button>
<button @click="sendKey('tab')" class="vk">Tab</button>
<button @click="sendKey('ctrl-c')" class="vk ctrl">^C</button>
<button @click="sendKey('alt-m')" class="vk alt">Alt+M</button>
<div class="vk-arrows">
<button @click="sendKey('up')" class="vk arrow">▲</button>
<div class="vk-row">
<button @click="sendKey('left')" class="vk arrow">◀</button>
<button @click="sendKey('down')" class="vk arrow">▼</button>
<button @click="sendKey('right')" class="vk arrow">▶</button>
</div>
</div>
</div>
<!-- Resize handle -->
<div class="resize-handle" @mousedown="startResize"></div>
</div>
@@ -722,12 +929,126 @@ defineExpose({
@keyframes pulse { 0%,100% { opacity:1; } 50% { opacity:.4; } }
@media (max-width: 640px) {
.aero-win {
inset: auto 0 0 0 !important;
width: 100% !important;
height: 55% !important;
}
.glass { border-radius: 6px 6px 0 0; }
/* Mobile/tablet bottom sheet styles */
.aero-win.mobile {
min-width: unset;
min-height: unset;
max-width: 100%;
transition: height 0.25s cubic-bezier(0.4, 0, 0.2, 1);
}
.aero-win.mobile.sheet-dragging {
transition: none;
}
.aero-win.mobile .glass {
border-radius: 16px 16px 0 0;
padding-bottom: env(safe-area-inset-bottom, 0);
}
.sheet-handle {
display: flex;
justify-content: center;
align-items: center;
height: 24px;
cursor: grab;
touch-action: none;
background: rgba(255,255,255,0.15);
}
.sheet-handle:active {
cursor: grabbing;
}
.handle-bar {
width: 36px;
height: 4px;
background: rgba(0,0,0,0.3);
border-radius: 2px;
}
.aero-win.mobile .titlebar {
cursor: grab;
touch-action: none;
}
.aero-win.mobile .resize-handle {
display: none;
}
.aero-win.mobile .content {
flex: 1;
min-height: 100px;
}
/* Mobile animations */
.aero-win.mobile.win-slide-enter-from,
.aero-win.mobile.win-slide-leave-to {
transform: translateY(100%);
opacity: 1;
}
/* Virtual keys for mobile */
.virtual-keys {
display: flex;
align-items: center;
gap: 6px;
padding: 8px 10px;
background: rgba(30, 30, 30, 0.95);
border-top: 1px solid rgba(255, 255, 255, 0.1);
flex-shrink: 0;
flex-wrap: wrap;
}
.vk {
min-width: 40px;
height: 32px;
padding: 0 8px;
background: linear-gradient(180deg, #444 0%, #333 100%);
border: 1px solid rgba(255, 255, 255, 0.15);
border-radius: 5px;
color: #fff;
font-size: 11px;
font-weight: 500;
font-family: system-ui, sans-serif;
cursor: pointer;
touch-action: manipulation;
transition: all 0.1s;
}
.vk:active {
background: linear-gradient(180deg, #555 0%, #444 100%);
transform: scale(0.95);
}
.vk.ctrl {
background: linear-gradient(180deg, #c53030 0%, #9b2c2c 100%);
border-color: rgba(255, 100, 100, 0.3);
}
.vk.alt {
background: linear-gradient(180deg, #2563eb 0%, #1d4ed8 100%);
border-color: rgba(100, 150, 255, 0.3);
}
.vk-arrows {
display: flex;
flex-direction: column;
align-items: center;
gap: 2px;
margin-left: auto;
}
.vk-row {
display: flex;
gap: 2px;
}
.vk.arrow {
min-width: 32px;
width: 32px;
height: 26px;
padding: 0;
font-size: 10px;
}
</style>