From 51e27344c12d47ca963f01e21f16e4f2066dd08f Mon Sep 17 00:00:00 2001 From: W1NDes Date: Sat, 28 Feb 2026 02:36:23 +0800 Subject: [PATCH] =?UTF-8?q?Feat(viewport):=20=E6=B7=BB=E5=8A=A0=E5=AE=9E?= =?UTF-8?q?=E6=97=B6=E8=A7=A6=E6=8E=A7=E6=8B=96=E5=8A=A8,=20=E8=B7=B3?= =?UTF-8?q?=E5=B8=A7=E8=8A=82=E7=9C=81=E5=B8=A6=E5=AE=BD,=20=E5=92=8C?= =?UTF-8?q?=E7=A9=BA=E9=97=B2=E6=9A=82=E5=81=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 支持实时touch事件流式传输(touch_down/move/up), 拖动即时反馈 - 修复拖动开头触发点击, 绕过minitouch 50ms延迟 - 5s无操作后跳过未变化帧, 300s空闲停止截屏并显示蓝色提示 - 前端事件重构为handlePointerStart/Move/End, 支持10px阈值区分点击和拖动 --- module/webui/viewport.html | 257 +++++++++++++++++++++++++++---------- module/webui/viewport.py | 181 +++++++++++++++++++++----- 2 files changed, 339 insertions(+), 99 deletions(-) diff --git a/module/webui/viewport.html b/module/webui/viewport.html index 5ec35a3b0..85e0b5694 100644 --- a/module/webui/viewport.html +++ b/module/webui/viewport.html @@ -1,5 +1,6 @@ + @@ -292,6 +293,43 @@ display: block; } + /* Idle overlay - blue version of script overlay */ + .idle-overlay { + position: absolute; + top: 0; + left: 0; + right: 0; + bottom: 0; + background: rgba(0, 0, 0, 0.4); + display: none; + cursor: pointer; + z-index: 15; + } + + .idle-overlay.visible { + display: block; + } + + .idle-banner { + position: absolute; + top: 50%; + left: 50%; + transform: translate(-50%, -50%); + background: rgba(52, 120, 246, 0.9); + color: #fff; + padding: 12px 24px; + border-radius: 4px; + font-size: 14px; + font-weight: 500; + pointer-events: none; + z-index: 20; + display: none; + } + + .idle-banner.visible { + display: block; + } + /* Control button styles */ .control-btn { background: var(--bg-tertiary); @@ -321,6 +359,7 @@ } +
@@ -352,8 +391,8 @@
- - 60 + + 30
@@ -409,6 +448,8 @@
Script Running - Touch Disabled
+
+
💤 Idle - Click to Resume
@@ -513,6 +554,22 @@ let isMouseDown = false; let startX = 0, startY = 0; let lastX = 0, lastY = 0; + let supportsRawTouch = false; // Server tells us if raw touch is available + let isSwiping = false; // True once we've sent touch_down and are swiping + let pendingTouch = false; // True between mousedown and deciding tap vs swipe + let lastTouchMoveTime = 0; // Throttle touch_move events + let isIdle = false; // True when backend reports idle + + // Idle overlay elements + const idleOverlay = document.getElementById('idleOverlay'); + const idleBanner = document.getElementById('idleBanner'); + + // Click on idle overlay to resume + idleOverlay.addEventListener('click', () => { + if (isIdle) { + sendAction({ action: 'resume_idle' }); + } + }); // Page visibility handling document.addEventListener('visibilitychange', () => { @@ -521,8 +578,8 @@ isPaused = true; sendAction({ action: 'pause', paused: true }); console.log('Page hidden, streaming paused'); - } else { - // Page is visible, resume streaming + } else if (!isIdle) { + // Page is visible, resume streaming (but not if idle) isPaused = false; sendAction({ action: 'pause', paused: false }); console.log('Page visible, streaming resumed'); @@ -669,6 +726,11 @@ if (data.screenshot_method) { methodInfo.textContent = data.screenshot_method; } + + // Update raw touch support from server + if (data.supports_raw_touch !== undefined) { + supportsRawTouch = data.supports_raw_touch; + } } // Update stats @@ -695,6 +757,19 @@ scriptBanner.classList.remove('visible'); scriptOverlay.classList.remove('visible'); } + + // Idle state from backend + if (data.idle !== undefined) { + if (data.idle && !isIdle) { + isIdle = true; + idleOverlay.classList.add('visible'); + idleBanner.classList.add('visible'); + } else if (!data.idle && isIdle) { + isIdle = false; + idleOverlay.classList.remove('visible'); + idleBanner.classList.remove('visible'); + } + } } else if (data.type === 'error') { showError(data.message, data.code); } @@ -737,101 +812,142 @@ } } - // Event handlers - canvas.addEventListener('mousedown', (e) => { + // Helper: reset all touch/swipe state + function resetTouchState() { + isMouseDown = false; + isSwiping = false; + pendingTouch = false; + } + + // Helper: handle pointer start (mousedown / touchstart) + function handlePointerStart(canvasX, canvasY) { if (scriptRunning) return; - const rect = canvas.getBoundingClientRect(); - startX = e.clientX - rect.left; - startY = e.clientY - rect.top; - lastX = startX; - lastY = startY; + startX = canvasX; + startY = canvasY; + lastX = canvasX; + lastY = canvasY; isMouseDown = true; + pendingTouch = true; + isSwiping = false; + } + + // Helper: handle pointer move (mousemove / touchmove) + function handlePointerMove(canvasX, canvasY) { + if (!isMouseDown || scriptRunning) return; + lastX = canvasX; + lastY = canvasY; + + if (supportsRawTouch && pendingTouch) { + // Check if we've moved far enough to start a swipe + const dx = canvasX - startX; + const dy = canvasY - startY; + const distance = Math.sqrt(dx * dx + dy * dy); + if (distance >= 10) { + // Crossed threshold — start real-time swipe + pendingTouch = false; + isSwiping = true; + const startCoords = mapCoordinates(startX, startY); + const curCoords = mapCoordinates(canvasX, canvasY); + // Send combined down+move atomically to avoid click at start + sendAction({ + action: 'swipe_start', + x1: startCoords.x, y1: startCoords.y, + x2: curCoords.x, y2: curCoords.y + }); + lastTouchMoveTime = Date.now(); + } + } else if (supportsRawTouch && isSwiping) { + // Throttle touch_move to every 16ms + const now = Date.now(); + if (now - lastTouchMoveTime >= 16) { + const coords = mapCoordinates(canvasX, canvasY); + sendAction({ action: 'touch_move', x: coords.x, y: coords.y }); + lastTouchMoveTime = now; + } + } + } + + // Helper: handle pointer end (mouseup / touchend) + function handlePointerEnd(canvasX, canvasY) { + if (!isMouseDown || scriptRunning) return; + + if (supportsRawTouch) { + if (isSwiping) { + // Send final touch_move to ensure we reach the exact release point + const coords = mapCoordinates(canvasX, canvasY); + sendAction({ action: 'touch_move', x: coords.x, y: coords.y }); + sendAction({ action: 'touch_up' }); + } else if (pendingTouch) { + // Didn't move far enough — it's a tap + const coords = mapCoordinates(canvasX, canvasY); + sendAction({ action: 'tap', x: coords.x, y: coords.y }); + } + } else { + // Fallback: batch swipe (for adb) + const dx = canvasX - startX; + const dy = canvasY - startY; + const distance = Math.sqrt(dx * dx + dy * dy); + + if (distance < 10) { + const coords = mapCoordinates(canvasX, canvasY); + sendAction({ action: 'tap', x: coords.x, y: coords.y }); + } else { + const start = mapCoordinates(startX, startY); + const end = mapCoordinates(canvasX, canvasY); + sendAction({ + action: 'swipe', + x1: start.x, y1: start.y, + x2: end.x, y2: end.y, + duration: 300 + }); + } + } + + resetTouchState(); + } + + // Mouse event handlers + canvas.addEventListener('mousedown', (e) => { + const rect = canvas.getBoundingClientRect(); + handlePointerStart(e.clientX - rect.left, e.clientY - rect.top); }); canvas.addEventListener('mousemove', (e) => { - if (!isMouseDown || scriptRunning) return; const rect = canvas.getBoundingClientRect(); - lastX = e.clientX - rect.left; - lastY = e.clientY - rect.top; + handlePointerMove(e.clientX - rect.left, e.clientY - rect.top); }); canvas.addEventListener('mouseup', (e) => { - if (!isMouseDown || scriptRunning) return; - isMouseDown = false; - const rect = canvas.getBoundingClientRect(); - const endX = e.clientX - rect.left; - const endY = e.clientY - rect.top; - - const dx = endX - startX; - const dy = endY - startY; - const distance = Math.sqrt(dx * dx + dy * dy); - - if (distance < 10) { - // Tap - const coords = mapCoordinates(endX, endY); - sendAction({ action: 'tap', x: coords.x, y: coords.y }); - } else { - // Swipe - const start = mapCoordinates(startX, startY); - const end = mapCoordinates(endX, endY); - sendAction({ - action: 'swipe', - x1: start.x, y1: start.y, - x2: end.x, y2: end.y, - duration: 300 - }); - } + handlePointerEnd(e.clientX - rect.left, e.clientY - rect.top); }); canvas.addEventListener('mouseleave', () => { - isMouseDown = false; + if (isSwiping && supportsRawTouch) { + // If swiping, send touch_up to avoid stuck gesture + sendAction({ action: 'touch_up' }); + } + resetTouchState(); }); // Touch events for mobile canvas.addEventListener('touchstart', (e) => { - if (scriptRunning) return; e.preventDefault(); const touch = e.touches[0]; const rect = canvas.getBoundingClientRect(); - startX = touch.clientX - rect.left; - startY = touch.clientY - rect.top; - lastX = startX; - lastY = startY; - isMouseDown = true; + handlePointerStart(touch.clientX - rect.left, touch.clientY - rect.top); }); canvas.addEventListener('touchmove', (e) => { - if (!isMouseDown || scriptRunning) return; e.preventDefault(); const touch = e.touches[0]; const rect = canvas.getBoundingClientRect(); - lastX = touch.clientX - rect.left; - lastY = touch.clientY - rect.top; + handlePointerMove(touch.clientX - rect.left, touch.clientY - rect.top); }); canvas.addEventListener('touchend', (e) => { - if (!isMouseDown || scriptRunning) return; e.preventDefault(); - isMouseDown = false; - - const dx = lastX - startX; - const dy = lastY - startY; - const distance = Math.sqrt(dx * dx + dy * dy); - - if (distance < 10) { - const coords = mapCoordinates(lastX, lastY); - sendAction({ action: 'tap', x: coords.x, y: coords.y }); - } else { - const start = mapCoordinates(startX, startY); - const end = mapCoordinates(lastX, lastY); - sendAction({ - action: 'swipe', - x1: start.x, y1: start.y, - x2: end.x, y2: end.y, - duration: 300 - }); - } + handlePointerEnd(lastX, lastY); }); // Quality slider @@ -1021,4 +1137,5 @@ init(); - + + \ No newline at end of file diff --git a/module/webui/viewport.py b/module/webui/viewport.py index 14fa068dd..30dcd3f3b 100644 --- a/module/webui/viewport.py +++ b/module/webui/viewport.py @@ -179,12 +179,16 @@ class DeviceConnection: self._connected = False return None - def screenshot_jpeg(self, quality: int = 60, scale: float = 1.0) -> Optional[bytes]: + def screenshot_encode(self, quality: int = 30, scale: float = 1.0, skip_unchanged: bool = False) -> Optional[bytes]: """Get screenshot as JPEG bytes. Args: quality: JPEG quality (1-100) scale: Resolution scale (0.25-1.0), e.g. 0.5 = half resolution + skip_unchanged: If True, skip encoding when frame content is unchanged + + Returns: + bytes: Encoded frame data, or None if screenshot failed or frame unchanged. """ import time t0 = time.perf_counter() @@ -207,15 +211,25 @@ class DeviceConnection: new_h = int(h * scale) img = cv2.resize(img, (new_w, new_h), interpolation=cv2.INTER_LINEAR) + # Frame-skip detection: only check when idle (skip_unchanged=True) + if skip_unchanged: + if hasattr(self, '_last_frame') and self._last_frame is not None: + if self._last_frame.shape == img.shape: + diff = cv2.absdiff(img, self._last_frame) + if np.mean(diff) < 1.0: + # Frame unchanged, skip + return None + self._last_frame = img + # Convert BGR to RGB for correct colors in browser img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) t2 = time.perf_counter() # Encode to JPEG - _, jpeg = cv2.imencode('.jpg', img, [cv2.IMWRITE_JPEG_QUALITY, quality]) + _, encoded = cv2.imencode('.jpg', img, [cv2.IMWRITE_JPEG_QUALITY, quality]) t3 = time.perf_counter() - result = jpeg.tobytes() + result = encoded.tobytes() t4 = time.perf_counter() # Log timing every 100 frames @@ -224,7 +238,7 @@ class DeviceConnection: self._total_times = [0, 0, 0, 0] self._frame_count += 1 self._total_times[0] += t1 - t0 # screenshot - self._total_times[1] += t2 - t1 # resize + self._total_times[1] += t2 - t1 # resize + diff self._total_times[2] += t3 - t2 # imencode self._total_times[3] += t4 - t3 # tobytes @@ -243,7 +257,7 @@ class DeviceConnection: return result except Exception as e: if self._error_count == 0: - logger.info(f'[Viewport] JPEG encode error: {e}') + logger.info(f'[Viewport] Encode error: {e}') return None def touch(self, x: int, y: int): @@ -259,7 +273,7 @@ class DeviceConnection: else: # Fallback to adb click self._device.click_adb(x, y) - logger.info(f'[Viewport] Touch ({x}, {y})') + # logger.info(f'[Viewport] Touch ({x}, {y})') except Exception as e: logger.debug(f'[Viewport] Touch error: {e}') @@ -284,6 +298,82 @@ class DeviceConnection: except Exception as e: logger.debug(f'[Viewport] Swipe error: {e}') + @property + def supports_raw_touch(self) -> bool: + """Whether the control method supports raw touch_down/move/up primitives.""" + return self._control_method in ('minitouch', 'MaaTouch', 'nemu_ipc', 'scrcpy') + + def _minitouch_send_no_delay(self): + """Send minitouch commands without the DEFAULT_DELAY sleep. + + The normal builder.send() calls minitouch_send() which sleeps 50ms+ after + each send. For real-time touch streaming this delay is unacceptable as it + causes the game to register a 'down' as a press/click before the 'move' arrives. + """ + builder = self._device.minitouch_builder + content = builder.to_minitouch() + byte_content = content.encode('utf-8') + self._device._minitouch_client.sendall(byte_content) + self._device._minitouch_client.recv(0) + builder.clear() + + def swipe_start(self, x_down: int, y_down: int, x_move: int, y_move: int): + """Atomically send touch down + first move (starts a swipe without click gap).""" + if not self._connected or self._device is None: + return + try: + if self._control_method in ('minitouch', 'MaaTouch'): + builder = self._device.minitouch_builder + builder.down(x_down, y_down).commit() + builder.move(x_move, y_move).commit() + self._minitouch_send_no_delay() + elif self._control_method == 'nemu_ipc': + self._device.nemu_ipc.down(x_down, y_down) + self._device.nemu_ipc.down(x_move, y_move) + elif self._control_method == 'scrcpy': + from module.device.method.scrcpy import const + self._device.scrcpy_ensure_running() + self._device._scrcpy_control.touch(x_down, y_down, const.ACTION_DOWN) + self._device._scrcpy_control.touch(x_move, y_move, const.ACTION_MOVE) + except Exception as e: + logger.debug(f'[Viewport] Swipe start error: {e}') + + def touch_move(self, x: int, y: int): + """Send touch move event (finger drag).""" + if not self._connected or self._device is None: + return + try: + if self._control_method in ('minitouch', 'MaaTouch'): + builder = self._device.minitouch_builder + builder.move(x, y).commit() + self._minitouch_send_no_delay() + elif self._control_method == 'nemu_ipc': + # nemu_ipc uses down() for move as well + self._device.nemu_ipc.down(x, y) + elif self._control_method == 'scrcpy': + from module.device.method.scrcpy import const + self._device._scrcpy_control.touch(x, y, const.ACTION_MOVE) + except Exception as e: + logger.debug(f'[Viewport] Touch move error: {e}') + + def touch_up(self): + """Send touch up event (finger release).""" + if not self._connected or self._device is None: + return + try: + if self._control_method in ('minitouch', 'MaaTouch'): + builder = self._device.minitouch_builder + builder.up().commit() + self._minitouch_send_no_delay() + elif self._control_method == 'nemu_ipc': + self._device.nemu_ipc.up() + elif self._control_method == 'scrcpy': + from module.device.method.scrcpy import const + # scrcpy needs coordinates for up, use (0,0) as placeholder + self._device._scrcpy_control.touch(0, 0, const.ACTION_UP) + except Exception as e: + logger.debug(f'[Viewport] Touch up error: {e}') + def disconnect(self): self._connected = False if self._device is not None: @@ -668,7 +758,7 @@ async def websocket_endpoint(websocket: WebSocket): manager.add_client(instance_name) # Track client connection try: - quality = 60 + quality = 30 scale = 0.5 # Resolution scale (1.0 = 720p, 0.5 = 360p, etc.) - default 360p for better performance target_fps = 30 # Default 30 FPS for smooth streaming is_paused = False # Pause state for visibility-based streaming @@ -693,10 +783,12 @@ async def websocket_endpoint(websocket: WebSocket): 'fps': target_fps, 'screenshot_method': conn.screenshot_method, 'control_method': conn.control_method, - 'client_count': manager.get_client_count(instance_name) + 'client_count': manager.get_client_count(instance_name), + 'supports_raw_touch': conn.supports_raw_touch }) last_status_time = time.monotonic() + last_interaction_time = time.monotonic() # Track last touch/swipe for idle frame-skip loop = asyncio.get_event_loop() while True: @@ -711,6 +803,7 @@ async def websocket_endpoint(websocket: WebSocket): if action == 'tap': if not manager.is_script_running(instance_name): loop.run_in_executor(executor, conn.touch, int(data['x']), int(data['y'])) + last_interaction_time = time.monotonic() elif action == 'swipe': if not manager.is_script_running(instance_name): loop.run_in_executor( @@ -719,12 +812,31 @@ async def websocket_endpoint(websocket: WebSocket): int(data['x2']), int(data['y2']), int(data.get('duration', 300)) ) + last_interaction_time = time.monotonic() + elif action == 'swipe_start': + if not manager.is_script_running(instance_name): + loop.run_in_executor( + executor, conn.swipe_start, + int(data['x1']), int(data['y1']), + int(data['x2']), int(data['y2']) + ) + last_interaction_time = time.monotonic() + elif action == 'touch_move': + if not manager.is_script_running(instance_name): + loop.run_in_executor(executor, conn.touch_move, int(data['x']), int(data['y'])) + last_interaction_time = time.monotonic() + elif action == 'touch_up': + if not manager.is_script_running(instance_name): + loop.run_in_executor(executor, conn.touch_up) + last_interaction_time = time.monotonic() elif action == 'set_quality': - quality = max(10, min(95, int(data['quality']))) + quality = max(10, min(99, int(data['quality']))) elif action == 'set_fps': target_fps = max(1, min(60, int(data['fps']))) elif action == 'set_scale': scale = max(0.25, min(1.0, float(data['scale']))) + elif action == 'resume_idle': + last_interaction_time = time.monotonic() elif action == 'pause': is_paused = data.get('paused', False) logger.info(f'[Viewport] Stream {"paused" if is_paused else "resumed"} for {instance_name}') @@ -746,7 +858,8 @@ async def websocket_endpoint(websocket: WebSocket): 'fps': target_fps, 'screenshot_method': conn.screenshot_method, 'control_method': conn.control_method, - 'client_count': manager.get_client_count(instance_name) + 'client_count': manager.get_client_count(instance_name), + 'supports_raw_touch': conn.supports_raw_touch }) else: await websocket.send_json({'type': 'error', 'message': 'Reconnect failed'}) @@ -775,7 +888,8 @@ async def websocket_endpoint(websocket: WebSocket): 'fps': target_fps, 'screenshot_method': conn.screenshot_method, 'control_method': conn.control_method, - 'client_count': manager.get_client_count(instance_name) + 'client_count': manager.get_client_count(instance_name), + 'supports_raw_touch': conn.supports_raw_touch }) else: await websocket.send_json({'type': 'error', 'message': 'Device disconnected'}) @@ -800,9 +914,17 @@ async def websocket_endpoint(websocket: WebSocket): # Capture and send frame t_cap_start = time.monotonic() - jpeg_data = await loop.run_in_executor( - executor, lambda: conn.screenshot_jpeg(quality, scale) - ) + idle_seconds = t_cap_start - last_interaction_time + skip_unchanged = idle_seconds >= 5.0 + is_idle = idle_seconds >= 300.0 + + # When idle for 300s, skip capturing entirely (save CPU) + if is_idle: + jpeg_data = None + else: + jpeg_data = await loop.run_in_executor( + executor, lambda: conn.screenshot_encode(quality, scale, skip_unchanged) + ) t_cap_end = time.monotonic() if jpeg_data: @@ -816,17 +938,6 @@ async def websocket_endpoint(websocket: WebSocket): stats_total_latency += frame_latency stats_total_bytes += len(jpeg_data) - # Calculate stats every second - stats_elapsed = time.monotonic() - stats_start_time - if stats_elapsed >= 1.0: - current_latency_ms = stats_total_latency / max(1, stats_frame_count) - current_bandwidth_kbps = (stats_total_bytes * 8) / stats_elapsed / 1000 # kbps - # Reset stats - stats_frame_count = 0 - stats_total_latency = 0.0 - stats_total_bytes = 0 - stats_start_time = time.monotonic() - # Track WebSocket timing if not hasattr(websocket, '_ws_frame_count'): websocket._ws_frame_count = 0 @@ -839,13 +950,24 @@ async def websocket_endpoint(websocket: WebSocket): if websocket._ws_frame_count >= 100: avg_cap = websocket._ws_cap_time / websocket._ws_frame_count * 1000 avg_send = websocket._ws_send_time / websocket._ws_frame_count * 1000 - logger.info( - f'[Viewport] WS Timing (avg ms): capture={avg_cap:.1f}, send={avg_send:.1f}' - ) + # logger.info( + # f'[Viewport] WS Timing (avg ms): capture={avg_cap:.1f}, send={avg_send:.1f}' + # ) websocket._ws_frame_count = 0 websocket._ws_cap_time = 0 websocket._ws_send_time = 0 + # Calculate stats every second (outside if jpeg_data so stats update during skips) + stats_elapsed = time.monotonic() - stats_start_time + if stats_elapsed >= 1.0: + current_latency_ms = stats_total_latency / max(1, stats_frame_count) + current_bandwidth_kbps = (stats_total_bytes * 8) / stats_elapsed / 1000 # kbps + # Reset stats + stats_frame_count = 0 + stats_total_latency = 0.0 + stats_total_bytes = 0 + stats_start_time = time.monotonic() + # Periodic status update now = time.monotonic() if now - last_status_time >= 1.0: # Update every second for stats @@ -860,7 +982,8 @@ async def websocket_endpoint(websocket: WebSocket): 'control_method': conn.control_method, 'latency_ms': round(current_latency_ms, 1), 'bandwidth_kbps': round(current_bandwidth_kbps, 0), - 'client_count': manager.get_client_count(instance_name) + 'client_count': manager.get_client_count(instance_name), + 'idle': is_idle }) # Frame rate limiting