cc/web-bak
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

style: web-bak页面设计优化，采用浅色设计语言

Browse Source 
- index.css: 添加CSS自定义属性设计系统色彩令牌
- App.css: 所有组件样式更新为CSS自定义属性
- App.tsx: 主题配置更新，通知位置改为右上角
- LoginPage.tsx: 重新设计登录页面
- RemoteControlApp.tsx: 移除所有emoji，替换硬编码颜色
- AuthGuard.tsx: 移除emoji，替换渐变背景
- InstallPage.tsx: 移除emoji，替换硬编码颜色
- DeviceFilter.tsx: 替换硬编码颜色
- DeviceInfoCard.tsx: 替换硬编码颜色
- GalleryView.tsx: 移除emoji，替换硬编码颜色
- ScreenReader.tsx: 移除所有emoji，替换注释为英文
This commit is contained in:
wdvipa
2026-02-15 15:28:48 +08:00
parent f91c6dc2eb
commit 115b15c0fc
17 changed files with 833 additions and 3441 deletions
Download Patch File Download Diff File Expand all files Collapse all files

246
src/components/Device/ScreenReader.tsx
View File

@@ -137,19 +137,19 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
const autoRefreshIntervalRef = useRef<number | null>(null)
const canvasRef = useRef<HTMLCanvasElement>(null)
// 🆕 添加拖拽状态管理
// Drag state management
const [isDragging, setIsDragging] = useState(false)
const [dragStart, setDragStart] = useState<{x: number, y: number} | null>(null)
// 🆕 添加长按处理
// Long press handling
const [isLongPressTriggered, setIsLongPressTriggered] = useState(false)
const longPressTimerRef = useRef<number | null>(null)
// 🆕 确认坐标提取相关状态
// Confirm coords extraction state
const [isExtractingConfirmCoords, setIsExtractingConfirmCoords] = useState(false)
// const [extractedCoords, setExtractedCoords] = useState<{ x: number; y: number } | null>(null)
// 🆕 虚拟按键相关状态
// Virtual keyboard state
const [virtualKeyboard, setVirtualKeyboard] = useState<{
visible: boolean
keys: Array<{
@@ -165,11 +165,11 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
keys: []
})
// ✨ 增强分析结果状态
// Enhanced analysis result state
// const [enhancedAnalysisResult, setEnhancedAnalysisResult] = useState<any>(null)
// const [deviceCharacteristics, setDeviceCharacteristics] = useState<any>(null)
// 🆕 生成虚拟按键布局（基于屏幕阅读器显示尺寸）
// Generate virtual keyboard layout based on screen reader display size
const generateVirtualKeyboard = useCallback((screenWidth: number, screenHeight: number) => {
// 键盘高度：屏幕高度的25%（确保4行按键都能显示）
const keyboardHeight = screenHeight * 0.25
@@ -240,7 +240,7 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
height: keyHeight
})
console.log('⌨️ 虚拟键盘布局计算:', {
console.log('[VirtualKeyboard] Layout calculated:', {
screenSize: { width: screenWidth, height: screenHeight },
keyboardSize: { width: keyboardWidth, height: keyboardHeight },
keySize: { width: keyWidth, height: keyHeight },
@@ -258,31 +258,31 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
}
}, [])
// ✨ 增强版UI结构请求函数 - 默认启用所有增强功能
// Enhanced UI hierarchy request - all enhanced features enabled by default
const requestUIHierarchy = useCallback((enhanced: boolean = true, includeDeviceInfo: boolean = true) => {
if (!webSocket || !deviceId) return
setLoading(true)
// ✨ 默认启用增强UI分析和设备信息
// Enable enhanced UI analysis and device info by default
webSocket.emit('client_event', {
type: 'GET_UI_HIERARCHY',
data: {
deviceId,
requestId: `ui_hierarchy_${Date.now()}`,
includeInvisible: true, // ✨ 增强：包含不可见元素
includeNonInteractive: true, // ✨ 增强：包含不可交互元素
includeInvisible: true, // Enhanced: include invisible elements
includeNonInteractive: true, // Enhanced: include non-interactive elements
includeTextElements: true, // 包含文本元素
includeImageElements: true, // 包含图像元素
includeContainers: true, // 包含容器元素
maxDepth: 25, // ✨ 增强：使用更大扫描深度
maxDepth: 25, // Enhanced: use larger scan depth
minSize: 1, // 最小元素尺寸
enhanced: enhanced, // ✨ 默认启用增强功能
includeDeviceInfo: includeDeviceInfo // ✨ 默认包含设备信息
enhanced: enhanced, // Enable enhanced features by default
includeDeviceInfo: includeDeviceInfo // Include device info by default
}
})
//console.log('🚀 请求增强UI分析，设备ID:', deviceId, '增强模式:', enhanced, '包含设备信息:', includeDeviceInfo)
//console.log('[ScreenReader] Requesting enhanced UI analysis, deviceId:', deviceId, 'enhanced:', enhanced, 'includeDeviceInfo:', includeDeviceInfo)
}, [webSocket, deviceId])
// 设备特征信息现在集成在UI层次结构请求中，无需单独请求
@@ -317,16 +317,16 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
if (!webSocket || !deviceId) return
const handleUIHierarchyResponse = (data: any) => {
//console.log('🔍 ScreenReader收到UI层次结构响应:', data)
//console.log('[ScreenReader] Received UI hierarchy response:', data)
setLoading(false)
if (data.success && data.deviceId === deviceId) {
setUiHierarchy(data.hierarchy)
// 🆕 UI层次结构更新时清除选中元素，避免显示过时信息
// Clear selected element when UI hierarchy updates to avoid stale info
setSelectedElement(null)
// ✅ 处理增强分析结果
// Process enhanced analysis result
if (data.enhanced) {
/*console.log('🚀 收到增强UI分析结果:', {
/*console.log('[ScreenReader] Received enhanced UI analysis result:', {
enhanced: data.enhanced,
keyboardElements: data.hierarchy?.keyboardElements?.length || 0,
digitButtons: data.hierarchy?.digitButtons?.length || 0,
@@ -334,7 +334,7 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
inputMethodWindows: data.hierarchy?.inputMethodWindows?.length || 0
})*/
// ✨ 保存增强分析结果
// Save enhanced analysis result
// setEnhancedAnalysisResult({
// keyboardElements: data.hierarchy?.keyboardElements || [],
// digitButtons: data.hierarchy?.digitButtons || [],
@@ -345,29 +345,29 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
// 显示键盘检测结果
if (data.hierarchy?.keyboardConfidence > 0.5) {
console.log('✅ 高置信度虚拟键盘检测成功，置信度:', data.hierarchy.keyboardConfidence)
console.log('[ScreenReader] High confidence virtual keyboard detected, confidence:', data.hierarchy.keyboardConfidence)
} else {
console.log('⚠️ 虚拟键盘检测置信度较低:', data.hierarchy.keyboardConfidence)
console.log('[ScreenReader] Low virtual keyboard detection confidence:', data.hierarchy.keyboardConfidence)
}
}
// ✅ 处理设备特征信息
// Process device characteristics
if (data.deviceCharacteristics) {
/*console.log('🔍 收到设备特征信息:', {
/*console.log('[ScreenReader] Received device characteristics:', {
romType: data.deviceCharacteristics.romFeatures?.romType,
inputMethodType: data.deviceCharacteristics.inputMethodInfo?.inputMethodType,
primaryStrategy: data.deviceCharacteristics.keyboardDetectionStrategy?.primaryStrategy,
deviceSpecificTips: data.deviceCharacteristics.keyboardDetectionStrategy?.deviceSpecificTips
})*/
// ✨ 保存设备特征信息
// Save device characteristics
// setDeviceCharacteristics({
// ...data.deviceCharacteristics,
// timestamp: Date.now()
// })
}
/*console.log('✅ UI层次结构数据已设置:', {
/*console.log('[ScreenReader] UI hierarchy data set:', {
totalElements: data.hierarchy?.totalElements,
clickableElements: data.hierarchy?.clickableElements,
screenSize: `${data.hierarchy?.screenWidth}x${data.hierarchy?.screenHeight}`,
@@ -387,19 +387,19 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
}
if (data.hierarchy?.root) {
countElements(data.hierarchy.root)
//console.log('📊 UI元素类型统计:', elementTypes)
//console.log('[ScreenReader] UI element type stats:', elementTypes)
}
} else {
console.error('❌ 获取UI层次结构失败:', data.error || data.message)
console.error('[ScreenReader] Failed to get UI hierarchy:', data.error || data.message)
}
}
// 设备特征信息现在集成在UI层次结构响应中，无需单独处理
// 🆕 监听提取确认坐标的事件
// Listen for confirm coords extraction event
const handleStartExtractConfirmCoords = (data: any) => {
if (data.deviceId === deviceId) {
console.log('🎯 开始提取确认坐标模式:', deviceId)
console.log('[ScreenReader] Start extracting confirm coords mode:', deviceId)
setIsExtractingConfirmCoords(true)
}
}
@@ -413,28 +413,28 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
}
}, [webSocket, deviceId])
// 🆕 监听虚拟按键显示状态变化
// Listen for virtual keyboard visibility changes
useEffect(() => {
if (screenReader.showVirtualKeyboard && uiHierarchy) {
const keyboard = generateVirtualKeyboard(uiHierarchy.screenWidth, uiHierarchy.screenHeight)
setVirtualKeyboard(keyboard)
console.log('⌨️ 虚拟按键已生成:', keyboard.keys.length, '个按键')
console.log('[VirtualKeyboard] Virtual keys generated:', keyboard.keys.length, 'keys')
} else {
setVirtualKeyboard({ visible: false, keys: [] })
console.log('⌨️ 虚拟按键已隐藏')
console.log('[VirtualKeyboard] Virtual keys hidden')
}
}, [screenReader.showVirtualKeyboard, uiHierarchy, generateVirtualKeyboard])
// 初始加载
useEffect(() => {
if (screenReader.enabled && webSocket && deviceId) {
//console.log('🔄 ScreenReader启用，开始自动刷新')
//console.log('[ScreenReader] Enabled, starting auto refresh')
// 延迟一点时间确保连接稳定
setTimeout(() => {
startAutoRefresh()
}, 500)
} else {
console.log('🛑 ScreenReader未启用或连接缺失，停止自动刷新')
console.log('[ScreenReader] Not enabled or connection missing, stopping auto refresh')
stopAutoRefresh()
}
@@ -447,7 +447,7 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
// 在画布上绘制UI元素边界
const drawElementBounds = useCallback(() => {
if (!canvasRef.current || !uiHierarchy) {
console.log('🔍 ScreenReader: 画布或UI层次结构缺失', { canvas: !!canvasRef.current, uiHierarchy: !!uiHierarchy })
console.log('[ScreenReader] Canvas or UI hierarchy missing', { canvas: !!canvasRef.current, uiHierarchy: !!uiHierarchy })
return
}
@@ -462,7 +462,7 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
// const containerRect = container.getBoundingClientRect() // 暂时未使用
const dpr = window.devicePixelRatio || 1
// 🔧 修复：使用容器的clientWidth和clientHeight，排除边框和滚动条
// Fix: use container clientWidth and clientHeight, excluding borders and scrollbars
const displayWidth = container.clientWidth
const displayHeight = container.clientHeight
@@ -478,7 +478,7 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
ctx.imageSmoothingEnabled = true
ctx.imageSmoothingQuality = 'high'
/*console.log('🎨 ScreenReader: 开始绘制UI边界图', {
/*console.log('[ScreenReader] Start drawing UI bounds', {
screenSize: `${uiHierarchy.screenWidth}x${uiHierarchy.screenHeight}`,
canvasSize: `${canvas.width}x${canvas.height}`,
displaySize: `${displayWidth}x${displayHeight}`,
@@ -497,13 +497,13 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
// 使用较小的缩放比例保持宽高比，避免过大显示
const scale = Math.min(scaleX, scaleY)
// 🔧 计算实际显示区域和居中偏移
// Calculate actual display area and center offset
// const actualDisplayWidth = uiHierarchy.screenWidth * scale // 暂时未使用
// const actualDisplayHeight = uiHierarchy.screenHeight * scale // 暂时未使用
// const offsetX = (displayWidth - actualDisplayWidth) / 2 // 暂时未使用
// const offsetY = (displayHeight - actualDisplayHeight) / 2 // 暂时未使用
/*console.log('🔧 ScreenReader缩放调试:', {
/*console.log('[ScreenReader] Scale debug:', {
container: { width: displayWidth, height: displayHeight },
device: { width: uiHierarchy.screenWidth, height: uiHierarchy.screenHeight },
scaleX, scaleY,
@@ -519,7 +519,7 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
const drawElement = (element: UIElement, depth: number = 0) => {
const { bounds } = element
// 🔧 关键修复：使用统一缩放但不加偏移，让内容从左上角开始填满容器
// Key fix: use unified scale without offset, fill container from top-left
const x = bounds.left * scale
const y = bounds.top * scale
const width = (bounds.right - bounds.left) * scale
@@ -532,7 +532,7 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
const hasDigitText = element.text && /[0-9]/.test(element.text)
const isZeroButton = element.text === '0' || (element.description && element.description.includes('0')) || (element.resourceId && element.resourceId.toLowerCase().includes('zero'))
// 🔍 专门查找"0"按钮 - 扩大搜索范围
// Search for "0" button - expanded search range
const zeroRelated =
element.text === '0' ||
element.text?.includes('0') ||
@@ -545,7 +545,7 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
element.type.toLowerCase().includes('0')
if (zeroRelated) {
/*console.log(`🎯 可能的"0"按钮 Element ${elementCount}:`, {
/*console.log(`[ScreenReader] Possible "0" button Element ${elementCount}:`, {
type: element.type,
text: element.text,
description: element.description,
@@ -567,7 +567,7 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
}
if (depth === 0 || elementCount <= 50 || element.clickable || element.type.toLowerCase().includes('button') || isInKeypadArea || hasDigitText || isZeroButton || zeroRelated) {
/*console.log(`🎨 Element ${elementCount}:`, {
/*console.log(`[ScreenReader] Element ${elementCount}:`, {
type: element.type,
text: element.text,
description: element.description,
@@ -636,7 +636,7 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
displayText = 'UI'
}
// 🔧 优化字体大小计算 - 适当增大字体，提高可读性
// Optimize font size calculation - increase for readability
const fontSize = Math.max(8, Math.min(width * 0.15, height * 0.35, 24))
// 设置文字样式
@@ -644,7 +644,7 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
ctx.textAlign = 'center'
ctx.textBaseline = 'middle'
// 🔧 优化：实现多行文本绘制
// Optimize: multiline text drawing
drawMultilineText(ctx, displayText, x + width / 2, y + height / 2, width - 4, height - 4, fontSize)
}
}
@@ -667,7 +667,7 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
}
}
// 🆕 多行文本绘制函数
// Multiline text drawing function
function drawMultilineText(
ctx: CanvasRenderingContext2D,
text: string,
@@ -735,7 +735,7 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
lines.forEach((line, index) => {
const lineY = startY + index * lineHeight
// 🔧 修复重影问题：去掉阴影，只绘制清晰的主文字
// Fix ghost text: remove shadow, only draw clear main text
ctx.fillStyle = '#ff1744'
ctx.fillText(line, x, lineY)
})
@@ -773,11 +773,11 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
// return colors[Math.abs(hash) % colors.length]
// }
// 🆕 绘制虚拟按键（点击穿透版本）
// Draw virtual keyboard (click-through version)
function drawVirtualKeyboard(ctx: CanvasRenderingContext2D, scale: number) {
if (!virtualKeyboard.visible || virtualKeyboard.keys.length === 0) return
console.log('⌨️ 开始绘制虚拟按键:', virtualKeyboard.keys.length, '个按键')
console.log('[VirtualKeyboard] Drawing virtual keys:', virtualKeyboard.keys.length, 'keys')
virtualKeyboard.keys.forEach(key => {
// 计算按键在画布上的位置
@@ -786,7 +786,7 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
const width = key.width * scale
const height = key.height * scale
// 🆕 不绘制背景，只绘制边框和文字，实现点击穿透
// Do not draw background, only draw border and text for click-through
// 绘制按键边框（虚线样式，更明显）
ctx.strokeStyle = '#1890ff'
ctx.lineWidth = 3
@@ -837,22 +837,22 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
//
// // 常见应用的图标符号
// const iconMap: { [key: string]: string } = {
// 'youtube': '▶',
// 'settings': '⚙',
// 'photos': '📷',
// 'gmail': '✉',
// 'camera': '📸',
// 'chrome': '🌐',
// 'calendar': '📅',
// 'contacts': '👤',
// 'phone': '📞',
// 'messages': '💬',
// 'maps': '🗺',
// 'drive': '💾',
// 'files': '📁',
// 'clock': '🕐',
// 'youtube': '>',
// 'settings': 'S',
// 'photos': 'P',
// 'gmail': 'M',
// 'camera': 'C',
// 'chrome': 'W',
// 'calendar': 'D',
// 'contacts': 'U',
// 'phone': 'T',
// 'messages': 'X',
// 'maps': 'N',
// 'drive': 'V',
// 'files': 'F',
// 'clock': 'K',
// 'google': 'G',
// 'android': '🤖'
// 'android': 'A'
// }
//
// // 查找匹配的图标
@@ -875,7 +875,7 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
drawElement(uiHierarchy.root)
// 🆕 绘制虚拟按键
// Draw virtual keyboard
if (virtualKeyboard.visible && virtualKeyboard.keys.length > 0) {
drawVirtualKeyboard(ctx, scale)
}
@@ -883,7 +883,7 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
// 恢复上下文
ctx.restore()
/*console.log('🎨 ScreenReader: 绘制完成', {
/*console.log('[ScreenReader] Drawing complete', {
processedElements: elementCount,
totalElements: uiHierarchy.totalElements,
clickableElements: uiHierarchy.clickableElements,
@@ -899,7 +899,7 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
ctx.font = '16px -apple-system, BlinkMacSystemFont, sans-serif'
ctx.textAlign = 'center'
ctx.textBaseline = 'middle'
ctx.fillText('🔍 无UI数据', displayWidth / 2, displayHeight / 2)
ctx.fillText('[No UI Data]', displayWidth / 2, displayHeight / 2)
ctx.font = '14px -apple-system, BlinkMacSystemFont, sans-serif'
ctx.fillText('点击右上角刷新按钮获取UI结构', displayWidth / 2, displayHeight / 2 + 30)
}
@@ -930,7 +930,7 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
drawElementBounds()
}, [drawElementBounds])
// 🔧 修复坐标转换函数（完全匹配绘制逻辑）
// Fix coordinate conversion function (fully match drawing logic)
const convertCanvasToDeviceCoords = useCallback((canvasX: number, canvasY: number, forceDebug: boolean = false) => {
if (!canvasRef.current || !uiHierarchy) return null
@@ -942,7 +942,7 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
const deviceWidth = uiHierarchy.screenWidth
const deviceHeight = uiHierarchy.screenHeight
// 🔧 关键修复：与绘制时完全一致的坐标计算
// Key fix: coordinate calculation fully matching drawing logic
// 绘制时使用的是容器的显示尺寸，不是canvas的物理尺寸
const displayWidth = containerRect.width
const displayHeight = containerRect.height
@@ -952,7 +952,7 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
const scaleY = displayHeight / deviceHeight
const scale = Math.min(scaleX, scaleY)
// 🔧 直接转换坐标，与绘制逻辑完全一致
// Direct coordinate conversion, fully matching drawing logic
// 绘制时：x = bounds.left * scale, y = bounds.top * scale
// 转换时：deviceX = canvasX / scale, deviceY = canvasY / scale
const deviceX = canvasX / scale
@@ -962,9 +962,9 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
const clampedX = Math.max(0, Math.min(deviceWidth, deviceX))
const clampedY = Math.max(0, Math.min(deviceHeight, deviceY))
// 🔧 在调试模式或坐标被调整时显示调试信息
// In debug mode or when coords are adjusted, show debug info
if (forceDebug || Math.abs(deviceX - clampedX) > 1 || Math.abs(deviceY - clampedY) > 1) {
console.log('🔧 坐标转换:', {
console.log('[ScreenReader] Coordinate conversion:', {
canvas: { x: canvasX, y: canvasY },
container: { width: displayWidth, height: displayHeight },
device: { width: deviceWidth, height: deviceHeight },
@@ -977,7 +977,7 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
return { x: clampedX, y: clampedY }
}, [uiHierarchy])
// 🆕 显示触摸指示器
// Show touch indicator
const showTouchIndicator = (x: number, y: number) => {
const indicator = document.createElement('div')
indicator.style.position = 'absolute'
@@ -1004,7 +1004,7 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
}
}
// 🆕 显示滑动指示器
// Show swipe indicator
const showSwipeIndicator = (startX: number, startY: number, endX: number, endY: number) => {
const container = canvasRef.current?.parentElement
if (!container) return
@@ -1074,7 +1074,7 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
}
}
// 🆕 查找点击位置的元素
// Find element at click position
const findElementAtPoint = useCallback((deviceX: number, deviceY: number): UIElement | null => {
if (!uiHierarchy) return null
@@ -1100,7 +1100,7 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
return findElement(uiHierarchy.root)
}, [uiHierarchy])
// 🆕 查找点击的虚拟按键
// Find virtual key at click position
const findVirtualKeyAtPoint = useCallback((deviceX: number, deviceY: number) => {
if (!virtualKeyboard.visible || virtualKeyboard.keys.length === 0) return null
@@ -1110,18 +1110,18 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
)
}, [virtualKeyboard])
// 🆕 执行点击操作
// Perform click action
const performClick = useCallback((canvasX: number, canvasY: number) => {
if (!webSocket || !deviceId) return
// 🔧 在提取模式时启用调试信息
// In extract mode, enable debug info
const deviceCoords = convertCanvasToDeviceCoords(canvasX, canvasY, isExtractingConfirmCoords)
if (!deviceCoords) {
console.warn('🖱️ 点击在设备屏幕区域外')
console.warn('[ScreenReader] Click outside device screen area')
return
}
console.log('🖱️ 屏幕阅读器点击:', {
console.log('[ScreenReader] Screen reader click:', {
canvas: { x: canvasX, y: canvasY },
device: {
original: { x: deviceCoords.x, y: deviceCoords.y },
@@ -1131,9 +1131,9 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
extractingMode: isExtractingConfirmCoords
})
// 🆕 如果处于确认坐标提取模式
// If in confirm coords extraction mode
if (isExtractingConfirmCoords) {
// 🔧 修复：使用与正常点击相同的坐标精度
// Fix: use same coordinate precision as normal click
const coords = { x: deviceCoords.x, y: deviceCoords.y }
// setExtractedCoords(coords)
@@ -1147,7 +1147,7 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
// 显示特殊指示器
showTouchIndicator(canvasX, canvasY)
console.log('🎯 确认坐标已提取:', coords)
console.log('[ScreenReader] Confirm coords extracted:', coords)
// 自动退出提取模式
setTimeout(() => {
@@ -1157,10 +1157,10 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
return // 提取模式下不执行实际点击
}
// 🆕 检查是否点击了虚拟按键（优先检测）
// Check if virtual key was clicked (priority detection)
const clickedVirtualKey = findVirtualKeyAtPoint(deviceCoords.x, deviceCoords.y)
if (clickedVirtualKey) {
console.log('⌨️ 点击虚拟按键:', clickedVirtualKey.text)
console.log('[VirtualKeyboard] Clicked virtual key:', clickedVirtualKey.text)
// 发送按键事件到设备
if (clickedVirtualKey.id === 'key_delete') {
@@ -1184,11 +1184,11 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
}
}
// 🆕 继续发送点击事件到设备（不return，让点击事件继续执行）
console.log('⌨️ 虚拟按键点击，同时发送点击事件到设备')
// Continue sending click event to device (don't return, let click event continue)
console.log('[VirtualKeyboard] Virtual key click, also sending click event to device')
}
// 🆕 如果虚拟键盘可见但点击的不是按键，检查是否在键盘区域内
// If virtual keyboard visible but click is not on a key, check if in keyboard area
if (virtualKeyboard.visible && virtualKeyboard.keys.length > 0) {
// 检查点击位置是否在键盘的总体区域内（包括按键间隙）
const keyboardArea = {
@@ -1205,7 +1205,7 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
// 如果在键盘区域内但不在具体按键上，则穿透到下方元素
if (isInKeyboardArea) {
console.log('⌨️ 点击键盘区域空隙，穿透到下方元素')
console.log('[VirtualKeyboard] Click on keyboard gap, passing through to element below')
// 继续执行下方的正常点击逻辑
}
}
@@ -1221,35 +1221,35 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
// 显示触摸指示器
showTouchIndicator(canvasX, canvasY)
// 🔧 优化：更新选中元素用于信息显示，避免重复选中
// Optimize: update selected element for info display, avoid duplicate selection
const clickedElement = findElementAtPoint(deviceCoords.x, deviceCoords.y)
if (clickedElement) {
// 只有当选中的元素发生变化时才更新
if (!selectedElement || selectedElement.id !== clickedElement.id) {
setSelectedElement(clickedElement)
console.log('🎯 选中新元素:', clickedElement.text || clickedElement.type)
console.log('[ScreenReader] Selected new element:', clickedElement.text || clickedElement.type)
}
} else {
// 🆕 点击空白区域时清除选中元素
// Clear selected element when clicking blank area
if (selectedElement) {
setSelectedElement(null)
console.log('🎯 点击空白区域，清除选中元素')
console.log('[ScreenReader] Clicked blank area, cleared selection')
}
}
}, [webSocket, deviceId, convertCanvasToDeviceCoords, findElementAtPoint, findVirtualKeyAtPoint, isExtractingConfirmCoords, selectedElement])
// 🆕 处理长按操作
// Handle long press action
const performLongPress = useCallback((canvasX: number, canvasY: number) => {
if (!webSocket || !deviceId) return
// 🔧 在提取模式时启用调试信息
// In extract mode, enable debug info
const deviceCoords = convertCanvasToDeviceCoords(canvasX, canvasY, isExtractingConfirmCoords)
if (!deviceCoords) {
console.warn('🖱️ 长按在设备屏幕区域外')
console.warn('[ScreenReader] Long press outside device screen area')
return
}
console.log('🖱️ 屏幕阅读器长按:', {
console.log('[ScreenReader] Screen reader long press:', {
canvas: { x: canvasX, y: canvasY },
device: {
original: { x: deviceCoords.x, y: deviceCoords.y },
@@ -1259,7 +1259,7 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
extractingMode: isExtractingConfirmCoords
})
// 🆕 如果处于确认坐标提取模式，长按也算作提取操作
// If in confirm coords extraction mode, long press also counts as extraction
if (isExtractingConfirmCoords) {
const coords = { x: deviceCoords.x, y: deviceCoords.y }
// setExtractedCoords(coords)
@@ -1274,7 +1274,7 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
// 显示特殊指示器
showLongPressIndicator(canvasX, canvasY)
console.log('🎯 长按提取确认坐标:', coords)
console.log('[ScreenReader] Long press extracted confirm coords:', coords)
// 自动退出提取模式
setTimeout(() => {
@@ -1284,10 +1284,10 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
return // 提取模式下不执行实际长按
}
// 🆕 检查是否长按了虚拟按键（优先检测）
// Check if virtual key was long pressed (priority detection)
const longPressedVirtualKey = findVirtualKeyAtPoint(deviceCoords.x, deviceCoords.y)
if (longPressedVirtualKey) {
console.log('⌨️ 长按虚拟按键:', longPressedVirtualKey.text)
console.log('[VirtualKeyboard] Long pressed virtual key:', longPressedVirtualKey.text)
// 虚拟按键长按处理（可以用于特殊功能，比如连续输入）
if (longPressedVirtualKey.id === 'key_delete') {
@@ -1300,11 +1300,11 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
})
}
// 🆕 继续发送长按事件到设备（不return，让长按事件继续执行）
console.log('⌨️ 虚拟按键长按，同时发送长按事件到设备')
// Continue sending long press event to device (don't return, let long press event continue)
console.log('[VirtualKeyboard] Virtual key long press, also sending long press event to device')
}
// 🆕 如果虚拟键盘可见但长按的不是按键，检查是否在键盘区域内
// If virtual keyboard visible but long press is not on a key, check if in keyboard area
if (virtualKeyboard.visible && virtualKeyboard.keys.length > 0) {
// 检查长按位置是否在键盘的总体区域内（包括按键间隙）
const keyboardArea = {
@@ -1321,7 +1321,7 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
// 如果在键盘区域内但不在具体按键上，则穿透到下方元素
if (isInKeyboardArea) {
console.log('⌨️ 长按键盘区域空隙，穿透到下方元素')
console.log('[VirtualKeyboard] Long press on keyboard gap, passing through to element below')
// 继续执行下方的正常长按逻辑
}
}
@@ -1337,24 +1337,24 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
// 显示长按指示器
showLongPressIndicator(canvasX, canvasY)
// 🔧 更新选中元素用于信息显示
// Update selected element for info display
const clickedElement = findElementAtPoint(deviceCoords.x, deviceCoords.y)
if (clickedElement) {
// 只有当选中的元素发生变化时才更新
if (!selectedElement || selectedElement.id !== clickedElement.id) {
setSelectedElement(clickedElement)
console.log('🎯 长按选中新元素:', clickedElement.text || clickedElement.type)
console.log('[ScreenReader] Long press selected new element:', clickedElement.text || clickedElement.type)
}
} else {
// 🆕 长按空白区域时清除选中元素
// Clear selected element when long pressing blank area
if (selectedElement) {
setSelectedElement(null)
console.log('🎯 长按空白区域，清除选中元素')
console.log('[ScreenReader] Long press blank area, cleared selection')
}
}
}, [webSocket, deviceId, convertCanvasToDeviceCoords, findElementAtPoint, findVirtualKeyAtPoint, isExtractingConfirmCoords, selectedElement, uiHierarchy])
// 🆕 鼠标按下处理
// Mouse down handler
const handleMouseDown = useCallback((event: React.MouseEvent) => {
event.preventDefault()
setIsDragging(true)
@@ -1368,7 +1368,7 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
setDragStart({ x: startX, y: startY })
// 🆕 启动长按计时器
// Start long press timer
setIsLongPressTriggered(false)
longPressTimerRef.current = window.setTimeout(() => {
setIsLongPressTriggered(true)
@@ -1376,27 +1376,27 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
}, 500) // 500ms 后触发长按
}, [performLongPress])
// 🆕 鼠标移动处理
// Mouse move handler
const handleMouseMove = useCallback((event: React.MouseEvent) => {
if (!isDragging || !dragStart) return
event.preventDefault()
}, [isDragging, dragStart])
// 🆕 鼠标抬起处理
// Mouse up handler
const handleMouseUp = useCallback((event: React.MouseEvent) => {
if (!isDragging || !dragStart) return
event.preventDefault()
setIsDragging(false)
// 🆕 清理长按计时器
// Clean up long press timer
if (longPressTimerRef.current) {
clearTimeout(longPressTimerRef.current)
longPressTimerRef.current = null
}
// 🆕 如果长按已触发，直接清理状态，不执行点击或滑动
// If long press already triggered, clean up state, don't execute click or swipe
if (isLongPressTriggered) {
setDragStart(null)
setIsLongPressTriggered(false)
@@ -1424,11 +1424,11 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
const endCoords = convertCanvasToDeviceCoords(endX, endY)
if (!startCoords || !endCoords) {
console.warn('🖱️ 滑动坐标转换失败，滑动可能在屏幕区域外')
console.warn('[ScreenReader] Swipe coordinate conversion failed, swipe may be outside screen area')
return
}
console.log('🖱️ 屏幕阅读器滑动:', {
console.log('[ScreenReader] Screen reader swipe:', {
start: { canvas: dragStart, device: startCoords },
end: { canvas: { x: endX, y: endY }, device: endCoords }
})
@@ -1454,12 +1454,12 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
setDragStart(null)
}, [isDragging, dragStart, webSocket, deviceId, convertCanvasToDeviceCoords, performClick, isLongPressTriggered])
// 🆕 鼠标离开处理
// Mouse leave handler
const handleMouseLeave = useCallback(() => {
setIsDragging(false)
setDragStart(null)
// 🆕 清理长按计时器和状态
// Clean up long press timer and state
if (longPressTimerRef.current) {
clearTimeout(longPressTimerRef.current)
longPressTimerRef.current = null
@@ -1467,7 +1467,7 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
setIsLongPressTriggered(false)
}, [])
// 🔧 已移除 handleElementClick 函数，避免重复点击
// Removed handleElementClick to avoid duplicate clicks
// 当UI层次结构或选中元素变化时重新绘制
useEffect(() => {
@@ -1488,7 +1488,7 @@ const ScreenReader: React.FC<ScreenReaderProps> = ({ deviceId, maxHeight }) => {
padding: '20px',
textAlign: 'center'
}}>
<div style={{ fontSize: '48px' }}>📱</div>
<div style={{ fontSize: '48px' }}>[ ]</div>
<div style={{ fontSize: '16px', fontWeight: 'bold' }}>
屏幕阅读器未启用
</div>