diff --git a/.codacy.yaml b/.codacy.yaml new file mode 100644 index 00000000..da4832e8 --- /dev/null +++ b/.codacy.yaml @@ -0,0 +1,25 @@ +--- +# Codacy repository configuration. +# Docs: https://docs.codacy.com/repositories-configure/codacy-configuration-file/ + +engines: + bandit: + enabled: true + exclude_paths: + # Test code legitimately uses `assert` (B101); pytest depends on it. + # Library/non-test code is constrained by CLAUDE.md "no assert outside tests". + - "test/**" + prospector: + enabled: true + exclude_paths: + - "test/**" + +# Drop generated docs / build outputs from analysis entirely. +exclude_paths: + - "docs/build/**" + - ".venv/**" + - "build/**" + - "dist/**" + # One-off GUI smoke scripts — not library code, not pytest targets. + - "test/gui_test/**" + - "test/manual_test/**" diff --git a/README.md b/README.md index 3f84e908..44e40575 100644 --- a/README.md +++ b/README.md @@ -18,15 +18,23 @@ - [Installation](#installation) - [Requirements](#requirements) - [Quick Start](#quick-start) -- [API Reference](#api-reference) - [Mouse Control](#mouse-control) - [Keyboard Control](#keyboard-control) - [Image Recognition](#image-recognition) - - [Screen Operations](#screen-operations) + - [Accessibility Element Finder](#accessibility-element-finder) + - [AI Element Locator (VLM)](#ai-element-locator-vlm) + - [OCR (Text on Screen)](#ocr-text-on-screen) + - [Clipboard](#clipboard) + - [Screenshot](#screenshot) - [Action Recording & Playback](#action-recording--playback) - - [Action Scripting (JSON Executor)](#action-scripting-json-executor) + - [JSON Action Scripting](#json-action-scripting) + - [Scheduler (Interval & Cron)](#scheduler-interval--cron) + - [Global Hotkey Daemon](#global-hotkey-daemon) + - [Event Triggers](#event-triggers) + - [Run History](#run-history) - [Report Generation](#report-generation) - - [Remote Automation (Socket Server)](#remote-automation-socket-server) + - [Remote Automation (Socket / REST)](#remote-automation-socket--rest) + - [Plugin Loader](#plugin-loader) - [Shell Command Execution](#shell-command-execution) - [Screen Recording](#screen-recording) - [Callback Executor](#callback-executor) @@ -46,17 +54,27 @@ - **Mouse Automation** — move, click, press, release, drag, and scroll with precise coordinate control - **Keyboard Automation** — press/release individual keys, type strings, hotkey combinations, key state detection - **Image Recognition** — locate UI elements on screen using OpenCV template matching with configurable threshold +- **Accessibility Element Finder** — query the OS accessibility tree (Windows UIA / macOS AX) to locate buttons, menus, and controls by name/role +- **AI Element Locator (VLM)** — describe a UI element in plain language and let a vision-language model (Anthropic / OpenAI) find its screen coordinates +- **OCR** — extract text from screen regions using Tesseract; wait for, click, or locate rendered text +- **Clipboard** — read/write system clipboard text on Windows, macOS, and Linux - **Screenshot & Screen Recording** — capture full screen or regions as images, record screen to video (AVI/MP4) - **Action Recording & Playback** — record mouse/keyboard events and replay them -- **JSON-Based Action Scripting** — define and execute automation flows using JSON action files +- **JSON-Based Action Scripting** — define and execute automation flows using JSON action files (dry-run + step debug) +- **Scheduler** — run scripts on an interval or cron expression; jobs persist across restarts +- **Global Hotkey Daemon** — bind OS-level hotkeys to action scripts (Windows today; macOS/Linux stubs in place) +- **Event Triggers** — fire scripts when an image appears, a window opens, a pixel changes, or a file is modified +- **Run History** — SQLite-backed run log across scheduler / triggers / hotkeys / REST with auto error-screenshot artifacts - **Report Generation** — export test records as HTML, JSON, or XML reports with success/failure status -- **Remote Automation** — start a TCP socket server to receive and execute automation commands from remote clients +- **Remote Automation** — TCP socket server **and** REST API server to receive automation commands +- **Plugin Loader** — drop `.py` files exposing `AC_*` callables into a directory and register them as executor commands at runtime - **Shell Integration** — execute shell commands within automation workflows with async output capture - **Callback Executor** — trigger automation functions with callback hooks for chaining operations - **Dynamic Package Loading** — extend the executor at runtime by importing external Python packages - **Project & Template Management** — scaffold automation projects with keyword/executor directory structure - **Window Management** — send keyboard/mouse events directly to specific windows (Windows/Linux) -- **GUI Application** — built-in PySide6 graphical interface for interactive automation +- **GUI Application** — built-in PySide6 graphical interface with live language switching (English / 繁體中文 / 简体中文 / 日本語) +- **CLI Runner** — `python -m je_auto_control.cli run|list-jobs|start-server|start-rest` - **Cross-Platform** — unified API across Windows, macOS, and Linux (X11) --- @@ -80,10 +98,23 @@ je_auto_control/ ├── executor/ # JSON action executor engine ├── callback/ # Callback function executor ├── cv2_utils/ # OpenCV screenshot, template matching, video recording + ├── accessibility/ # UIA (Windows) / AX (macOS) element finder + ├── vision/ # VLM-based locator (Anthropic / OpenAI backends) + ├── ocr/ # Tesseract-backed text locator + ├── clipboard/ # Cross-platform clipboard + ├── scheduler/ # Interval + cron scheduler + ├── hotkey/ # Global hotkey daemon + ├── triggers/ # Image/window/pixel/file triggers + ├── run_history/ # SQLite run log + error-screenshot artifacts + ├── rest_api/ # Stdlib HTTP/REST server + ├── plugin_loader/ # Dynamic AC_* plugin discovery ├── socket_server/ # TCP socket server for remote automation ├── shell_process/ # Shell command manager ├── generate_report/ # HTML / JSON / XML report generators ├── test_record/ # Test action recording + ├── script_vars/ # Script variable interpolation + ├── watcher/ # Mouse / pixel / log watchers (Live HUD) + ├── recording_edit/ # Trim, filter, re-scale recorded actions ├── json/ # JSON action file read/write ├── project/ # Project scaffolding & templates ├── package_manager/ # Dynamic package loading @@ -135,6 +166,13 @@ sudo apt-get install cmake libssl-dev | `python-Xlib` | Linux X11 backend (auto-installed on Linux) | | `PySide6` | GUI application (optional, install with `[gui]`) | | `qt-material` | GUI theme (optional, install with `[gui]`) | +| `uiautomation` | Windows accessibility backend (optional, loaded on demand) | +| `pytesseract` + Tesseract | OCR engine (optional, loaded on demand) | +| `anthropic` | VLM locator — Anthropic backend (optional, loaded on demand) | +| `openai` | VLM locator — OpenAI backend (optional, loaded on demand) | + +See [Third_Party_License.md](Third_Party_License.md) for a full list of +third-party components and their licenses. --- @@ -197,6 +235,95 @@ print(f"Found at: ({cx}, {cy})") je_auto_control.locate_and_click("submit_button.png", mouse_keycode="mouse_left") ``` +### Accessibility Element Finder + +Query the OS accessibility tree to locate controls by name, role, or app. +Works on Windows (UIA, via `uiautomation`) and macOS (AX). + +```python +import je_auto_control + +# List all visible buttons in the Calculator app +elements = je_auto_control.list_accessibility_elements(app_name="Calculator") + +# Find a specific element +ok = je_auto_control.find_accessibility_element(name="OK", role="Button") +if ok is not None: + print(ok.bounds, ok.center) + +# Click it directly +je_auto_control.click_accessibility_element(name="OK", app_name="Calculator") +``` + +Raises `AccessibilityNotAvailableError` if no accessibility backend is +installed for the current platform. + +### AI Element Locator (VLM) + +When template matching and accessibility both fail, describe the element +in plain language and let a vision-language model find its coordinates. + +```python +import je_auto_control + +# Uses Anthropic by default if ANTHROPIC_API_KEY is set, else OpenAI. +x, y = je_auto_control.locate_by_description("the green Submit button") + +# Or click it in one shot +je_auto_control.click_by_description( + "the cookie-banner 'Accept all' button", + screen_region=[0, 800, 1920, 1080], # optional crop +) +``` + +Configuration (environment variables only — keys are never persisted or +logged): + +| Variable | Effect | +|---|---| +| `ANTHROPIC_API_KEY` | Enables the Anthropic backend | +| `OPENAI_API_KEY` | Enables the OpenAI backend | +| `AUTOCONTROL_VLM_BACKEND` | `anthropic` or `openai` to force a backend | +| `AUTOCONTROL_VLM_MODEL` | Override the default model (e.g. `claude-opus-4-7`, `gpt-4o-mini`) | + +Raises `VLMNotAvailableError` if neither SDK is installed or no API key +is set. + +### OCR (Text on Screen) + +```python +import je_auto_control as ac + +# Locate all matches of a piece of text +matches = ac.find_text_matches("Submit") + +# Center of the first match, or None +cx, cy = ac.locate_text_center("Submit") + +# Click text in one call +ac.click_text("Submit") + +# Block until text appears (or timeout) +ac.wait_for_text("Loading complete", timeout=15.0) +``` + +If Tesseract is not on `PATH`, point at it explicitly: + +```python +ac.set_tesseract_cmd(r"C:\Program Files\Tesseract-OCR\tesseract.exe") +``` + +### Clipboard + +```python +import je_auto_control as ac +ac.set_clipboard("hello") +text = ac.get_clipboard() +``` + +Backends: Windows (Win32 via `ctypes`), macOS (`pbcopy`/`pbpaste`), +Linux (`xclip` or `xsel`). + ### Screenshot ```python @@ -270,6 +397,10 @@ je_auto_control.execute_action([ | Keyboard | `AC_type_keyboard`, `AC_press_keyboard_key`, `AC_release_keyboard_key`, `AC_write`, `AC_hotkey`, `AC_check_key_is_press` | | Image | `AC_locate_all_image`, `AC_locate_image_center`, `AC_locate_and_click` | | Screen | `AC_screen_size`, `AC_screenshot` | +| Accessibility | `AC_a11y_list`, `AC_a11y_find`, `AC_a11y_click` | +| VLM (AI Locator) | `AC_vlm_locate`, `AC_vlm_click` | +| OCR | `AC_locate_text`, `AC_click_text`, `AC_wait_text` | +| Clipboard | `AC_clipboard_get`, `AC_clipboard_set` | | Record | `AC_record`, `AC_stop_record` | | Report | `AC_generate_html`, `AC_generate_json`, `AC_generate_xml`, `AC_generate_html_report`, `AC_generate_json_report`, `AC_generate_xml_report` | | Project | `AC_create_project` | @@ -277,6 +408,73 @@ je_auto_control.execute_action([ | Process | `AC_execute_process` | | Executor | `AC_execute_action`, `AC_execute_files` | +### Scheduler (Interval & Cron) + +```python +import je_auto_control as ac + +# Interval job — run every 30 seconds +job = ac.default_scheduler.add_job( + script_path="scripts/poll.json", interval_seconds=30, repeat=True, +) + +# Cron job — 09:00 on weekdays (minute hour dom month dow) +cron_job = ac.default_scheduler.add_cron_job( + script_path="scripts/daily.json", cron_expression="0 9 * * 1-5", +) + +ac.default_scheduler.start() +``` + +Both flavours coexist; `job.is_cron` tells them apart. + +### Global Hotkey Daemon + +Bind OS-level hotkeys to action JSON scripts (Windows backend today; +macOS / Linux raise `NotImplementedError` on `start()` with Strategy- +pattern seams in place). + +```python +from je_auto_control import default_hotkey_daemon + +default_hotkey_daemon.bind("ctrl+alt+1", "scripts/greet.json") +default_hotkey_daemon.start() +``` + +### Event Triggers + +Poll-based triggers that fire a script when a condition becomes true: + +```python +from je_auto_control import ( + default_trigger_engine, ImageAppearsTrigger, + WindowAppearsTrigger, PixelColorTrigger, FilePathTrigger, +) + +default_trigger_engine.add(ImageAppearsTrigger( + trigger_id="", script_path="scripts/click_ok.json", + image_path="templates/ok_button.png", threshold=0.85, repeat=True, +)) +default_trigger_engine.start() +``` + +### Run History + +Every run from the scheduler, trigger engine, hotkey daemon, REST API, +and manual GUI replay is recorded to `~/.je_auto_control/history.db`. +Errors automatically attach a screenshot under +`~/.je_auto_control/artifacts/run_{id}_{ms}.png` for post-mortem. + +```python +from je_auto_control import default_history_store + +for run in default_history_store.list_runs(limit=20): + print(run.id, run.source, run.status, run.artifact_path) +``` + +The GUI **Run History** tab exposes filter/refresh/clear and +double-click-to-open on the artifact column. + ### Report Generation ```python @@ -302,19 +500,24 @@ xml_string = je_auto_control.generate_xml() Reports include: function name, parameters, timestamp, and exception info (if any) for each recorded action. HTML reports display successful actions in cyan and failed actions in red. -### Remote Automation (Socket Server) +### Remote Automation (Socket / REST) -Start a TCP server to receive JSON automation commands from remote clients: +Two servers are available — a raw TCP socket and a stdlib HTTP/REST +server. Both default to `127.0.0.1`; binding to `0.0.0.0` is an explicit, +documented opt-in. ```python -import je_auto_control +import je_auto_control as ac -# Start the server (default: localhost:9938) -server = je_auto_control.start_autocontrol_socket_server(host="localhost", port=9938) +# TCP socket server (default: 127.0.0.1:9938) +ac.start_autocontrol_socket_server(host="127.0.0.1", port=9938) -# The server runs in a background thread -# Send JSON action commands via TCP to execute remotely -# Send "quit_server" to shut down +# REST API server (default: 127.0.0.1:9939) +ac.start_rest_api_server(host="127.0.0.1", port=9939) +# Endpoints: +# GET /health liveness probe +# GET /jobs scheduler job list +# POST /execute body: {"actions": [...]} ``` Client example: @@ -339,6 +542,26 @@ print(response) sock.close() ``` +### Plugin Loader + +Drop `.py` files defining top-level `AC_*` callables into a directory, +then register them as executor commands at runtime: + +```python +from je_auto_control import ( + load_plugin_directory, register_plugin_commands, +) + +commands = load_plugin_directory("./my_plugins") +register_plugin_commands(commands) + +# Now usable from any JSON action script: +# [["AC_greet", {"name": "world"}]] +``` + +> **Warning:** Plugin files execute arbitrary Python on load. Only load +> from directories you control. + ### Shell Command Execution ```python @@ -497,6 +720,24 @@ python -m je_auto_control --execute_str '[["AC_screenshot", {"file_path": "test. python -m je_auto_control -c ./my_project ``` +A richer subcommand CLI built on the headless APIs: + +```bash +# Run a script, optionally with variables, and/or a dry-run +python -m je_auto_control.cli run script.json +python -m je_auto_control.cli run script.json --var name=alice --dry-run + +# List scheduler jobs +python -m je_auto_control.cli list-jobs + +# Start the socket or REST server +python -m je_auto_control.cli start-server --port 9938 +python -m je_auto_control.cli start-rest --port 9939 +``` + +`--var name=value` is parsed as JSON when possible (so `count=10` becomes +an int), otherwise treated as a string. + --- ## Platform Support @@ -541,4 +782,6 @@ python -m pytest test/integrated_test/ ## License -[MIT License](LICENSE) © JE-Chen +[MIT License](LICENSE) © JE-Chen. +See [Third_Party_License.md](Third_Party_License.md) for the licenses of +bundled and optional third-party dependencies. diff --git a/README/README_zh-CN.md b/README/README_zh-CN.md index 71f4fc06..7ff22fc5 100644 --- a/README/README_zh-CN.md +++ b/README/README_zh-CN.md @@ -17,15 +17,23 @@ - [安装](#安装) - [系统要求](#系统要求) - [快速开始](#快速开始) -- [API 参考](#api-参考) - [鼠标控制](#鼠标控制) - [键盘控制](#键盘控制) - [图像识别](#图像识别) - - [屏幕操作](#屏幕操作) + - [Accessibility 元件搜索](#accessibility-元件搜索) + - [AI 元件定位（VLM）](#ai-元件定位vlm) + - [OCR 屏幕文字识别](#ocr-屏幕文字识别) + - [剪贴板](#剪贴板) + - [截图](#截图) - [动作录制与回放](#动作录制与回放) - [JSON 脚本执行器](#json-脚本执行器) + - [调度器（Interval & Cron）](#调度器interval--cron) + - [全局热键](#全局热键) + - [事件触发器](#事件触发器) + - [执行历史](#执行历史) - [报告生成](#报告生成) - - [远程自动化（Socket 服务器）](#远程自动化socket-服务器) + - [远程自动化（Socket / REST）](#远程自动化socket--rest) + - [插件加载器](#插件加载器) - [Shell 命令执行](#shell-命令执行) - [屏幕录制](#屏幕录制) - [回调执行器](#回调执行器) @@ -45,17 +53,27 @@ - **鼠标自动化** — 移动、点击、按下、释放、拖拽、滚动，支持精确坐标控制 - **键盘自动化** — 按下/释放单一按键、输入字符串、组合键、按键状态检测 - **图像识别** — 使用 OpenCV 模板匹配在屏幕上定位 UI 元素，支持可配置的检测阈值 +- **Accessibility 元件搜索** — 通过操作系统无障碍树（Windows UIA / macOS AX）按名称/角色定位按钮、菜单、控件 +- **AI 元件定位（VLM）** — 用自然语言描述 UI 元素，由视觉语言模型（Anthropic / OpenAI）返回屏幕坐标 +- **OCR** — 使用 Tesseract 从屏幕提取文字，可搜索、点击或等待文字出现 +- **剪贴板** — 于 Windows / macOS / Linux 读写系统剪贴板文本 - **截图与屏幕录制** — 捕获全屏或指定区域为图片，录制屏幕为视频（AVI/MP4） - **动作录制与回放** — 录制鼠标/键盘事件并重新播放 -- **JSON 脚本执行** — 使用 JSON 动作文件定义并执行自动化流程 +- **JSON 脚本执行** — 使用 JSON 动作文件定义并执行自动化流程（支持 dry-run 与逐步调试） +- **调度器** — 以 interval 或 cron 表达式执行脚本，两类调度可同时存在 +- **全局热键** — 将 OS 热键绑定到 action 脚本（当前支持 Windows，macOS/Linux 保留扩展接口） +- **事件触发器** — 检测到图像出现、窗口出现、像素变化或文件变动时自动执行脚本 +- **执行历史** — 使用 SQLite 记录 scheduler / triggers / hotkeys / REST 的执行结果；错误时自动附带截图 - **报告生成** — 将测试记录导出为 HTML、JSON 或 XML 报告，包含成功/失败状态 -- **远程自动化** — 启动 TCP Socket 服务器，接收并执行来自远程客户端的自动化命令 +- **远程自动化** — 同时提供 TCP Socket 服务器与 REST API 服务器 +- **插件加载器** — 将定义 `AC_*` 可调用对象的 `.py` 文件放入目录，运行时即可注册为 executor 命令 - **Shell 集成** — 在自动化流程中执行 Shell 命令，支持异步输出捕获 - **回调执行器** — 触发自动化函数后自动调用回调函数，实现操作串联 - **动态包加载** — 在运行时导入外部 Python 包，扩展执行器功能 - **项目与模板管理** — 快速创建包含 keyword/executor 目录结构的自动化项目 - **窗口管理** — 直接将键盘/鼠标事件发送至指定窗口（Windows/Linux） -- **GUI 应用程序** — 内置 PySide6 图形界面，支持交互式自动化操作 +- **GUI 应用程序** — 内置 PySide6 图形界面，支持即时切换语言（English / 繁體中文 / 简体中文 / 日本語） +- **CLI 运行器** — `python -m je_auto_control.cli run|list-jobs|start-server|start-rest` - **跨平台** — 统一 API，支持 Windows、macOS、Linux（X11） --- @@ -79,10 +97,23 @@ je_auto_control/ ├── executor/ # JSON 动作执行引擎 ├── callback/ # 回调函数执行器 ├── cv2_utils/ # OpenCV 截图、模板匹配、视频录制 + ├── accessibility/ # UIA (Windows) / AX (macOS) 元件搜索 + ├── vision/ # VLM 元件定位（Anthropic / OpenAI） + ├── ocr/ # Tesseract 文字定位 + ├── clipboard/ # 跨平台剪贴板 + ├── scheduler/ # Interval + cron 调度器 + ├── hotkey/ # 全局热键守护进程 + ├── triggers/ # 图像/窗口/像素/文件 触发器 + ├── run_history/ # SQLite 执行记录 + 错误截图 + ├── rest_api/ # 纯 stdlib HTTP/REST 服务器 + ├── plugin_loader/ # 动态 AC_* 插件搜索与注册 ├── socket_server/ # TCP Socket 服务器（远程自动化） ├── shell_process/ # Shell 命令管理器 ├── generate_report/ # HTML / JSON / XML 报告生成器 ├── test_record/ # 测试动作记录 + ├── script_vars/ # 脚本变量插值 + ├── watcher/ # 鼠标 / 像素 / log 监视器（Live HUD） + ├── recording_edit/ # 录制内容的裁剪、过滤、缩放 ├── json/ # JSON 动作文件读写 ├── project/ # 项目创建与模板 ├── package_manager/ # 动态包加载 @@ -134,6 +165,12 @@ sudo apt-get install cmake libssl-dev | `python-Xlib` | Linux X11 后端（在 Linux 上自动安装） | | `PySide6` | GUI 应用程序（可选，使用 `[gui]` 安装） | | `qt-material` | GUI 主题（可选，使用 `[gui]` 安装） | +| `uiautomation` | Windows Accessibility 后端（可选，首次使用时加载） | +| `pytesseract` + Tesseract | OCR 文字识别（可选，首次使用时加载） | +| `anthropic` | VLM 定位 — Anthropic 后端（可选，首次使用时加载） | +| `openai` | VLM 定位 — OpenAI 后端（可选，首次使用时加载） | + +完整第三方依赖及其许可证请见 [Third_Party_License.md](../Third_Party_License.md)。 --- @@ -196,6 +233,92 @@ print(f"找到位置: ({cx}, {cy})") je_auto_control.locate_and_click("submit_button.png", mouse_keycode="mouse_left") ``` +### Accessibility 元件搜索 + +通过操作系统无障碍树按名称/角色/App 搜索控件（Windows UIA，via +`uiautomation`；macOS AX）。 + +```python +import je_auto_control + +# 列出 Calculator 中所有可见按钮 +elements = je_auto_control.list_accessibility_elements(app_name="Calculator") + +# 查找特定元件 +ok = je_auto_control.find_accessibility_element(name="OK", role="Button") +if ok is not None: + print(ok.bounds, ok.center) + +# 一步定位并点击 +je_auto_control.click_accessibility_element(name="OK", app_name="Calculator") +``` + +当前平台无可用后端时会抛出 `AccessibilityNotAvailableError`。 + +### AI 元件定位（VLM） + +当模板匹配与 Accessibility 都失效时，可用自然语言描述元件，交给视觉 +语言模型返回坐标。 + +```python +import je_auto_control + +# 默认优先 Anthropic（若已设置 ANTHROPIC_API_KEY），否则使用 OpenAI +x, y = je_auto_control.locate_by_description("绿色的 Submit 按钮") + +# 一步定位并点击 +je_auto_control.click_by_description( + "Cookie 横幅上的『全部接受』按钮", + screen_region=[0, 800, 1920, 1080], # 可选：只在该区域内搜索 +) +``` + +配置（仅从环境变量读取 — 密钥不会写入代码或日志）： + +| 变量 | 作用 | +|---|---| +| `ANTHROPIC_API_KEY` | 启用 Anthropic 后端 | +| `OPENAI_API_KEY` | 启用 OpenAI 后端 | +| `AUTOCONTROL_VLM_BACKEND` | 强制指定 `anthropic` 或 `openai` | +| `AUTOCONTROL_VLM_MODEL` | 覆盖默认模型（如 `claude-opus-4-7`、`gpt-4o-mini`） | + +若两个 SDK 均未安装或未设置 API key，会抛出 `VLMNotAvailableError`。 + +### OCR 屏幕文字识别 + +```python +import je_auto_control as ac + +# 查找所有匹配的文字位置 +matches = ac.find_text_matches("Submit") + +# 获取第一个匹配的中心坐标（找不到返回 None） +cx, cy = ac.locate_text_center("Submit") + +# 一步定位并点击 +ac.click_text("Submit") + +# 等待文字出现（或 timeout） +ac.wait_for_text("加载完成", timeout=15.0) +``` + +若 Tesseract 不在 `PATH` 中，可手动指定路径： + +```python +ac.set_tesseract_cmd(r"C:\Program Files\Tesseract-OCR\tesseract.exe") +``` + +### 剪贴板 + +```python +import je_auto_control as ac +ac.set_clipboard("hello") +text = ac.get_clipboard() +``` + +后端：Windows（Win32 + ctypes）、macOS（`pbcopy`/`pbpaste`）、Linux +（`xclip` 或 `xsel`）。 + ### 截图 ```python @@ -269,6 +392,10 @@ je_auto_control.execute_action([ | 键盘 | `AC_type_keyboard`, `AC_press_keyboard_key`, `AC_release_keyboard_key`, `AC_write`, `AC_hotkey`, `AC_check_key_is_press` | | 图像 | `AC_locate_all_image`, `AC_locate_image_center`, `AC_locate_and_click` | | 屏幕 | `AC_screen_size`, `AC_screenshot` | +| Accessibility | `AC_a11y_list`, `AC_a11y_find`, `AC_a11y_click` | +| VLM（AI 定位） | `AC_vlm_locate`, `AC_vlm_click` | +| OCR | `AC_locate_text`, `AC_click_text`, `AC_wait_text` | +| 剪贴板 | `AC_clipboard_get`, `AC_clipboard_set` | | 录制 | `AC_record`, `AC_stop_record` | | 报告 | `AC_generate_html`, `AC_generate_json`, `AC_generate_xml`, `AC_generate_html_report`, `AC_generate_json_report`, `AC_generate_xml_report` | | 项目 | `AC_create_project` | @@ -276,6 +403,72 @@ je_auto_control.execute_action([ | 进程 | `AC_execute_process` | | 执行器 | `AC_execute_action`, `AC_execute_files` | +### 调度器（Interval & Cron） + +```python +import je_auto_control as ac + +# Interval：每 30 秒执行一次 +job = ac.default_scheduler.add_job( + script_path="scripts/poll.json", interval_seconds=30, repeat=True, +) + +# Cron：周一到周五 09:00（字段为 minute hour dom month dow） +cron_job = ac.default_scheduler.add_cron_job( + script_path="scripts/daily.json", cron_expression="0 9 * * 1-5", +) + +ac.default_scheduler.start() +``` + +两种调度可同时存在，通过 `job.is_cron` 区分类型。 + +### 全局热键 + +将 OS 热键绑定到 action JSON 脚本（Windows 后端；macOS / Linux 的 +`start()` 目前会抛出 `NotImplementedError`，接口已按 Strategy pattern +保留）。 + +```python +from je_auto_control import default_hotkey_daemon + +default_hotkey_daemon.bind("ctrl+alt+1", "scripts/greet.json") +default_hotkey_daemon.start() +``` + +### 事件触发器 + +轮询式触发器，检测到条件成立时自动执行脚本： + +```python +from je_auto_control import ( + default_trigger_engine, ImageAppearsTrigger, + WindowAppearsTrigger, PixelColorTrigger, FilePathTrigger, +) + +default_trigger_engine.add(ImageAppearsTrigger( + trigger_id="", script_path="scripts/click_ok.json", + image_path="templates/ok_button.png", threshold=0.85, repeat=True, +)) +default_trigger_engine.start() +``` + +### 执行历史 + +调度器、触发器、热键、REST API 及 GUI 手动回放的每一次执行都会写入 +`~/.je_auto_control/history.db`。错误时会自动在 +`~/.je_auto_control/artifacts/run_{id}_{ms}.png` 附上截图以便排查。 + +```python +from je_auto_control import default_history_store + +for run in default_history_store.list_runs(limit=20): + print(run.id, run.source, run.status, run.artifact_path) +``` + +GUI **执行历史** 标签页提供筛选 / 刷新 / 清除功能，并可双击截图列打开 +附件。 + ### 报告生成 ```python @@ -301,43 +494,44 @@ xml_string = je_auto_control.generate_xml() 报告内容包含：每个记录动作的函数名称、参数、时间戳及异常信息（如有）。HTML 报告中成功的动作以青色显示，失败的动作以红色显示。 -### 远程自动化（Socket 服务器） +### 远程自动化（Socket / REST） -启动 TCP 服务器，接收来自远程客户端的 JSON 自动化命令： +提供两种服务器：原始 TCP socket 与纯 stdlib HTTP/REST。默认均绑定 +`127.0.0.1`，绑定到 `0.0.0.0` 需显式指定。 ```python -import je_auto_control +import je_auto_control as ac -# 启动服务器（默认：localhost:9938） -server = je_auto_control.start_autocontrol_socket_server(host="localhost", port=9938) +# TCP Socket 服务器（默认：127.0.0.1:9938） +ac.start_autocontrol_socket_server(host="127.0.0.1", port=9938) -# 服务器在后台线程中运行 -# 通过 TCP 发送 JSON 动作命令即可远程执行 -# 发送 "quit_server" 关闭服务器 +# REST API 服务器（默认：127.0.0.1:9939） +ac.start_rest_api_server(host="127.0.0.1", port=9939) +# 端点： +# GET /health 存活检查 +# GET /jobs 列出调度任务 +# POST /execute body: {"actions": [...]} ``` -客户端示例： +### 插件加载器 + +将定义顶层 `AC_*` 可调用对象的 `.py` 文件放入一个目录，运行时即可注 +册为 executor 命令： ```python -import socket -import json +from je_auto_control import ( + load_plugin_directory, register_plugin_commands, +) -sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) -sock.connect(("localhost", 9938)) +commands = load_plugin_directory("./my_plugins") +register_plugin_commands(commands) -# 发送自动化命令 -command = json.dumps([ - ["AC_set_mouse_position", {"x": 500, "y": 300}], - ["AC_click_mouse", {"mouse_keycode": "mouse_left"}] -]) -sock.sendall(command.encode("utf-8")) - -# 接收响应 -response = sock.recv(8192).decode("utf-8") -print(response) -sock.close() +# 之后任何 JSON 脚本都能使用： +# [["AC_greet", {"name": "world"}]] ``` +> **警告：** 插件文件会直接执行任意 Python，请仅加载自己信任的目录。 + ### Shell 命令执行 ```python @@ -496,6 +690,24 @@ python -m je_auto_control --execute_str '[["AC_screenshot", {"file_path": "test. python -m je_auto_control -c ./my_project ``` +另外还有以 headless API 为基础的子命令 CLI： + +```bash +# 执行脚本（可带变量或 dry-run） +python -m je_auto_control.cli run script.json +python -m je_auto_control.cli run script.json --var name=alice --dry-run + +# 列出调度任务 +python -m je_auto_control.cli list-jobs + +# 启动 Socket / REST 服务器 +python -m je_auto_control.cli start-server --port 9938 +python -m je_auto_control.cli start-rest --port 9939 +``` + +`--var name=value` 优先以 JSON 解析（`count=10` 会变成 int），解析失败 +则视为字符串。 + --- ## 平台支持 @@ -540,4 +752,6 @@ python -m pytest test/integrated_test/ ## 许可证 -[MIT License](../LICENSE) © JE-Chen +[MIT License](../LICENSE) © JE-Chen。 +第三方依赖的许可证请见 +[Third_Party_License.md](../Third_Party_License.md)。 diff --git a/README/README_zh-TW.md b/README/README_zh-TW.md index 817235cf..ab1896cd 100644 --- a/README/README_zh-TW.md +++ b/README/README_zh-TW.md @@ -17,15 +17,23 @@ - [安裝](#安裝) - [系統需求](#系統需求) - [快速開始](#快速開始) -- [API 參考](#api-參考) - [滑鼠控制](#滑鼠控制) - [鍵盤控制](#鍵盤控制) - [圖像辨識](#圖像辨識) - - [螢幕操作](#螢幕操作) + - [Accessibility 元件搜尋](#accessibility-元件搜尋) + - [AI 元件定位（VLM）](#ai-元件定位vlm) + - [OCR 螢幕文字辨識](#ocr-螢幕文字辨識) + - [剪貼簿](#剪貼簿) + - [截圖](#截圖) - [動作錄製與回放](#動作錄製與回放) - [JSON 腳本執行器](#json-腳本執行器) + - [排程器（Interval & Cron）](#排程器interval--cron) + - [全域熱鍵](#全域熱鍵) + - [事件觸發器](#事件觸發器) + - [執行歷史](#執行歷史) - [報告產生](#報告產生) - - [遠端自動化（Socket 伺服器）](#遠端自動化socket-伺服器) + - [遠端自動化（Socket / REST）](#遠端自動化socket--rest) + - [外掛載入器](#外掛載入器) - [Shell 命令執行](#shell-命令執行) - [螢幕錄製](#螢幕錄製) - [回呼執行器](#回呼執行器) @@ -45,17 +53,27 @@ - **滑鼠自動化** — 移動、點擊、按下、釋放、拖曳、滾動，支援精確座標控制 - **鍵盤自動化** — 按下/釋放單一按鍵、輸入字串、組合鍵、按鍵狀態偵測 - **圖像辨識** — 使用 OpenCV 模板匹配在螢幕上定位 UI 元素，支援可設定的偵測閾值 +- **Accessibility 元件搜尋** — 透過作業系統無障礙樹（Windows UIA / macOS AX）依名稱/角色定位按鈕、選單、控制項 +- **AI 元件定位（VLM）** — 用自然語言描述 UI 元素，交由視覺語言模型（Anthropic / OpenAI）取得螢幕座標 +- **OCR** — 使用 Tesseract 從螢幕擷取文字，可搜尋、點擊或等待文字出現 +- **剪貼簿** — 於 Windows / macOS / Linux 讀寫系統剪貼簿文字 - **截圖與螢幕錄製** — 擷取全螢幕或指定區域為圖片，錄製螢幕為影片（AVI/MP4） - **動作錄製與回放** — 錄製滑鼠/鍵盤事件並重新播放 -- **JSON 腳本執行** — 使用 JSON 動作檔案定義並執行自動化流程 +- **JSON 腳本執行** — 使用 JSON 動作檔案定義並執行自動化流程（支援 dry-run 與逐步除錯） +- **排程器** — 以 interval 或 cron 表示式執行腳本，interval 與 cron job 可同時存在 +- **全域熱鍵** — 將 OS 熱鍵綁定到 action 腳本（目前為 Windows，macOS/Linux 保留擴充介面） +- **事件觸發器** — 偵測到影像出現、視窗出現、像素變化或檔案變動時自動執行腳本 +- **執行歷史** — 以 SQLite 紀錄 scheduler / triggers / hotkeys / REST 的執行結果；錯誤時自動附上截圖 - **報告產生** — 將測試紀錄匯出為 HTML、JSON 或 XML 報告，包含成功/失敗狀態 -- **遠端自動化** — 啟動 TCP Socket 伺服器，接收並執行來自遠端客戶端的自動化命令 +- **遠端自動化** — 同時提供 TCP Socket 伺服器與 REST API 伺服器 +- **外掛載入器** — 將定義 `AC_*` 可呼叫物的 `.py` 檔放入目錄，執行時即可註冊成 executor 指令 - **Shell 整合** — 在自動化流程中執行 Shell 命令，支援非同步輸出擷取 - **回呼執行器** — 觸發自動化函式後自動呼叫回呼函式，實現操作串接 - **動態套件載入** — 在執行時匯入外部 Python 套件，擴充執行器功能 - **專案與範本管理** — 快速建立包含 keyword/executor 目錄結構的自動化專案 - **視窗管理** — 直接將鍵盤/滑鼠事件送至指定視窗（Windows/Linux） -- **GUI 應用程式** — 內建 PySide6 圖形介面，支援互動式自動化操作 +- **GUI 應用程式** — 內建 PySide6 圖形介面，支援即時切換語系（English / 繁體中文 / 简体中文 / 日本語） +- **CLI 執行介面** — `python -m je_auto_control.cli run|list-jobs|start-server|start-rest` - **跨平台** — 統一 API，支援 Windows、macOS、Linux（X11） --- @@ -79,10 +97,23 @@ je_auto_control/ ├── executor/ # JSON 動作執行引擎 ├── callback/ # 回呼函式執行器 ├── cv2_utils/ # OpenCV 截圖、模板匹配、影片錄製 + ├── accessibility/ # UIA (Windows) / AX (macOS) 元件搜尋 + ├── vision/ # VLM 元件定位（Anthropic / OpenAI） + ├── ocr/ # Tesseract 文字定位 + ├── clipboard/ # 跨平台剪貼簿 + ├── scheduler/ # Interval + cron 排程器 + ├── hotkey/ # 全域熱鍵守護程序 + ├── triggers/ # 影像/視窗/像素/檔案 觸發器 + ├── run_history/ # SQLite 執行紀錄 + 錯誤截圖 + ├── rest_api/ # 純 stdlib HTTP/REST 伺服器 + ├── plugin_loader/ # 動態 AC_* 外掛搜尋與註冊 ├── socket_server/ # TCP Socket 伺服器（遠端自動化） ├── shell_process/ # Shell 命令管理器 ├── generate_report/ # HTML / JSON / XML 報告產生器 ├── test_record/ # 測試動作紀錄 + ├── script_vars/ # 腳本變數插值 + ├── watcher/ # 滑鼠 / 像素 / log 監看器（Live HUD） + ├── recording_edit/ # 錄製內容的修剪、過濾、縮放 ├── json/ # JSON 動作檔案讀寫 ├── project/ # 專案建立與範本 ├── package_manager/ # 動態套件載入 @@ -134,6 +165,12 @@ sudo apt-get install cmake libssl-dev | `python-Xlib` | Linux X11 後端（在 Linux 上自動安裝） | | `PySide6` | GUI 應用程式（選用，使用 `[gui]` 安裝） | | `qt-material` | GUI 主題（選用，使用 `[gui]` 安裝） | +| `uiautomation` | Windows Accessibility 後端（選用，首次使用時載入） | +| `pytesseract` + Tesseract | OCR 文字辨識（選用，首次使用時載入） | +| `anthropic` | VLM 定位 — Anthropic 後端（選用，首次使用時載入） | +| `openai` | VLM 定位 — OpenAI 後端（選用，首次使用時載入） | + +完整第三方相依套件與授權資訊請見 [Third_Party_License.md](../Third_Party_License.md)。 --- @@ -196,6 +233,92 @@ print(f"找到位置: ({cx}, {cy})") je_auto_control.locate_and_click("submit_button.png", mouse_keycode="mouse_left") ``` +### Accessibility 元件搜尋 + +透過作業系統無障礙樹依名稱/角色/App 搜尋控制項（Windows UIA，via +`uiautomation`；macOS AX）。 + +```python +import je_auto_control + +# 列出 Calculator 中所有可見按鈕 +elements = je_auto_control.list_accessibility_elements(app_name="Calculator") + +# 搜尋特定元件 +ok = je_auto_control.find_accessibility_element(name="OK", role="Button") +if ok is not None: + print(ok.bounds, ok.center) + +# 一步定位並點擊 +je_auto_control.click_accessibility_element(name="OK", app_name="Calculator") +``` + +若當前平台無可用後端，會拋出 `AccessibilityNotAvailableError`。 + +### AI 元件定位（VLM） + +當模板匹配與 Accessibility 都失效時，可用自然語言描述元件，交給視覺 +語言模型取得座標。 + +```python +import je_auto_control + +# 預設偏好 Anthropic（若有設定 ANTHROPIC_API_KEY），否則用 OpenAI +x, y = je_auto_control.locate_by_description("綠色的 Submit 按鈕") + +# 一次定位並點擊 +je_auto_control.click_by_description( + "Cookie 橫幅中的『全部接受』按鈕", + screen_region=[0, 800, 1920, 1080], # 可選：只在此區域找 +) +``` + +設定（僅從環境變數讀取 — 金鑰不會被寫入程式碼或日誌）： + +| 變數 | 作用 | +|---|---| +| `ANTHROPIC_API_KEY` | 啟用 Anthropic 後端 | +| `OPENAI_API_KEY` | 啟用 OpenAI 後端 | +| `AUTOCONTROL_VLM_BACKEND` | 強制指定 `anthropic` 或 `openai` | +| `AUTOCONTROL_VLM_MODEL` | 覆寫預設模型（如 `claude-opus-4-7`、`gpt-4o-mini`） | + +若兩個 SDK 皆未安裝或未設定 API key，會拋出 `VLMNotAvailableError`。 + +### OCR 螢幕文字辨識 + +```python +import je_auto_control as ac + +# 找出所有吻合的文字位置 +matches = ac.find_text_matches("Submit") + +# 取得第一個吻合位置的中心座標（找不到則回傳 None） +cx, cy = ac.locate_text_center("Submit") + +# 一步定位並點擊 +ac.click_text("Submit") + +# 等待文字出現（或 timeout） +ac.wait_for_text("載入完成", timeout=15.0) +``` + +若 Tesseract 不在 `PATH` 中，可手動指定路徑： + +```python +ac.set_tesseract_cmd(r"C:\Program Files\Tesseract-OCR\tesseract.exe") +``` + +### 剪貼簿 + +```python +import je_auto_control as ac +ac.set_clipboard("hello") +text = ac.get_clipboard() +``` + +後端：Windows（Win32 + ctypes）、macOS（`pbcopy`/`pbpaste`）、Linux +（`xclip` 或 `xsel`）。 + ### 截圖 ```python @@ -269,6 +392,10 @@ je_auto_control.execute_action([ | 鍵盤 | `AC_type_keyboard`, `AC_press_keyboard_key`, `AC_release_keyboard_key`, `AC_write`, `AC_hotkey`, `AC_check_key_is_press` | | 圖像 | `AC_locate_all_image`, `AC_locate_image_center`, `AC_locate_and_click` | | 螢幕 | `AC_screen_size`, `AC_screenshot` | +| Accessibility | `AC_a11y_list`, `AC_a11y_find`, `AC_a11y_click` | +| VLM（AI 定位） | `AC_vlm_locate`, `AC_vlm_click` | +| OCR | `AC_locate_text`, `AC_click_text`, `AC_wait_text` | +| 剪貼簿 | `AC_clipboard_get`, `AC_clipboard_set` | | 錄製 | `AC_record`, `AC_stop_record` | | 報告 | `AC_generate_html`, `AC_generate_json`, `AC_generate_xml`, `AC_generate_html_report`, `AC_generate_json_report`, `AC_generate_xml_report` | | 專案 | `AC_create_project` | @@ -276,6 +403,72 @@ je_auto_control.execute_action([ | 程序 | `AC_execute_process` | | 執行器 | `AC_execute_action`, `AC_execute_files` | +### 排程器（Interval & Cron） + +```python +import je_auto_control as ac + +# Interval：每 30 秒執行一次 +job = ac.default_scheduler.add_job( + script_path="scripts/poll.json", interval_seconds=30, repeat=True, +) + +# Cron：週一到週五 09:00（欄位為 minute hour dom month dow） +cron_job = ac.default_scheduler.add_cron_job( + script_path="scripts/daily.json", cron_expression="0 9 * * 1-5", +) + +ac.default_scheduler.start() +``` + +兩種排程可同時存在，可由 `job.is_cron` 判斷類型。 + +### 全域熱鍵 + +將 OS 熱鍵綁定到 action JSON 腳本（Windows 後端；macOS / Linux 的 +`start()` 目前會拋出 `NotImplementedError`，介面已依 Strategy pattern +預留）。 + +```python +from je_auto_control import default_hotkey_daemon + +default_hotkey_daemon.bind("ctrl+alt+1", "scripts/greet.json") +default_hotkey_daemon.start() +``` + +### 事件觸發器 + +輪詢式觸發器，偵測到條件成立時自動執行腳本： + +```python +from je_auto_control import ( + default_trigger_engine, ImageAppearsTrigger, + WindowAppearsTrigger, PixelColorTrigger, FilePathTrigger, +) + +default_trigger_engine.add(ImageAppearsTrigger( + trigger_id="", script_path="scripts/click_ok.json", + image_path="templates/ok_button.png", threshold=0.85, repeat=True, +)) +default_trigger_engine.start() +``` + +### 執行歷史 + +排程器、觸發器、熱鍵、REST API 與 GUI 手動回放的每一次執行都會被寫入 +`~/.je_auto_control/history.db`。錯誤時會自動在 +`~/.je_auto_control/artifacts/run_{id}_{ms}.png` 附上截圖以便除錯。 + +```python +from je_auto_control import default_history_store + +for run in default_history_store.list_runs(limit=20): + print(run.id, run.source, run.status, run.artifact_path) +``` + +GUI **執行歷史** 分頁提供篩選 / 更新 / 清除功能，並可雙擊截圖欄位開啟 +附件。 + ### 報告產生 ```python @@ -301,43 +494,44 @@ xml_string = je_auto_control.generate_xml() 報告內容包含：每個紀錄動作的函式名稱、參數、時間戳記及例外資訊（如有）。HTML 報告中成功的動作以青色顯示，失敗的動作以紅色顯示。 -### 遠端自動化（Socket 伺服器） +### 遠端自動化（Socket / REST） -啟動 TCP 伺服器，接收來自遠端客戶端的 JSON 自動化命令： +提供兩種伺服器：原始 TCP socket 與純 stdlib HTTP/REST。預設均綁定 +`127.0.0.1`，綁定到 `0.0.0.0` 須明確指定。 ```python -import je_auto_control +import je_auto_control as ac -# 啟動伺服器（預設：localhost:9938） -server = je_auto_control.start_autocontrol_socket_server(host="localhost", port=9938) +# TCP Socket 伺服器（預設：127.0.0.1:9938） +ac.start_autocontrol_socket_server(host="127.0.0.1", port=9938) -# 伺服器在背景執行緒中運行 -# 透過 TCP 發送 JSON 動作命令即可遠端執行 -# 發送 "quit_server" 關閉伺服器 +# REST API 伺服器（預設：127.0.0.1:9939） +ac.start_rest_api_server(host="127.0.0.1", port=9939) +# 端點： +# GET /health 存活檢查 +# GET /jobs 列出排程工作 +# POST /execute body: {"actions": [...]} ``` -客戶端範例： +### 外掛載入器 + +將定義頂層 `AC_*` 可呼叫物的 `.py` 檔放進一個目錄，執行時即可註冊成 +executor 指令： ```python -import socket -import json +from je_auto_control import ( + load_plugin_directory, register_plugin_commands, +) -sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) -sock.connect(("localhost", 9938)) +commands = load_plugin_directory("./my_plugins") +register_plugin_commands(commands) -# 發送自動化命令 -command = json.dumps([ - ["AC_set_mouse_position", {"x": 500, "y": 300}], - ["AC_click_mouse", {"mouse_keycode": "mouse_left"}] -]) -sock.sendall(command.encode("utf-8")) - -# 接收回應 -response = sock.recv(8192).decode("utf-8") -print(response) -sock.close() +# 之後任何 JSON 腳本都能使用： +# [["AC_greet", {"name": "world"}]] ``` +> **警告：** 外掛檔案會直接執行任意 Python，請僅載入自己信任的目錄。 + ### Shell 命令執行 ```python @@ -496,6 +690,24 @@ python -m je_auto_control --execute_str '[["AC_screenshot", {"file_path": "test. python -m je_auto_control -c ./my_project ``` +另外還有以 headless API 為基礎的子命令 CLI： + +```bash +# 執行腳本（可帶變數或 dry-run） +python -m je_auto_control.cli run script.json +python -m je_auto_control.cli run script.json --var name=alice --dry-run + +# 列出排程工作 +python -m je_auto_control.cli list-jobs + +# 啟動 Socket / REST 伺服器 +python -m je_auto_control.cli start-server --port 9938 +python -m je_auto_control.cli start-rest --port 9939 +``` + +`--var name=value` 會優先以 JSON 解析（`count=10` 會變成 int），失敗 +則視為字串。 + --- ## 平台支援 @@ -540,4 +752,6 @@ python -m pytest test/integrated_test/ ## 授權條款 -[MIT License](../LICENSE) © JE-Chen +[MIT License](../LICENSE) © JE-Chen。 +第三方相依套件之授權請見 +[Third_Party_License.md](../Third_Party_License.md)。 diff --git a/Third_Party_License.md b/Third_Party_License.md new file mode 100644 index 00000000..56cd8556 --- /dev/null +++ b/Third_Party_License.md @@ -0,0 +1,90 @@ +# Third-Party Licenses + +AutoControl is distributed under the [MIT License](LICENSE). It depends on +the third-party components listed below, each covered by its own license. +Full copies of the upstream license texts are archived under `LICENSEs/`. + +--- + +## Runtime dependencies (pinned in `pyproject.toml`) + +| Package | Version | License | Purpose | +|---|---|---|---| +| [je_open_cv](https://pypi.org/project/je-open-cv/) | 0.0.22 | MIT | OpenCV-based image recognition helpers (`locate_all_image`, template match) | +| [Pillow](https://pypi.org/project/Pillow/) | 12.2.0 | MIT-CMU (HPND) | Screenshot encoding, image I/O | +| [mss](https://pypi.org/project/mss/) | 10.1.0 | MIT | Fast multi-monitor screenshot backend | +| [pyobjc-core](https://pypi.org/project/pyobjc-core/) | 12.1 | MIT | macOS backend — Python/Objective-C bridge *(Darwin only)* | +| [pyobjc](https://pypi.org/project/pyobjc/) | 12.1 | MIT | macOS backend — Cocoa / Quartz bindings *(Darwin only)* | +| [python-Xlib](https://pypi.org/project/python-xlib/) | 0.33 | LGPL-2.1-or-later | Linux X11 backend *(Linux only)* | + +### Optional GUI extras (`pip install je_auto_control[gui]`) + +| Package | Version | License | Purpose | +|---|---|---|---| +| [PySide6](https://pypi.org/project/PySide6/) | 6.11.0 | LGPL-3.0 / Qt Commercial | Qt 6 GUI framework used by `start_autocontrol_gui()` | +| [qt-material](https://pypi.org/project/qt-material/) | 2.17 | BSD-2-Clause | Material Design themes for PySide6 | + +### Optional feature dependencies (loaded lazily, not pinned) + +| Package | License | Purpose | +|---|---|---| +| [pytesseract](https://pypi.org/project/pytesseract/) + Tesseract OCR | Apache-2.0 / Apache-2.0 | OCR engine behind `find_text_matches`, `click_text` | +| [anthropic](https://pypi.org/project/anthropic/) | MIT | VLM element locator — Anthropic backend (`locate_by_description`) | +| [openai](https://pypi.org/project/openai/) | Apache-2.0 | VLM element locator — OpenAI backend | +| [uiautomation](https://pypi.org/project/uiautomation/) | Apache-2.0 | Windows accessibility backend (UIA) *(Windows only)* | + +These are imported on first use; AutoControl degrades gracefully when any +are absent (see `VLMNotAvailableError`, `AccessibilityNotAvailableError`, +and the OCR engine's `pytesseract is required` error message). + +--- + +## Development-only dependencies + +Listed in `dev_requirements.txt`. Build/packaging and documentation only — +not shipped with the wheel. + +| Package | License | +|---|---| +| [wheel](https://pypi.org/project/wheel/) | MIT | +| [build](https://pypi.org/project/build/) | MIT | +| [twine](https://pypi.org/project/twine/) | Apache-2.0 | +| [sphinx](https://pypi.org/project/Sphinx/) | BSD-2-Clause | +| [sphinx-rtd-theme](https://pypi.org/project/sphinx-rtd-theme/) | MIT | +| [pytest](https://pypi.org/project/pytest/) | MIT | +| [ruff](https://pypi.org/project/ruff/) | MIT | +| [pylint](https://pypi.org/project/pylint/) | GPL-2.0 | +| [bandit](https://pypi.org/project/bandit/) | Apache-2.0 | +| [radon](https://pypi.org/project/radon/) | MIT | + +--- + +## External runtime services + +Optional backends AutoControl talks to over HTTPS. API keys are read only +from environment variables and never logged or persisted. + +| Service | Used by | Env var | Terms | +|---|---|---|---| +| Anthropic API | VLM locator | `ANTHROPIC_API_KEY` | https://www.anthropic.com/legal | +| OpenAI API | VLM locator | `OPENAI_API_KEY` | https://openai.com/policies | + +Override the preferred backend with `AUTOCONTROL_VLM_BACKEND=anthropic|openai` +and the default model with `AUTOCONTROL_VLM_MODEL=`. + +--- + +## Attributions + +Local copies of upstream license texts are kept under `LICENSEs/` for: + +- `AutoControl/LICENSE` — this project's MIT license +- `Numpy/` — NumPy (transitive via OpenCV) +- `OpenCV/` — OpenCV (transitive via `je_open_cv`) +- `Pillow/` — Pillow +- `python_xlib/` — python-Xlib + +If you redistribute AutoControl, include these notices alongside your +distribution and comply with each dependency's license terms (in +particular LGPL components — PySide6 and python-Xlib — require that users +can relink against modified versions of the library). diff --git a/dev.toml b/dev.toml index 72fa2d18..5f4acacf 100644 --- a/dev.toml +++ b/dev.toml @@ -1,7 +1,7 @@ # Rename to build dev version # This is dev version [build-system] -requires = ["setuptools"] +requires = ["setuptools>=82.0.1"] build-backend = "setuptools.build_meta" [project] diff --git a/dev_requirements.txt b/dev_requirements.txt index e6cf0ecd..18ea09f9 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -6,4 +6,5 @@ sphinx sphinx-rtd-theme PySide6==6.11.0 qt-material==2.17 -mss==10.1.0 +mss==10.2.0 +defusedxml==0.7.1 diff --git a/docs/requirements.txt b/docs/requirements.txt index 38840222..90bf9540 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,2 @@ sphinx-rtd-theme -sphinx>=7.0 +sphinx>=8.1.3 diff --git a/docs/source/Eng/doc/new_features/new_features_doc.rst b/docs/source/Eng/doc/new_features/new_features_doc.rst index 29fb41ef..e58dd6ad 100644 --- a/docs/source/Eng/doc/new_features/new_features_doc.rst +++ b/docs/source/Eng/doc/new_features/new_features_doc.rst @@ -209,3 +209,90 @@ The main window is now a ``QMainWindow`` with: - **Help** → About Close any tab with its ✕ button; re-open it via *View → Tabs*. + + +OCR (text on screen) +==================== + +Tesseract-backed text locator. Useful when a button or label has no +stable accessibility name and no template image:: + + import je_auto_control as ac + + matches = ac.find_text_matches("Submit") + cx, cy = ac.locate_text_center("Submit") + ac.click_text("Submit") + ac.wait_for_text("Loading complete", timeout=15.0) + +If Tesseract isn't on ``PATH``:: + + ac.set_tesseract_cmd(r"C:\Program Files\Tesseract-OCR\tesseract.exe") + +Action-JSON commands: ``AC_locate_text``, ``AC_click_text``, +``AC_wait_text``. + + +Accessibility element finder +============================ + +Query the OS accessibility tree (Windows UIA via ``uiautomation``, +macOS AX) by name / role / app name:: + + import je_auto_control as ac + + elements = ac.list_accessibility_elements(app_name="Calculator") + ok = ac.find_accessibility_element(name="OK", role="Button") + ac.click_accessibility_element(name="OK", app_name="Calculator") + +Raises ``AccessibilityNotAvailableError`` on platforms where no backend +is installed. Action-JSON commands: ``AC_a11y_list``, ``AC_a11y_find``, +``AC_a11y_click``. GUI: **Accessibility** tab. + + +VLM (AI) element locator +======================== + +When neither template matching nor accessibility can find the element, +describe it in plain language and let a vision-language model return +pixel coordinates:: + + import je_auto_control as ac + + x, y = ac.locate_by_description("the green Submit button") + ac.click_by_description( + "the cookie-banner 'Accept all' button", + screen_region=[0, 800, 1920, 1080], # optional crop + ) + +Backends (loaded lazily, zero imports at package import time): + +- Anthropic (``anthropic`` SDK, ``ANTHROPIC_API_KEY``) +- OpenAI (``openai`` SDK, ``OPENAI_API_KEY``) + +Environment variables (keys are never logged or persisted): + +- ``ANTHROPIC_API_KEY`` / ``OPENAI_API_KEY`` +- ``AUTOCONTROL_VLM_BACKEND=anthropic|openai`` +- ``AUTOCONTROL_VLM_MODEL=`` + +Action-JSON commands: ``AC_vlm_locate``, ``AC_vlm_click``. GUI: +**AI Locator** tab. + + +Run history + error-snapshot artifacts +====================================== + +Every run from the scheduler, trigger engine, hotkey daemon, REST API, +and manual GUI replay is recorded to ``~/.je_auto_control/history.db`` +(SQLite). When a run finishes with an error, a screenshot is captured +automatically and attached to the row:: + + from je_auto_control import default_history_store + + for run in default_history_store.list_runs(limit=20): + print(run.id, run.source, run.status, run.artifact_path) + +Artifacts are stored under ``~/.je_auto_control/artifacts/`` and are +removed when the matching run is pruned or the history is cleared. GUI: +**Run History** tab — double-click the artifact column to open the +screenshot in the OS image viewer. diff --git a/docs/source/Zh/doc/new_features/new_features_doc.rst b/docs/source/Zh/doc/new_features/new_features_doc.rst index 155a1285..de5420f0 100644 --- a/docs/source/Zh/doc/new_features/new_features_doc.rst +++ b/docs/source/Zh/doc/new_features/new_features_doc.rst @@ -198,3 +198,87 @@ GUI 多語系 - **Help** → 關於 點擊分頁的 ✕ 即可關閉，之後可從 *View → Tabs* 恢復。 + + +OCR 螢幕文字辨識 +================ + +以 Tesseract 為後端的文字定位。適用於沒有穩定 Accessibility 名稱、也 +不方便擷取模板影像的按鈕或標籤:: + + import je_auto_control as ac + + matches = ac.find_text_matches("Submit") + cx, cy = ac.locate_text_center("Submit") + ac.click_text("Submit") + ac.wait_for_text("載入完成", timeout=15.0) + +若 Tesseract 不在 ``PATH`` 中:: + + ac.set_tesseract_cmd(r"C:\Program Files\Tesseract-OCR\tesseract.exe") + +Action-JSON 指令：``AC_locate_text``、``AC_click_text``、 +``AC_wait_text``。 + + +Accessibility 元件搜尋 +====================== + +透過作業系統無障礙樹查詢控制項（Windows UIA 透過 ``uiautomation``； +macOS AX），支援依名稱 / 角色 / 應用程式過濾:: + + import je_auto_control as ac + + elements = ac.list_accessibility_elements(app_name="Calculator") + ok = ac.find_accessibility_element(name="OK", role="Button") + ac.click_accessibility_element(name="OK", app_name="Calculator") + +當前平台若沒有可用後端會拋出 ``AccessibilityNotAvailableError``。 +Action-JSON 指令：``AC_a11y_list``、``AC_a11y_find``、 +``AC_a11y_click``。GUI：**Accessibility** 分頁。 + + +VLM（AI）元件定位 +================= + +當模板匹配與 Accessibility 都無法找到目標時，可用自然語言描述元件， +交給視覺語言模型回傳像素座標:: + + import je_auto_control as ac + + x, y = ac.locate_by_description("綠色的 Submit 按鈕") + ac.click_by_description( + "Cookie 橫幅中的『全部接受』按鈕", + screen_region=[0, 800, 1920, 1080], # 可選：只在此區域搜尋 + ) + +後端（延遲載入，import ``je_auto_control`` 時不會引入）： + +- Anthropic (``anthropic`` SDK，``ANTHROPIC_API_KEY``) +- OpenAI (``openai`` SDK，``OPENAI_API_KEY``) + +環境變數（金鑰不會被記錄或寫入磁碟）： + +- ``ANTHROPIC_API_KEY`` / ``OPENAI_API_KEY`` +- ``AUTOCONTROL_VLM_BACKEND=anthropic|openai`` +- ``AUTOCONTROL_VLM_MODEL=`` + +Action-JSON 指令：``AC_vlm_locate``、``AC_vlm_click``。GUI： +**AI Locator** 分頁。 + + +執行歷史 + 錯誤截圖附件 +======================= + +排程器、觸發器、熱鍵守護程序、REST API 與 GUI 手動回放的每一次執行 +都會被寫入 ``~/.je_auto_control/history.db``（SQLite）。失敗時會自動 +擷取螢幕截圖並附到該筆紀錄上:: + + from je_auto_control import default_history_store + + for run in default_history_store.list_runs(limit=20): + print(run.id, run.source, run.status, run.artifact_path) + +截圖檔存於 ``~/.je_auto_control/artifacts/``，相關紀錄被 prune 或整個 +歷史被清除時會一併刪除。GUI：**Run History** 分頁 — 雙擊截圖欄位可開 +啟 OS 預覽。 diff --git a/docs/source/conf.py b/docs/source/conf.py index b5e1f297..b773c10c 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -11,7 +11,7 @@ # -- Project information ----------------------------------------------------- project = 'AutoControl' -copyright = '2020 ~ Now, JE-Chen' +copyright = '2020 ~ Now, JE-Chen' # noqa: A001 # reason: Sphinx-required name author = 'JE-Chen' release = '0.0.179' diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 51f5192f..9bd2d399 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -39,6 +39,16 @@ execute_action_with_vars from je_auto_control.utils.executor.action_executor import execute_files from je_auto_control.utils.executor.action_executor import executor +# Accessibility (headless) +from je_auto_control.utils.accessibility import ( + AccessibilityElement, AccessibilityNotAvailableError, + click_accessibility_element, find_accessibility_element, + list_accessibility_elements, +) +# VLM element locator (headless) +from je_auto_control.utils.vision import ( + VLMNotAvailableError, click_by_description, locate_by_description, +) # Clipboard (headless) from je_auto_control.utils.clipboard.clipboard import ( get_clipboard, set_clipboard, @@ -61,6 +71,10 @@ from je_auto_control.utils.rest_api.rest_server import ( RestApiServer, start_rest_api_server, ) +# Run history (headless) +from je_auto_control.utils.run_history.history_store import ( + HistoryStore, RunRecord, default_history_store, +) # Triggers (headless) from je_auto_control.utils.triggers.trigger_engine import ( FilePathTrigger, ImageAppearsTrigger, PixelColorTrigger, TriggerEngine, @@ -124,7 +138,7 @@ from je_auto_control.wrapper.auto_control_image import locate_all_image from je_auto_control.wrapper.auto_control_image import locate_and_click from je_auto_control.wrapper.auto_control_image import locate_image_center -# import keyboard +# Keyboard wrappers from je_auto_control.wrapper.auto_control_keyboard import check_key_is_press from je_auto_control.wrapper.auto_control_keyboard import get_keyboard_keys_table from je_auto_control.wrapper.auto_control_keyboard import hotkey @@ -134,7 +148,7 @@ from je_auto_control.wrapper.auto_control_keyboard import send_key_event_to_window from je_auto_control.wrapper.auto_control_keyboard import type_keyboard from je_auto_control.wrapper.auto_control_keyboard import write -# import mouse +# Mouse wrappers from je_auto_control.wrapper.auto_control_mouse import click_mouse from je_auto_control.wrapper.auto_control_mouse import get_mouse_position from je_auto_control.wrapper.auto_control_mouse import mouse_keys_table @@ -148,7 +162,7 @@ # record from je_auto_control.wrapper.auto_control_record import record from je_auto_control.wrapper.auto_control_record import stop_record -# import screen +# Screen wrappers from je_auto_control.wrapper.auto_control_screen import screen_size from je_auto_control.wrapper.auto_control_screen import screenshot from je_auto_control.wrapper.auto_control_screen import get_pixel @@ -205,6 +219,14 @@ def start_autocontrol_gui(*args, **kwargs): "TriggerEngine", "default_trigger_engine", "ImageAppearsTrigger", "WindowAppearsTrigger", "PixelColorTrigger", "FilePathTrigger", + # Run history + "HistoryStore", "RunRecord", "default_history_store", + # Accessibility + "AccessibilityElement", "AccessibilityNotAvailableError", + "list_accessibility_elements", "find_accessibility_element", + "click_accessibility_element", + # VLM locator + "VLMNotAvailableError", "locate_by_description", "click_by_description", "generate_html", "generate_html_report", "generate_json", "generate_json_report", "generate_xml", "generate_xml_report", "get_dir_files_as_list", "create_project_dir", "start_autocontrol_socket_server", "callback_executor", "package_manager", "ShellManager", "default_shell_manager", diff --git a/je_auto_control/gui/_auto_click_tab.py b/je_auto_control/gui/_auto_click_tab.py index 650b389f..684732d1 100644 --- a/je_auto_control/gui/_auto_click_tab.py +++ b/je_auto_control/gui/_auto_click_tab.py @@ -5,7 +5,6 @@ QGroupBox, ) -from je_auto_control.gui.language_wrapper.multi_language_wrapper import language_wrapper from je_auto_control.wrapper.auto_control_keyboard import ( type_keyboard, hotkey, write, get_keyboard_keys_table, ) @@ -15,28 +14,25 @@ ) -def _t(key: str) -> str: - return language_wrapper.language_word_dict.get(key, key) - - class AutoClickTabMixin: """ Mixin that provides the auto-click tab UI and handlers. Requires the host widget to expose `self.timer`, `self.repeat_count`, - `self.repeat_max` attributes set in its __init__. + `self.repeat_max` attributes, and the ``TranslatableMixin`` helpers + (``self._tr(...)``) set up by its __init__. """ def _build_auto_click_tab(self) -> QWidget: tab = QWidget() outer = QVBoxLayout() - click_group = QGroupBox(_t("tab_auto_click")) + click_group = self._tr(QGroupBox(), "tab_auto_click") grid = QGridLayout() row = 0 - grid.addWidget(QLabel(_t("input_method")), row, 0) - self.mouse_radio = QRadioButton(_t("mouse_radio")) - self.keyboard_radio = QRadioButton(_t("keyboard_radio")) + grid.addWidget(self._tr(QLabel(), "input_method"), row, 0) + self.mouse_radio = self._tr(QRadioButton(), "mouse_radio") + self.keyboard_radio = self._tr(QRadioButton(), "keyboard_radio") self.mouse_radio.setChecked(True) self._input_group = QButtonGroup() self._input_group.addButton(self.mouse_radio) @@ -47,25 +43,25 @@ def _build_auto_click_tab(self) -> QWidget: grid.addLayout(h, row, 1) row += 1 - grid.addWidget(QLabel(_t("interval_time")), row, 0) + grid.addWidget(self._tr(QLabel(), "interval_time"), row, 0) self.interval_input = QLineEdit("1000") self.interval_input.setValidator(QIntValidator(1, 999999999)) grid.addWidget(self.interval_input, row, 1) row += 1 - grid.addWidget(QLabel(_t("cursor_x")), row, 0) + grid.addWidget(self._tr(QLabel(), "cursor_x"), row, 0) self.cursor_x_input = QLineEdit() self.cursor_x_input.setValidator(QIntValidator()) grid.addWidget(self.cursor_x_input, row, 1) row += 1 - grid.addWidget(QLabel(_t("cursor_y")), row, 0) + grid.addWidget(self._tr(QLabel(), "cursor_y"), row, 0) self.cursor_y_input = QLineEdit() self.cursor_y_input.setValidator(QIntValidator()) grid.addWidget(self.cursor_y_input, row, 1) row += 1 - grid.addWidget(QLabel(_t("mouse_button")), row, 0) + grid.addWidget(self._tr(QLabel(), "mouse_button"), row, 0) self.mouse_button_combo = QComboBox() self.mouse_button_combo.addItems( list(mouse_keys_table.keys()) if isinstance(mouse_keys_table, dict) else list(mouse_keys_table) @@ -73,23 +69,27 @@ def _build_auto_click_tab(self) -> QWidget: grid.addWidget(self.mouse_button_combo, row, 1) row += 1 - grid.addWidget(QLabel(_t("keyboard_button")), row, 0) + grid.addWidget(self._tr(QLabel(), "keyboard_button"), row, 0) self.keyboard_button_combo = QComboBox() self.keyboard_button_combo.addItems(list(get_keyboard_keys_table().keys())) grid.addWidget(self.keyboard_button_combo, row, 1) row += 1 - grid.addWidget(QLabel(_t("click_type")), row, 0) + grid.addWidget(self._tr(QLabel(), "click_type"), row, 0) self.click_type_combo = QComboBox() - self.click_type_combo.addItems([_t("single_click"), _t("double_click")]) + self._click_type_keys = ["single_click", "double_click"] + self.click_type_combo.addItems( + [self._translate(key) for key in self._click_type_keys] + ) grid.addWidget(self.click_type_combo, row, 1) row += 1 - self.repeat_until_stopped = QRadioButton(_t("repeat_until_stopped_radio")) - self.repeat_count_times = QRadioButton(_t("repeat_radio")) - self.repeat_count_input = QLineEdit() + self.repeat_until_stopped = self._tr( + QRadioButton(), "repeat_until_stopped_radio", + ) + self.repeat_count_times = self._tr(QRadioButton(), "repeat_radio") + self.repeat_count_input = self._tr(QLineEdit(), "times") self.repeat_count_input.setValidator(QIntValidator(1, 999999999)) - self.repeat_count_input.setPlaceholderText(_t("times")) rg = QButtonGroup(tab) rg.addButton(self.repeat_until_stopped) rg.addButton(self.repeat_count_times) @@ -102,9 +102,9 @@ def _build_auto_click_tab(self) -> QWidget: row += 1 btn_h = QHBoxLayout() - self.start_button = QPushButton(_t("start")) + self.start_button = self._tr(QPushButton(), "start") self.start_button.clicked.connect(self._start_auto_click) - self.stop_button = QPushButton(_t("stop")) + self.stop_button = self._tr(QPushButton(), "stop") self.stop_button.clicked.connect(self._stop_auto_click) btn_h.addWidget(self.start_button) btn_h.addWidget(self.stop_button) @@ -113,42 +113,46 @@ def _build_auto_click_tab(self) -> QWidget: click_group.setLayout(grid) outer.addWidget(click_group) - pos_group = QGroupBox(_t("get_position")) + pos_group = self._tr(QGroupBox(), "get_position") pos_layout = QHBoxLayout() - self.pos_btn = QPushButton(_t("get_position")) + self.pos_btn = self._tr(QPushButton(), "get_position") self.pos_btn.clicked.connect(self._get_mouse_pos) - self.pos_label = QLabel(_t("current_position") + " --") + self.pos_label = QLabel() + self._pos_label_suffix = " --" + self.pos_label.setText( + self._translate("current_position") + self._pos_label_suffix, + ) pos_layout.addWidget(self.pos_btn) pos_layout.addWidget(self.pos_label) pos_group.setLayout(pos_layout) outer.addWidget(pos_group) - hotkey_group = QGroupBox(_t("hotkey_label")) + hotkey_group = self._tr(QGroupBox(), "hotkey_label") hk_layout = QHBoxLayout() self.hotkey_input = QLineEdit() self.hotkey_input.setPlaceholderText("ctrl,a") - self.hotkey_btn = QPushButton(_t("hotkey_send")) + self.hotkey_btn = self._tr(QPushButton(), "hotkey_send") self.hotkey_btn.clicked.connect(self._send_hotkey) hk_layout.addWidget(self.hotkey_input) hk_layout.addWidget(self.hotkey_btn) hotkey_group.setLayout(hk_layout) outer.addWidget(hotkey_group) - write_group = QGroupBox(_t("write_label")) + write_group = self._tr(QGroupBox(), "write_label") wr_layout = QHBoxLayout() self.write_input = QLineEdit() - self.write_btn = QPushButton(_t("write_send")) + self.write_btn = self._tr(QPushButton(), "write_send") self.write_btn.clicked.connect(self._send_write) wr_layout.addWidget(self.write_input) wr_layout.addWidget(self.write_btn) write_group.setLayout(wr_layout) outer.addWidget(write_group) - scroll_group = QGroupBox(_t("mouse_scroll_label")) + scroll_group = self._tr(QGroupBox(), "mouse_scroll_label") sc_layout = QHBoxLayout() self.scroll_value_input = QLineEdit("3") self.scroll_value_input.setValidator(QIntValidator()) - sc_layout.addWidget(QLabel(_t("mouse_scroll_label"))) + sc_layout.addWidget(self._tr(QLabel(), "mouse_scroll_label")) sc_layout.addWidget(self.scroll_value_input) if special_mouse_keys_table: self.scroll_dir_combo = QComboBox() @@ -156,7 +160,7 @@ def _build_auto_click_tab(self) -> QWidget: sc_layout.addWidget(self.scroll_dir_combo) else: self.scroll_dir_combo = None - self.scroll_btn = QPushButton(_t("scroll_send")) + self.scroll_btn = self._tr(QPushButton(), "scroll_send") self.scroll_btn.clicked.connect(self._send_scroll) sc_layout.addWidget(self.scroll_btn) scroll_group.setLayout(sc_layout) @@ -170,6 +174,21 @@ def _build_auto_click_tab(self) -> QWidget: tab.setLayout(outer) return tab + def _auto_click_retranslate(self) -> None: + """Re-translate composite labels and combo items that ``_tr`` can't own.""" + if hasattr(self, "click_type_combo") and hasattr(self, "_click_type_keys"): + current_index = self.click_type_combo.currentIndex() + self.click_type_combo.clear() + self.click_type_combo.addItems( + [self._translate(key) for key in self._click_type_keys] + ) + if 0 <= current_index < self.click_type_combo.count(): + self.click_type_combo.setCurrentIndex(current_index) + if hasattr(self, "pos_label"): + self.pos_label.setText( + self._translate("current_position") + self._pos_label_suffix, + ) + def _update_click_mode(self): use_mouse = self.mouse_radio.isChecked() self.cursor_x_input.setEnabled(use_mouse) @@ -232,7 +251,10 @@ def _do_click(self): def _get_mouse_pos(self): try: x, y = get_mouse_position() - self.pos_label.setText(_t("current_position") + f" ({x}, {y})") + self._pos_label_suffix = f" ({x}, {y})" + self.pos_label.setText( + self._translate("current_position") + self._pos_label_suffix, + ) self.cursor_x_input.setText(str(x)) self.cursor_y_input.setText(str(y)) except (OSError, ValueError, TypeError, RuntimeError) as error: diff --git a/je_auto_control/gui/_i18n_helpers.py b/je_auto_control/gui/_i18n_helpers.py new file mode 100644 index 00000000..081f2b00 --- /dev/null +++ b/je_auto_control/gui/_i18n_helpers.py @@ -0,0 +1,66 @@ +"""Translation-registry mixin shared by tabs that need live language switching. + +Widgets register their (widget, translation-key, setter-name) triples via +``self._tr(widget, key, setter)`` during UI construction. Calling +``self.retranslate()`` re-pulls every key from the language wrapper and +re-applies it through the recorded setter. Destroyed widgets are skipped +silently so removing a row never breaks a later language switch. +""" +from typing import List, Tuple + +from PySide6.QtWidgets import ( + QAbstractButton, QGroupBox, QLabel, QLineEdit, QTabWidget, QWidget, +) + +from je_auto_control.gui.language_wrapper.multi_language_wrapper import ( + language_wrapper, +) + + +def _default_setter(widget: QWidget) -> str: + if isinstance(widget, QGroupBox): + return "setTitle" + if isinstance(widget, (QLabel, QAbstractButton)): + return "setText" + if isinstance(widget, QLineEdit): + return "setPlaceholderText" + return "setText" + + +class TranslatableMixin: + """Provides ``_tr(...)`` / ``retranslate()`` for a widget-building class.""" + + def _tr_init(self) -> None: + self._tr_registry: List[Tuple[QWidget, str, str]] = [] + self._tr_tabs: List[Tuple[QTabWidget, int, str]] = [] + + def _tr(self, widget: QWidget, key: str, setter: str = "") -> QWidget: + """Set ``widget`` text from ``key`` now and on every retranslate.""" + if not hasattr(self, "_tr_registry"): + self._tr_init() + resolved = setter or _default_setter(widget) + translated = language_wrapper.translate(key, key) + getattr(widget, resolved)(translated) + self._tr_registry.append((widget, key, resolved)) + return widget + + def _tr_tab(self, tab_widget: QTabWidget, index: int, key: str) -> None: + """Register a tab title so it re-translates.""" + if not hasattr(self, "_tr_tabs"): + self._tr_init() + tab_widget.setTabText(index, language_wrapper.translate(key, key)) + self._tr_tabs.append((tab_widget, index, key)) + + def retranslate(self) -> None: + """Re-apply every registered translation key.""" + for widget, key, setter in getattr(self, "_tr_registry", []): + try: + getattr(widget, setter)(language_wrapper.translate(key, key)) + except RuntimeError: + # Widget destroyed; leave it for a future cleanup pass. + continue + for tab_widget, index, key in getattr(self, "_tr_tabs", []): + try: + tab_widget.setTabText(index, language_wrapper.translate(key, key)) + except RuntimeError: + continue diff --git a/je_auto_control/gui/_report_tab.py b/je_auto_control/gui/_report_tab.py new file mode 100644 index 00000000..f598d30c --- /dev/null +++ b/je_auto_control/gui/_report_tab.py @@ -0,0 +1,89 @@ +"""Report-generation tab builder (extracted mixin).""" +from PySide6.QtWidgets import ( + QGroupBox, QHBoxLayout, QLabel, QLineEdit, QPushButton, + QTextEdit, QVBoxLayout, QWidget, +) + +from je_auto_control.utils.generate_report.generate_html_report import generate_html_report +from je_auto_control.utils.generate_report.generate_json_report import generate_json_report +from je_auto_control.utils.generate_report.generate_xml_report import generate_xml_report +from je_auto_control.utils.test_record.record_test_class import test_record_instance + + +class ReportTabMixin: + """Provides the report-generation tab builder/handlers. + + Host widget must expose the TranslatableMixin API (``self._tr(...)``) + so every label/button registers for live language switching. + """ + + def _build_report_tab(self) -> QWidget: + tab = QWidget() + layout = QVBoxLayout() + + tr_group = self._tr(QGroupBox(), "test_record_status") + tr_h = QHBoxLayout() + self.tr_enable_btn = self._tr(QPushButton(), "enable_test_record") + self.tr_enable_btn.clicked.connect(lambda: self._set_test_record(True)) + self.tr_disable_btn = self._tr(QPushButton(), "disable_test_record") + self.tr_disable_btn.clicked.connect(lambda: self._set_test_record(False)) + self.tr_status_label = QLabel("OFF") + tr_h.addWidget(self.tr_enable_btn) + tr_h.addWidget(self.tr_disable_btn) + tr_h.addWidget(self.tr_status_label) + tr_group.setLayout(tr_h) + layout.addWidget(tr_group) + + name_h = QHBoxLayout() + name_h.addWidget(self._tr(QLabel(), "report_name")) + self.report_name_input = QLineEdit("autocontrol_report") + name_h.addWidget(self.report_name_input) + layout.addLayout(name_h) + + btn_h = QHBoxLayout() + self.html_report_btn = self._tr(QPushButton(), "generate_html_report") + self.html_report_btn.clicked.connect(self._gen_html) + self.json_report_btn = self._tr(QPushButton(), "generate_json_report") + self.json_report_btn.clicked.connect(self._gen_json) + self.xml_report_btn = self._tr(QPushButton(), "generate_xml_report") + self.xml_report_btn.clicked.connect(self._gen_xml) + btn_h.addWidget(self.html_report_btn) + btn_h.addWidget(self.json_report_btn) + btn_h.addWidget(self.xml_report_btn) + layout.addLayout(btn_h) + + layout.addWidget(self._tr(QLabel(), "report_result")) + self.report_result_text = QTextEdit() + self.report_result_text.setReadOnly(True) + layout.addWidget(self.report_result_text) + layout.addStretch() + tab.setLayout(layout) + return tab + + def _set_test_record(self, enable: bool): + test_record_instance.set_record_enable(enable) + self.tr_status_label.setText("ON" if enable else "OFF") + + def _gen_html(self): + try: + name = self.report_name_input.text() or "autocontrol_report" + generate_html_report(name) + self.report_result_text.setText(f"HTML report generated: {name}") + except (OSError, ValueError, TypeError, RuntimeError) as error: + self.report_result_text.setText(f"Error: {error}") + + def _gen_json(self): + try: + name = self.report_name_input.text() or "autocontrol_report" + generate_json_report(name) + self.report_result_text.setText(f"JSON report generated: {name}") + except (OSError, ValueError, TypeError, RuntimeError) as error: + self.report_result_text.setText(f"Error: {error}") + + def _gen_xml(self): + try: + name = self.report_name_input.text() or "autocontrol_report" + generate_xml_report(name) + self.report_result_text.setText(f"XML report generated: {name}") + except (OSError, ValueError, TypeError, RuntimeError) as error: + self.report_result_text.setText(f"Error: {error}") diff --git a/je_auto_control/gui/_shell_report_tabs.py b/je_auto_control/gui/_shell_report_tabs.py deleted file mode 100644 index 5f509a5a..00000000 --- a/je_auto_control/gui/_shell_report_tabs.py +++ /dev/null @@ -1,159 +0,0 @@ -"""Shell-command and Report-generation tab builders (extracted mixin).""" -from PySide6.QtWidgets import ( - QFileDialog, QGroupBox, QHBoxLayout, QLabel, QLineEdit, QPushButton, - QTextEdit, QVBoxLayout, QWidget, -) - -from je_auto_control.gui.language_wrapper.multi_language_wrapper import language_wrapper -from je_auto_control.utils.generate_report.generate_html_report import generate_html_report -from je_auto_control.utils.generate_report.generate_json_report import generate_json_report -from je_auto_control.utils.generate_report.generate_xml_report import generate_xml_report -from je_auto_control.utils.shell_process.shell_exec import ShellManager -from je_auto_control.utils.start_exe.start_another_process import start_exe -from je_auto_control.utils.test_record.record_test_class import test_record_instance - - -def _t(key: str) -> str: - return language_wrapper.translate(key, key) - - -class ShellReportTabsMixin: - """Provides shell-command and report-generation tab builders/handlers.""" - - def _build_shell_tab(self) -> QWidget: - tab = QWidget() - layout = QVBoxLayout() - - shell_group = QGroupBox(_t("shell_command_label")) - sg = QVBoxLayout() - self.shell_input = QLineEdit() - self.shell_input.setPlaceholderText("echo hello") - self.shell_exec_btn = QPushButton(_t("execute_shell")) - self.shell_exec_btn.clicked.connect(self._execute_shell) - sh = QHBoxLayout() - sh.addWidget(self.shell_input) - sh.addWidget(self.shell_exec_btn) - sg.addLayout(sh) - shell_group.setLayout(sg) - layout.addWidget(shell_group) - - exe_group = QGroupBox(_t("start_exe_label")) - eg = QHBoxLayout() - self.exe_path_input = QLineEdit() - self.exe_browse_btn = QPushButton(_t("browse")) - self.exe_browse_btn.clicked.connect(self._browse_exe) - self.exe_start_btn = QPushButton(_t("start_exe")) - self.exe_start_btn.clicked.connect(self._start_exe) - eg.addWidget(self.exe_path_input) - eg.addWidget(self.exe_browse_btn) - eg.addWidget(self.exe_start_btn) - exe_group.setLayout(eg) - layout.addWidget(exe_group) - - layout.addWidget(QLabel(_t("shell_output"))) - self.shell_output_text = QTextEdit() - self.shell_output_text.setReadOnly(True) - layout.addWidget(self.shell_output_text) - tab.setLayout(layout) - return tab - - def _execute_shell(self): - try: - cmd = self.shell_input.text() - if not cmd: - return - mgr = ShellManager() - mgr.exec_shell(cmd) - self.shell_output_text.setText( - f"Executed: {cmd}\n(Check console for output)" - ) - except (OSError, ValueError, TypeError, RuntimeError) as error: - self.shell_output_text.setText(f"Error: {error}") - - def _browse_exe(self): - path, _ = QFileDialog.getOpenFileName( - self, _t("start_exe_label"), "", "Executable (*.exe);;All (*)", - ) - if path: - self.exe_path_input.setText(path) - - def _start_exe(self): - try: - path = self.exe_path_input.text() - if not path: - return - start_exe(path) - self.shell_output_text.setText(f"Started: {path}") - except (OSError, ValueError, TypeError, RuntimeError) as error: - self.shell_output_text.setText(f"Error: {error}") - - def _build_report_tab(self) -> QWidget: - tab = QWidget() - layout = QVBoxLayout() - - tr_group = QGroupBox(_t("test_record_status")) - tr_h = QHBoxLayout() - self.tr_enable_btn = QPushButton(_t("enable_test_record")) - self.tr_enable_btn.clicked.connect(lambda: self._set_test_record(True)) - self.tr_disable_btn = QPushButton(_t("disable_test_record")) - self.tr_disable_btn.clicked.connect(lambda: self._set_test_record(False)) - self.tr_status_label = QLabel("OFF") - tr_h.addWidget(self.tr_enable_btn) - tr_h.addWidget(self.tr_disable_btn) - tr_h.addWidget(self.tr_status_label) - tr_group.setLayout(tr_h) - layout.addWidget(tr_group) - - name_h = QHBoxLayout() - name_h.addWidget(QLabel(_t("report_name"))) - self.report_name_input = QLineEdit("autocontrol_report") - name_h.addWidget(self.report_name_input) - layout.addLayout(name_h) - - btn_h = QHBoxLayout() - self.html_report_btn = QPushButton(_t("generate_html_report")) - self.html_report_btn.clicked.connect(self._gen_html) - self.json_report_btn = QPushButton(_t("generate_json_report")) - self.json_report_btn.clicked.connect(self._gen_json) - self.xml_report_btn = QPushButton(_t("generate_xml_report")) - self.xml_report_btn.clicked.connect(self._gen_xml) - btn_h.addWidget(self.html_report_btn) - btn_h.addWidget(self.json_report_btn) - btn_h.addWidget(self.xml_report_btn) - layout.addLayout(btn_h) - - layout.addWidget(QLabel(_t("report_result"))) - self.report_result_text = QTextEdit() - self.report_result_text.setReadOnly(True) - layout.addWidget(self.report_result_text) - layout.addStretch() - tab.setLayout(layout) - return tab - - def _set_test_record(self, enable: bool): - test_record_instance.set_record_enable(enable) - self.tr_status_label.setText("ON" if enable else "OFF") - - def _gen_html(self): - try: - name = self.report_name_input.text() or "autocontrol_report" - generate_html_report(name) - self.report_result_text.setText(f"HTML report generated: {name}") - except (OSError, ValueError, TypeError, RuntimeError) as error: - self.report_result_text.setText(f"Error: {error}") - - def _gen_json(self): - try: - name = self.report_name_input.text() or "autocontrol_report" - generate_json_report(name) - self.report_result_text.setText(f"JSON report generated: {name}") - except (OSError, ValueError, TypeError, RuntimeError) as error: - self.report_result_text.setText(f"Error: {error}") - - def _gen_xml(self): - try: - name = self.report_name_input.text() or "autocontrol_report" - generate_xml_report(name) - self.report_result_text.setText(f"XML report generated: {name}") - except (OSError, ValueError, TypeError, RuntimeError) as error: - self.report_result_text.setText(f"Error: {error}") diff --git a/je_auto_control/gui/accessibility_tab.py b/je_auto_control/gui/accessibility_tab.py new file mode 100644 index 00000000..19c1c907 --- /dev/null +++ b/je_auto_control/gui/accessibility_tab.py @@ -0,0 +1,130 @@ +"""Accessibility tab: browse the OS UI tree and click elements by role/name.""" +from typing import Optional + +from PySide6.QtCore import Qt +from PySide6.QtWidgets import ( + QAbstractItemView, QHBoxLayout, QHeaderView, QLabel, QLineEdit, + QMessageBox, QPushButton, QTableWidget, QTableWidgetItem, + QVBoxLayout, QWidget, +) + +from je_auto_control.gui._i18n_helpers import TranslatableMixin +from je_auto_control.gui.language_wrapper.multi_language_wrapper import ( + language_wrapper, +) +from je_auto_control.utils.accessibility.accessibility_api import ( + click_accessibility_element, list_accessibility_elements, +) +from je_auto_control.utils.accessibility.element import ( + AccessibilityNotAvailableError, +) + +_COLUMN_COUNT = 5 + + +def _t(key: str) -> str: + return language_wrapper.translate(key, key) + + +class AccessibilityTab(TranslatableMixin, QWidget): + """Discover GUI elements via UIA / AX and click them headlessly.""" + + def __init__(self, parent: Optional[QWidget] = None) -> None: + super().__init__(parent) + self._tr_init() + self._app_filter = QLineEdit() + self._name_filter = QLineEdit() + self._table = QTableWidget(0, _COLUMN_COUNT) + self._table.setEditTriggers(QAbstractItemView.NoEditTriggers) + self._table.setSelectionBehavior(QAbstractItemView.SelectRows) + self._table.verticalHeader().setVisible(False) + header = self._table.horizontalHeader() + header.setSectionResizeMode(QHeaderView.Interactive) + header.setStretchLastSection(True) + self._status = QLabel() + self._apply_table_headers() + self._build_layout() + + def retranslate(self) -> None: + TranslatableMixin.retranslate(self) + self._apply_table_headers() + self._app_filter.setPlaceholderText(_t("a11y_app_placeholder")) + self._name_filter.setPlaceholderText(_t("a11y_name_placeholder")) + + def _apply_table_headers(self) -> None: + self._table.setHorizontalHeaderLabels([ + _t("a11y_col_app"), _t("a11y_col_role"), + _t("a11y_col_name"), _t("a11y_col_bounds"), + _t("a11y_col_center"), + ]) + + def _build_layout(self) -> None: + root = QVBoxLayout(self) + row = QHBoxLayout() + row.addWidget(self._tr(QLabel(), "a11y_app_label")) + self._app_filter.setPlaceholderText(_t("a11y_app_placeholder")) + row.addWidget(self._app_filter, stretch=1) + row.addWidget(self._tr(QLabel(), "a11y_name_label")) + self._name_filter.setPlaceholderText(_t("a11y_name_placeholder")) + row.addWidget(self._name_filter, stretch=1) + refresh = self._tr(QPushButton(), "a11y_refresh") + refresh.clicked.connect(self._refresh) + row.addWidget(refresh) + root.addLayout(row) + root.addWidget(self._table, stretch=1) + action_row = QHBoxLayout() + click_btn = self._tr(QPushButton(), "a11y_click_selected") + click_btn.clicked.connect(self._click_selected) + action_row.addWidget(click_btn) + action_row.addStretch() + root.addLayout(action_row) + root.addWidget(self._status) + + def _refresh(self) -> None: + app = self._app_filter.text().strip() or None + try: + elements = list_accessibility_elements(app_name=app) + except AccessibilityNotAvailableError as error: + self._status.setText(str(error)) + self._table.setRowCount(0) + return + name_filter = self._name_filter.text().strip().lower() + if name_filter: + elements = [e for e in elements + if name_filter in e.name.lower()] + self._populate(elements) + self._status.setText( + _t("a11y_count_label").replace("{n}", str(len(elements))), + ) + + def _populate(self, elements) -> None: + self._table.setRowCount(len(elements)) + for row, element in enumerate(elements): + values = ( + element.app_name, element.role, element.name, + f"({element.bounds[0]},{element.bounds[1]}) " + f"{element.bounds[2]}x{element.bounds[3]}", + f"{element.center[0]},{element.center[1]}", + ) + for col, text in enumerate(values): + item = QTableWidgetItem(text) + item.setFlags(item.flags() & ~Qt.ItemIsEditable) + self._table.setItem(row, col, item) + + def _click_selected(self) -> None: + row = self._table.currentRow() + if row < 0: + self._status.setText(_t("a11y_no_selection")) + return + app = self._table.item(row, 0).text() or None + role = self._table.item(row, 1).text() or None + name = self._table.item(row, 2).text() or None + try: + ok = click_accessibility_element( + name=name, role=role, app_name=app, + ) + except AccessibilityNotAvailableError as error: + QMessageBox.warning(self, _t("a11y_click_selected"), str(error)) + return + if not ok: + self._status.setText(_t("a11y_click_not_found")) diff --git a/je_auto_control/gui/hotkeys_tab.py b/je_auto_control/gui/hotkeys_tab.py index a3c7746c..161039a7 100644 --- a/je_auto_control/gui/hotkeys_tab.py +++ b/je_auto_control/gui/hotkeys_tab.py @@ -7,22 +7,31 @@ QTableWidget, QTableWidgetItem, QVBoxLayout, QWidget, ) +from je_auto_control.gui._i18n_helpers import TranslatableMixin +from je_auto_control.gui.language_wrapper.multi_language_wrapper import ( + language_wrapper, +) from je_auto_control.utils.hotkey.hotkey_daemon import default_hotkey_daemon -class HotkeysTab(QWidget): +def _t(key: str) -> str: + return language_wrapper.translate(key, key) + + +class HotkeysTab(TranslatableMixin, QWidget): """Add / remove hotkey bindings and toggle the daemon.""" def __init__(self, parent: Optional[QWidget] = None) -> None: super().__init__(parent) + self._tr_init() self._combo_input = QLineEdit() self._combo_input.setPlaceholderText("ctrl+alt+1") self._script_input = QLineEdit() - self._status = QLabel("Daemon stopped") + self._daemon_running = False + self._status = QLabel() self._table = QTableWidget(0, 4) - self._table.setHorizontalHeaderLabels( - ["ID", "Combo", "Script", "Fired"] - ) + self._apply_status_label() + self._apply_table_headers() self._timer = QTimer(self) self._timer.setInterval(1000) self._timer.timeout.connect(self._refresh) @@ -31,14 +40,14 @@ def __init__(self, parent: Optional[QWidget] = None) -> None: def _build_layout(self) -> None: root = QVBoxLayout(self) form = QHBoxLayout() - form.addWidget(QLabel("Combo:")) + form.addWidget(self._tr(QLabel(), "hk_combo_label")) form.addWidget(self._combo_input) - form.addWidget(QLabel("Script:")) + form.addWidget(self._tr(QLabel(), "hk_script_label")) form.addWidget(self._script_input, stretch=1) - browse = QPushButton("Browse") + browse = self._tr(QPushButton(), "browse") browse.clicked.connect(self._browse) form.addWidget(browse) - add = QPushButton("Bind") + add = self._tr(QPushButton(), "hk_bind") add.clicked.connect(self._on_bind) form.addWidget(add) root.addLayout(form) @@ -46,20 +55,37 @@ def _build_layout(self) -> None: root.addWidget(self._table, stretch=1) ctl = QHBoxLayout() - for label, handler in ( - ("Remove selected", self._on_remove), - ("Start daemon", self._on_start), - ("Stop daemon", self._on_stop), + for key, handler in ( + ("hk_remove_selected", self._on_remove), + ("hk_start_daemon", self._on_start), + ("hk_stop_daemon", self._on_stop), ): - btn = QPushButton(label) + btn = self._tr(QPushButton(), key) btn.clicked.connect(handler) ctl.addWidget(btn) ctl.addStretch() root.addLayout(ctl) root.addWidget(self._status) + def _apply_status_label(self) -> None: + key = "hk_daemon_running" if self._daemon_running else "hk_daemon_stopped" + self._status.setText(_t(key)) + + def _apply_table_headers(self) -> None: + self._table.setHorizontalHeaderLabels([ + _t("hk_col_id"), _t("hk_col_combo"), + _t("hk_col_script"), _t("hk_col_fired"), + ]) + + def retranslate(self) -> None: + TranslatableMixin.retranslate(self) + self._apply_status_label() + self._apply_table_headers() + def _browse(self) -> None: - path, _ = QFileDialog.getOpenFileName(self, "Select script", "", "JSON (*.json)") + path, _ = QFileDialog.getOpenFileName( + self, _t("hk_dialog_select_script"), "", "JSON (*.json)", + ) if path: self._script_input.setText(path) @@ -91,12 +117,14 @@ def _on_start(self) -> None: QMessageBox.warning(self, "Error", str(error)) return self._timer.start() - self._status.setText("Daemon running") + self._daemon_running = True + self._apply_status_label() def _on_stop(self) -> None: default_hotkey_daemon.stop() self._timer.stop() - self._status.setText("Daemon stopped") + self._daemon_running = False + self._apply_status_label() def _refresh(self) -> None: bindings = default_hotkey_daemon.list_bindings() diff --git a/je_auto_control/gui/language_wrapper/english.py b/je_auto_control/gui/language_wrapper/english.py index 450e15b7..40efca04 100644 --- a/je_auto_control/gui/language_wrapper/english.py +++ b/je_auto_control/gui/language_wrapper/english.py @@ -1,3 +1,7 @@ +_SCRIPT_LABEL = "Script:" +_REMOVE_SELECTED = "Remove selected" +_SELECT_SCRIPT = "Select script" + english_word_dict = { # Main "application_name": "AutoControlGUI", @@ -20,6 +24,9 @@ "tab_screen_record": "Screen Recording", "tab_shell": "Shell Command", "tab_report": "Report", + "tab_run_history": "Run History", + "tab_accessibility": "Accessibility", + "tab_vlm": "AI Locator", # Auto Click Tab "interval_time": "Interval (ms):", @@ -121,6 +128,195 @@ # Language "language_label": "Language:", + # Hotkeys Tab + "hk_combo_label": "Combo:", + "hk_script_label": _SCRIPT_LABEL, + "hk_bind": "Bind", + "hk_remove_selected": _REMOVE_SELECTED, + "hk_start_daemon": "Start daemon", + "hk_stop_daemon": "Stop daemon", + "hk_daemon_stopped": "Daemon stopped", + "hk_daemon_running": "Daemon running", + "hk_col_id": "ID", + "hk_col_combo": "Combo", + "hk_col_script": "Script", + "hk_col_fired": "Fired", + "hk_dialog_select_script": _SELECT_SCRIPT, + + # Triggers Tab + "tr_script_label": _SCRIPT_LABEL, + "tr_type_label": "Type:", + "tr_repeat": "Repeat", + "tr_add": "Add trigger", + "tr_remove_selected": _REMOVE_SELECTED, + "tr_start_engine": "Start engine", + "tr_stop_engine": "Stop engine", + "tr_engine_stopped": "Engine stopped", + "tr_engine_running": "Engine running", + "tr_type_image": "Image appears", + "tr_type_window": "Window appears", + "tr_type_pixel": "Pixel matches", + "tr_type_file": "File changed", + "tr_image_label": "Image:", + "tr_threshold_label": "Threshold:", + "tr_title_contains_label": "Title contains:", + "tr_watch_label": "Watch path:", + "tr_col_id": "ID", + "tr_col_type": "Type", + "tr_col_detail": "Detail", + "tr_col_fired": "Fired", + "tr_col_enabled": "Enabled", + "tr_yes": "Yes", + "tr_no": "No", + "tr_dialog_select_script": _SELECT_SCRIPT, + "tr_dialog_select_image": "Select image", + "tr_dialog_select_file": "Select file to watch", + + # Plugins Tab + "pl_dir_label": "Plugin dir:", + "pl_load": "Load + register", + "pl_no_loaded": "No plugins loaded", + "pl_registered_label": "Registered commands:", + "pl_dialog_plugin_dir": "Plugin directory", + + # Scheduler Tab + "sch_script_label": _SCRIPT_LABEL, + "sch_interval_label": "Every (s):", + "sch_repeat": "Repeat", + "sch_add": "Add", + "sch_remove_selected": _REMOVE_SELECTED, + "sch_start": "Start scheduler", + "sch_stop": "Stop scheduler", + "sch_status_running": "Scheduler running", + "sch_status_stopped": "Scheduler stopped", + "sch_col_job_id": "Job ID", + "sch_col_script": "Script", + "sch_col_interval": "Interval (s)", + "sch_col_runs": "Runs", + "sch_col_enabled": "Enabled", + "sch_dialog_select_script": _SELECT_SCRIPT, + + # Socket / REST Tab + "ss_tcp_group": "TCP socket server", + "ss_rest_group": "REST API server", + "ss_host_label": "Host:", + "ss_port_label": "Port:", + "ss_tcp_any_check": "Bind TCP to 0.0.0.0 (exposes to network)", + "ss_rest_any_check": "Bind REST to 0.0.0.0 (exposes to network)", + "ss_tcp_stopped": "TCP stopped", + "ss_rest_stopped": "REST stopped", + "ss_tcp_start": "Start TCP", + "ss_tcp_stop": "Stop TCP", + "ss_rest_start": "Start REST", + "ss_rest_stop": "Stop REST", + + # Live HUD Tab + "hud_watchers_group": "Watchers", + "hud_start": "Start HUD", + "hud_stop": "Stop HUD", + "hud_clear": "Clear log", + "hud_recent_log": "Recent log:", + "hud_mouse_prefix": "Mouse:", + "hud_pixel_prefix": "Pixel:", + + # Window Manager Tab + "win_refresh": "Refresh", + "win_filter_placeholder": "Filter by title substring", + "win_col_hwnd": "HWND", + "win_col_title": "Title", + "win_focus_selected": "Focus selected", + "win_close_selected": "Close selected", + "win_status_count": "{n} windows", + + # Recording Editor Tab + "re_file_label": "File:", + "re_browse": "Browse", + "re_load": "Load", + "re_save_as": "Save As", + "re_trim_start": "Trim start:", + "re_trim_end": "end:", + "re_apply_trim": "Apply trim", + "re_remove_selected": _REMOVE_SELECTED, + "re_delay_x": "Delay x", + "re_floor_ms": "floor ms:", + "re_apply_delays": "Apply delays", + "re_scale_x": "Scale x:", + "re_scale_y": "y:", + "re_apply_scale": "Apply scale", + "re_keep_mouse": "Keep mouse only", + "re_keep_keyboard": "Keep keyboard only", + "re_preview": "Preview:", + "re_dialog_open": "Open recording", + "re_dialog_save": "Save recording", + + # Script Builder Tab + "sb_delete": "Delete", + "sb_up": "Up", + "sb_down": "Down", + "sb_load_json": "Load JSON", + "sb_save_json": "Save JSON", + "sb_run": "Run", + "sb_add_step": "Add Step", + "sb_dialog_save": "Save script", + "sb_dialog_load": "Load script", + "sb_no_step_selected": "No step selected", + + # Run History Tab + "rh_filter_label": "Source:", + "rh_source_all": "All", + "rh_source_scheduler": "Scheduler", + "rh_source_trigger": "Trigger", + "rh_source_hotkey": "Hotkey", + "rh_source_manual": "Manual", + "rh_source_rest": "REST", + "rh_refresh": "Refresh", + "rh_clear": "Clear", + "rh_confirm_clear": "Delete all run history? This cannot be undone.", + "rh_count_label": "{n} runs", + "rh_col_id": "ID", + "rh_col_source": "Source", + "rh_col_target": "Target", + "rh_col_script": "Script", + "rh_col_started": "Started", + "rh_col_duration": "Duration", + "rh_col_status": "Status", + "rh_status_ok": "OK", + "rh_status_error": "Error", + "rh_status_running": "Running", + "rh_col_artifact": "Artifact", + "rh_open_artifact": "Open artifact", + "rh_no_artifact": "Selected run has no artifact.", + "rh_artifact_missing": "Artifact file no longer exists.", + + # Accessibility Tab + "a11y_app_label": "App:", + "a11y_app_placeholder": "e.g. Calculator", + "a11y_name_label": "Name contains:", + "a11y_name_placeholder": "partial match", + "a11y_refresh": "Refresh", + "a11y_click_selected": "Click selected", + "a11y_col_app": "App", + "a11y_col_role": "Role", + "a11y_col_name": "Name", + "a11y_col_bounds": "Bounds", + "a11y_col_center": "Center", + "a11y_count_label": "{n} elements", + "a11y_no_selection": "Select a row first", + "a11y_click_not_found": "No matching element to click", + + # VLM (AI Locator) Tab + "vlm_desc_label": "Describe:", + "vlm_desc_placeholder": "e.g. the green Submit button", + "vlm_model_label": "Model:", + "vlm_model_placeholder": "optional override (e.g. claude-opus-4-7)", + "vlm_locate": "Locate", + "vlm_click": "Locate & click", + "vlm_result": "Match at ({x}, {y})", + "vlm_ok": "OK", + "vlm_not_found": "No match found", + "vlm_error": "Error", + "vlm_desc_required": "Describe the target first", + # Menu bar "menu_file": "File", "menu_file_open_script": "Open Script...", diff --git a/je_auto_control/gui/language_wrapper/japanese.py b/je_auto_control/gui/language_wrapper/japanese.py index 095dd1b6..aff81d72 100644 --- a/je_auto_control/gui/language_wrapper/japanese.py +++ b/je_auto_control/gui/language_wrapper/japanese.py @@ -1,3 +1,8 @@ +_SCRIPT = "スクリプト" +_SCRIPT_LABEL = "スクリプト:" +_REMOVE_SELECTED = "選択項目を削除" +_SELECT_SCRIPT = "スクリプトを選択" + japanese_word_dict = { "application_name": "AutoControlGUI", @@ -19,6 +24,298 @@ "tab_screen_record": "画面録画", "tab_shell": "シェル", "tab_report": "レポート", + "tab_run_history": "実行履歴", + "tab_accessibility": "アクセシビリティ", + "tab_vlm": "AI ロケーター", + + # Auto Click Tab + "interval_time": "間隔 (ms):", + "cursor_x": "カーソル X:", + "cursor_y": "カーソル Y:", + "mouse_button": "マウスボタン:", + "keyboard_button": "キーボード:", + "click_type": "クリックタイプ:", + "single_click": "シングルクリック", + "double_click": "ダブルクリック", + "input_method": "入力方式:", + "mouse_radio": "マウス", + "keyboard_radio": "キーボード", + "repeat_until_stopped_radio": "停止まで繰り返し", + "repeat_radio": "繰り返し", + "times": "回数", + "start": "開始", + "stop": "停止", + "hotkey_label": "ホットキー（カンマ区切り）:", + "hotkey_send": "ホットキー送信", + "write_label": "テキスト入力:", + "write_send": "入力", + "mouse_scroll_label": "スクロール値:", + "scroll_direction_label": "スクロール方向:", + "scroll_send": "スクロール", + "get_position": "マウス位置取得", + "current_position": "現在位置:", + + # Screenshot Tab + "take_screenshot": "スクリーンショット撮影", + "save_screenshot": "スクリーンショット保存", + "file_path_label": "ファイルパス:", + "browse": "参照", + "region_label": "領域 (x1, y1, x2, y2):", + "pick_region": "領域選択", + "crop_template": "テンプレート切り抜き", + "get_pixel_label": "ピクセル色取得", + "pixel_x": "X:", + "pixel_y": "Y:", + "pixel_result": "ピクセル色:", + "screen_size_label": "画面サイズ:", + "get_screen_size": "画面サイズ取得", + + # Image Detection Tab + "template_image": "テンプレート画像:", + "threshold_label": "閾値 (0.0~1.0):", + "locate_image": "画像検索", + "locate_all": "全て検索", + "locate_click": "検索してクリック", + "detection_result": "検出結果:", + "draw_image_check": "検出結果を描画", + + # Record / Playback Tab + "start_record": "記録開始", + "stop_record": "記録停止", + "playback": "再生", + "save_record": "JSON 保存", + "load_record": "JSON 読込", + "record_status": "状態:", + "record_idle": "待機", + "record_recording": "記録中...", + "record_list_label": "記録済みアクション:", + + # Script Executor Tab + "load_script": "スクリプト読込 (JSON)", + "execute_script": "実行", + "execute_dir_label": "実行ディレクトリ:", + "execute_dir": "全て実行", + "script_content": "スクリプト内容:", + "execution_result": "実行結果:", + + # Screen Recording Tab + "recorder_name": "レコーダー名:", + "output_file": "出力ファイル:", + "codec_label": "コーデック:", + "fps_label": "FPS:", + "resolution_label": "解像度 (幅x高さ):", + "start_screen_record": "録画開始", + "stop_screen_record": "録画停止", + "screen_record_status": "状態:", + + # Shell Tab + "shell_command_label": "シェルコマンド:", + "execute_shell": "実行", + "shell_output": "出力:", + "start_exe_label": "実行ファイルパス:", + "start_exe": "実行ファイル起動", + + # Report Tab + "report_name": "レポート名:", + "generate_html_report": "HTML レポート生成", + "generate_json_report": "JSON レポート生成", + "generate_xml_report": "XML レポート生成", + "enable_test_record": "テスト記録を有効化", + "disable_test_record": "テスト記録を無効化", + "test_record_status": "テスト記録:", + "report_result": "結果:", + + # Hotkeys Tab + "hk_combo_label": "組合せ:", + "hk_script_label": _SCRIPT_LABEL, + "hk_bind": "バインド", + "hk_remove_selected": _REMOVE_SELECTED, + "hk_start_daemon": "デーモン開始", + "hk_stop_daemon": "デーモン停止", + "hk_daemon_stopped": "デーモン停止中", + "hk_daemon_running": "デーモン実行中", + "hk_col_id": "ID", + "hk_col_combo": "組合せ", + "hk_col_script": _SCRIPT, + "hk_col_fired": "発火", + "hk_dialog_select_script": _SELECT_SCRIPT, + + # Triggers Tab + "tr_script_label": _SCRIPT_LABEL, + "tr_type_label": "タイプ:", + "tr_repeat": "繰り返し", + "tr_add": "トリガー追加", + "tr_remove_selected": _REMOVE_SELECTED, + "tr_start_engine": "エンジン開始", + "tr_stop_engine": "エンジン停止", + "tr_engine_stopped": "エンジン停止中", + "tr_engine_running": "エンジン実行中", + "tr_type_image": "画像出現", + "tr_type_window": "ウィンドウ出現", + "tr_type_pixel": "ピクセル一致", + "tr_type_file": "ファイル変更", + "tr_image_label": "画像:", + "tr_threshold_label": "閾値:", + "tr_title_contains_label": "タイトルに含む:", + "tr_watch_label": "監視パス:", + "tr_col_id": "ID", + "tr_col_type": "タイプ", + "tr_col_detail": "詳細", + "tr_col_fired": "発火", + "tr_col_enabled": "有効", + "tr_yes": "はい", + "tr_no": "いいえ", + "tr_dialog_select_script": _SELECT_SCRIPT, + "tr_dialog_select_image": "画像を選択", + "tr_dialog_select_file": "監視するファイルを選択", + + # Plugins Tab + "pl_dir_label": "プラグインディレクトリ:", + "pl_load": "読込と登録", + "pl_no_loaded": "プラグイン未読込", + "pl_registered_label": "登録済みコマンド:", + "pl_dialog_plugin_dir": "プラグインディレクトリ", + + # Scheduler Tab + "sch_script_label": _SCRIPT_LABEL, + "sch_interval_label": "毎 (秒):", + "sch_repeat": "繰り返し", + "sch_add": "追加", + "sch_remove_selected": _REMOVE_SELECTED, + "sch_start": "スケジューラー開始", + "sch_stop": "スケジューラー停止", + "sch_status_running": "スケジューラー実行中", + "sch_status_stopped": "スケジューラー停止中", + "sch_col_job_id": "ジョブ ID", + "sch_col_script": _SCRIPT, + "sch_col_interval": "間隔 (秒)", + "sch_col_runs": "実行回数", + "sch_col_enabled": "有効", + "sch_dialog_select_script": _SELECT_SCRIPT, + + # Socket / REST Tab + "ss_tcp_group": "TCP ソケットサーバー", + "ss_rest_group": "REST API サーバー", + "ss_host_label": "ホスト:", + "ss_port_label": "ポート:", + "ss_tcp_any_check": "TCP を 0.0.0.0 にバインド（ネットワーク公開）", + "ss_rest_any_check": "REST を 0.0.0.0 にバインド（ネットワーク公開）", + "ss_tcp_stopped": "TCP 停止中", + "ss_rest_stopped": "REST 停止中", + "ss_tcp_start": "TCP 開始", + "ss_tcp_stop": "TCP 停止", + "ss_rest_start": "REST 開始", + "ss_rest_stop": "REST 停止", + + # Live HUD Tab + "hud_watchers_group": "ウォッチャー", + "hud_start": "HUD 開始", + "hud_stop": "HUD 停止", + "hud_clear": "ログクリア", + "hud_recent_log": "最近のログ:", + "hud_mouse_prefix": "マウス:", + "hud_pixel_prefix": "ピクセル:", + + # Window Manager Tab + "win_refresh": "更新", + "win_filter_placeholder": "タイトル部分文字列でフィルタ", + "win_col_hwnd": "HWND", + "win_col_title": "タイトル", + "win_focus_selected": "選択項目をフォーカス", + "win_close_selected": "選択項目を閉じる", + "win_status_count": "{n} ウィンドウ", + + # Recording Editor Tab + "re_file_label": "ファイル:", + "re_browse": "参照", + "re_load": "読込", + "re_save_as": "名前を付けて保存", + "re_trim_start": "トリム開始:", + "re_trim_end": "終了:", + "re_apply_trim": "トリム適用", + "re_remove_selected": _REMOVE_SELECTED, + "re_delay_x": "ディレイ倍率", + "re_floor_ms": "下限 ms:", + "re_apply_delays": "ディレイ適用", + "re_scale_x": "スケール x:", + "re_scale_y": "y:", + "re_apply_scale": "スケール適用", + "re_keep_mouse": "マウスのみ保持", + "re_keep_keyboard": "キーボードのみ保持", + "re_preview": "プレビュー:", + "re_dialog_open": "記録を開く", + "re_dialog_save": "記録を保存", + + # Script Builder Tab + "sb_delete": "削除", + "sb_up": "上へ", + "sb_down": "下へ", + "sb_load_json": "JSON 読込", + "sb_save_json": "JSON 保存", + "sb_run": "実行", + "sb_add_step": "ステップ追加", + "sb_dialog_save": "スクリプト保存", + "sb_dialog_load": "スクリプト読込", + "sb_no_step_selected": "ステップ未選択", + + # Language + "language_label": "言語:", + + # Run History Tab + "rh_filter_label": "ソース:", + "rh_source_all": "すべて", + "rh_source_scheduler": "スケジューラー", + "rh_source_trigger": "トリガー", + "rh_source_hotkey": "ホットキー", + "rh_source_manual": "手動", + "rh_source_rest": "REST", + "rh_refresh": "更新", + "rh_clear": "クリア", + "rh_confirm_clear": "すべての実行履歴を削除しますか？この操作は取り消せません。", + "rh_count_label": "{n} 件", + "rh_col_id": "ID", + "rh_col_source": "ソース", + "rh_col_target": "対象", + "rh_col_script": _SCRIPT, + "rh_col_started": "開始時刻", + "rh_col_duration": "所要時間", + "rh_col_status": "ステータス", + "rh_status_ok": "成功", + "rh_status_error": "エラー", + "rh_status_running": "実行中", + "rh_col_artifact": "スクリーンショット", + "rh_open_artifact": "スクリーンショットを開く", + "rh_no_artifact": "選択した実行にスクリーンショットはありません。", + "rh_artifact_missing": "スクリーンショットファイルが存在しません。", + + # Accessibility Tab + "a11y_app_label": "アプリ:", + "a11y_app_placeholder": "例: 電卓", + "a11y_name_label": "名前に含む:", + "a11y_name_placeholder": "部分一致", + "a11y_refresh": "更新", + "a11y_click_selected": "選択項目をクリック", + "a11y_col_app": "アプリ", + "a11y_col_role": "ロール", + "a11y_col_name": "名前", + "a11y_col_bounds": "境界", + "a11y_col_center": "中心", + "a11y_count_label": "{n} 個の要素", + "a11y_no_selection": "行を選択してください", + "a11y_click_not_found": "クリック可能な要素が見つかりません", + + # VLM (AI Locator) Tab + "vlm_desc_label": "説明:", + "vlm_desc_placeholder": "例: 緑色の送信ボタン", + "vlm_model_label": "モデル:", + "vlm_model_placeholder": "任意 (例: claude-opus-4-7)", + "vlm_locate": "検索", + "vlm_click": "検索してクリック", + "vlm_result": "位置: ({x}, {y})", + "vlm_ok": "完了", + "vlm_not_found": "一致する要素が見つかりません", + "vlm_error": "エラー", + "vlm_desc_required": "まず対象の説明を入力してください", # Menu bar "menu_file": "ファイル", @@ -33,10 +330,4 @@ "menu_language": "言語", "menu_help": "ヘルプ", "menu_help_about": "AutoControlGUI について", - - # Status/messages - "language_label": "言語:", - "browse": "参照", - "start": "開始", - "stop": "停止", } diff --git a/je_auto_control/gui/language_wrapper/multi_language_wrapper.py b/je_auto_control/gui/language_wrapper/multi_language_wrapper.py index 5ef0aba9..29fdf17b 100644 --- a/je_auto_control/gui/language_wrapper/multi_language_wrapper.py +++ b/je_auto_control/gui/language_wrapper/multi_language_wrapper.py @@ -49,7 +49,7 @@ def reset_language(self, language: str) -> None: return self.language = language self.language_word_dict = self._merged(language) - for listener in list(self._listeners): + for listener in self._listeners[:]: try: listener(language) except (OSError, RuntimeError) as error: diff --git a/je_auto_control/gui/language_wrapper/simplified_chinese.py b/je_auto_control/gui/language_wrapper/simplified_chinese.py index ee6b3af7..d6c5fa02 100644 --- a/je_auto_control/gui/language_wrapper/simplified_chinese.py +++ b/je_auto_control/gui/language_wrapper/simplified_chinese.py @@ -19,6 +19,298 @@ "tab_screen_record": "屏幕录像", "tab_shell": "Shell 命令", "tab_report": "报告生成", + "tab_run_history": "执行记录", + "tab_accessibility": "无障碍树", + "tab_vlm": "AI 定位", + + # Auto Click Tab + "interval_time": "间隔时间 (ms):", + "cursor_x": "光标 X:", + "cursor_y": "光标 Y:", + "mouse_button": "鼠标按键:", + "keyboard_button": "键盘按键:", + "click_type": "点击类型:", + "single_click": "单击", + "double_click": "双击", + "input_method": "输入方式:", + "mouse_radio": "鼠标", + "keyboard_radio": "键盘", + "repeat_until_stopped_radio": "重复直到停止", + "repeat_radio": "重复", + "times": "次数", + "start": "开始", + "stop": "停止", + "hotkey_label": "组合键（逗号分隔）：", + "hotkey_send": "发送组合键", + "write_label": "输入文字:", + "write_send": "输入", + "mouse_scroll_label": "滚动值:", + "scroll_direction_label": "滚动方向:", + "scroll_send": "滚动", + "get_position": "获取鼠标位置", + "current_position": "当前位置:", + + # Screenshot Tab + "take_screenshot": "截取屏幕", + "save_screenshot": "保存截图", + "file_path_label": "文件路径:", + "browse": "浏览", + "region_label": "区域 (x1, y1, x2, y2):", + "pick_region": "框选区域", + "crop_template": "框选模板", + "get_pixel_label": "获取像素颜色", + "pixel_x": "X:", + "pixel_y": "Y:", + "pixel_result": "像素颜色:", + "screen_size_label": "屏幕大小:", + "get_screen_size": "获取屏幕大小", + + # Image Detection Tab + "template_image": "模板图像:", + "threshold_label": "阈值 (0.0~1.0):", + "locate_image": "定位图像", + "locate_all": "定位全部", + "locate_click": "定位并点击", + "detection_result": "检测结果:", + "draw_image_check": "标记检测结果", + + # Record / Playback Tab + "start_record": "开始录制", + "stop_record": "停止录制", + "playback": "回放", + "save_record": "保存为 JSON", + "load_record": "载入 JSON", + "record_status": "状态:", + "record_idle": "闲置", + "record_recording": "录制中...", + "record_list_label": "已录制动作:", + + # Script Executor Tab + "load_script": "载入脚本 (JSON)", + "execute_script": "执行", + "execute_dir_label": "执行目录:", + "execute_dir": "执行全部", + "script_content": "脚本内容:", + "execution_result": "执行结果:", + + # Screen Recording Tab + "recorder_name": "录像器名称:", + "output_file": "输出文件:", + "codec_label": "编码器:", + "fps_label": "FPS:", + "resolution_label": "分辨率 (宽x高):", + "start_screen_record": "开始录像", + "stop_screen_record": "停止录像", + "screen_record_status": "状态:", + + # Shell Tab + "shell_command_label": "Shell 命令:", + "execute_shell": "执行", + "shell_output": "输出:", + "start_exe_label": "可执行文件路径:", + "start_exe": "启动可执行文件", + + # Report Tab + "report_name": "报告名称:", + "generate_html_report": "生成 HTML 报告", + "generate_json_report": "生成 JSON 报告", + "generate_xml_report": "生成 XML 报告", + "enable_test_record": "启用测试记录", + "disable_test_record": "禁用测试记录", + "test_record_status": "测试记录:", + "report_result": "结果:", + + # Hotkeys Tab + "hk_combo_label": "组合键:", + "hk_script_label": "脚本:", + "hk_bind": "绑定", + "hk_remove_selected": "移除所选", + "hk_start_daemon": "启动服务", + "hk_stop_daemon": "停止服务", + "hk_daemon_stopped": "服务已停止", + "hk_daemon_running": "服务运行中", + "hk_col_id": "ID", + "hk_col_combo": "组合键", + "hk_col_script": "脚本", + "hk_col_fired": "已触发", + "hk_dialog_select_script": "选择脚本", + + # Triggers Tab + "tr_script_label": "脚本:", + "tr_type_label": "类型:", + "tr_repeat": "重复", + "tr_add": "新增触发器", + "tr_remove_selected": "移除所选", + "tr_start_engine": "启动引擎", + "tr_stop_engine": "停止引擎", + "tr_engine_stopped": "引擎已停止", + "tr_engine_running": "引擎运行中", + "tr_type_image": "图像出现", + "tr_type_window": "窗口出现", + "tr_type_pixel": "像素匹配", + "tr_type_file": "文件变更", + "tr_image_label": "图像:", + "tr_threshold_label": "阈值:", + "tr_title_contains_label": "标题包含:", + "tr_watch_label": "监看路径:", + "tr_col_id": "ID", + "tr_col_type": "类型", + "tr_col_detail": "细节", + "tr_col_fired": "已触发", + "tr_col_enabled": "启用", + "tr_yes": "是", + "tr_no": "否", + "tr_dialog_select_script": "选择脚本", + "tr_dialog_select_image": "选择图像", + "tr_dialog_select_file": "选择要监看的文件", + + # Plugins Tab + "pl_dir_label": "插件目录:", + "pl_load": "载入并注册", + "pl_no_loaded": "尚未载入插件", + "pl_registered_label": "已注册命令:", + "pl_dialog_plugin_dir": "插件目录", + + # Scheduler Tab + "sch_script_label": "脚本:", + "sch_interval_label": "每 (秒):", + "sch_repeat": "重复", + "sch_add": "新增", + "sch_remove_selected": "移除所选", + "sch_start": "启动调度器", + "sch_stop": "停止调度器", + "sch_status_running": "调度器运行中", + "sch_status_stopped": "调度器已停止", + "sch_col_job_id": "任务 ID", + "sch_col_script": "脚本", + "sch_col_interval": "间隔 (秒)", + "sch_col_runs": "执行次数", + "sch_col_enabled": "启用", + "sch_dialog_select_script": "选择脚本", + + # Socket / REST Tab + "ss_tcp_group": "TCP Socket 服务器", + "ss_rest_group": "REST API 服务器", + "ss_host_label": "主机:", + "ss_port_label": "端口:", + "ss_tcp_any_check": "绑定 TCP 至 0.0.0.0（暴露于网络）", + "ss_rest_any_check": "绑定 REST 至 0.0.0.0（暴露于网络）", + "ss_tcp_stopped": "TCP 已停止", + "ss_rest_stopped": "REST 已停止", + "ss_tcp_start": "启动 TCP", + "ss_tcp_stop": "停止 TCP", + "ss_rest_start": "启动 REST", + "ss_rest_stop": "停止 REST", + + # Live HUD Tab + "hud_watchers_group": "监看器", + "hud_start": "启动 HUD", + "hud_stop": "停止 HUD", + "hud_clear": "清除日志", + "hud_recent_log": "最近日志:", + "hud_mouse_prefix": "鼠标:", + "hud_pixel_prefix": "像素:", + + # Window Manager Tab + "win_refresh": "刷新", + "win_filter_placeholder": "以标题子字符串过滤", + "win_col_hwnd": "HWND", + "win_col_title": "标题", + "win_focus_selected": "聚焦所选", + "win_close_selected": "关闭所选", + "win_status_count": "{n} 个窗口", + + # Recording Editor Tab + "re_file_label": "文件:", + "re_browse": "浏览", + "re_load": "载入", + "re_save_as": "另存为", + "re_trim_start": "裁剪起点:", + "re_trim_end": "终点:", + "re_apply_trim": "应用裁剪", + "re_remove_selected": "移除所选", + "re_delay_x": "延迟倍数", + "re_floor_ms": "下限 ms:", + "re_apply_delays": "应用延迟", + "re_scale_x": "缩放 x:", + "re_scale_y": "y:", + "re_apply_scale": "应用缩放", + "re_keep_mouse": "仅保留鼠标", + "re_keep_keyboard": "仅保留键盘", + "re_preview": "预览:", + "re_dialog_open": "打开录制", + "re_dialog_save": "保存录制", + + # Script Builder Tab + "sb_delete": "删除", + "sb_up": "上移", + "sb_down": "下移", + "sb_load_json": "载入 JSON", + "sb_save_json": "保存 JSON", + "sb_run": "执行", + "sb_add_step": "新增步骤", + "sb_dialog_save": "保存脚本", + "sb_dialog_load": "载入脚本", + "sb_no_step_selected": "尚未选取步骤", + + # Language + "language_label": "语言:", + + # Run History Tab + "rh_filter_label": "来源:", + "rh_source_all": "全部", + "rh_source_scheduler": "调度器", + "rh_source_trigger": "触发器", + "rh_source_hotkey": "热键", + "rh_source_manual": "手动", + "rh_source_rest": "REST", + "rh_refresh": "刷新", + "rh_clear": "清除", + "rh_confirm_clear": "要删除全部执行记录吗？此操作无法撤销。", + "rh_count_label": "{n} 条记录", + "rh_col_id": "ID", + "rh_col_source": "来源", + "rh_col_target": "目标", + "rh_col_script": "脚本", + "rh_col_started": "开始时间", + "rh_col_duration": "耗时", + "rh_col_status": "状态", + "rh_status_ok": "成功", + "rh_status_error": "错误", + "rh_status_running": "执行中", + "rh_col_artifact": "截图", + "rh_open_artifact": "打开截图", + "rh_no_artifact": "所选记录没有截图。", + "rh_artifact_missing": "截图文件已不存在。", + + # Accessibility Tab + "a11y_app_label": "应用:", + "a11y_app_placeholder": "例如：计算器", + "a11y_name_label": "名称包含:", + "a11y_name_placeholder": "部分匹配", + "a11y_refresh": "刷新", + "a11y_click_selected": "点击所选", + "a11y_col_app": "应用", + "a11y_col_role": "角色", + "a11y_col_name": "名称", + "a11y_col_bounds": "边界", + "a11y_col_center": "中心", + "a11y_count_label": "{n} 个元素", + "a11y_no_selection": "请先选择一行", + "a11y_click_not_found": "找不到可点击的元素", + + # VLM (AI Locator) Tab + "vlm_desc_label": "描述：", + "vlm_desc_placeholder": "例如：绿色的提交按钮", + "vlm_model_label": "模型：", + "vlm_model_placeholder": "选填（例如 claude-opus-4-7）", + "vlm_locate": "定位", + "vlm_click": "定位并点击", + "vlm_result": "位置：({x}, {y})", + "vlm_ok": "完成", + "vlm_not_found": "找不到匹配元素", + "vlm_error": "错误", + "vlm_desc_required": "请先输入目标描述", # Menu bar "menu_file": "文件", @@ -33,10 +325,4 @@ "menu_language": "语言", "menu_help": "帮助", "menu_help_about": "关于 AutoControlGUI", - - # Status/messages - "language_label": "语言：", - "browse": "浏览", - "start": "开始", - "stop": "停止", } diff --git a/je_auto_control/gui/language_wrapper/traditional_chinese.py b/je_auto_control/gui/language_wrapper/traditional_chinese.py index e672be12..2fa9eb1c 100644 --- a/je_auto_control/gui/language_wrapper/traditional_chinese.py +++ b/je_auto_control/gui/language_wrapper/traditional_chinese.py @@ -20,6 +20,9 @@ "tab_screen_record": "螢幕錄影", "tab_shell": "Shell 命令", "tab_report": "報告產生", + "tab_run_history": "執行紀錄", + "tab_accessibility": "無障礙樹", + "tab_vlm": "AI 定位", # Auto Click Tab "interval_time": "間隔時間 (ms)：", @@ -49,7 +52,7 @@ "current_position": "目前位置：", # Screenshot Tab - "take_screenshot": "截取螢���", + "take_screenshot": "截取螢幕", "save_screenshot": "儲存截圖", "file_path_label": "檔案路徑：", "browse": "瀏覽", @@ -121,6 +124,195 @@ # Language "language_label": "語言：", + # Hotkeys Tab + "hk_combo_label": "組合鍵：", + "hk_script_label": "腳本：", + "hk_bind": "綁定", + "hk_remove_selected": "移除所選", + "hk_start_daemon": "啟動服務", + "hk_stop_daemon": "停止服務", + "hk_daemon_stopped": "服務已停止", + "hk_daemon_running": "服務運行中", + "hk_col_id": "ID", + "hk_col_combo": "組合鍵", + "hk_col_script": "腳本", + "hk_col_fired": "已觸發", + "hk_dialog_select_script": "選擇腳本", + + # Triggers Tab + "tr_script_label": "腳本：", + "tr_type_label": "類型：", + "tr_repeat": "重複", + "tr_add": "新增觸發器", + "tr_remove_selected": "移除所選", + "tr_start_engine": "啟動引擎", + "tr_stop_engine": "停止引擎", + "tr_engine_stopped": "引擎已停止", + "tr_engine_running": "引擎運行中", + "tr_type_image": "影像出現", + "tr_type_window": "視窗出現", + "tr_type_pixel": "像素符合", + "tr_type_file": "檔案變更", + "tr_image_label": "影像：", + "tr_threshold_label": "精確度：", + "tr_title_contains_label": "標題包含：", + "tr_watch_label": "監看路徑：", + "tr_col_id": "ID", + "tr_col_type": "類型", + "tr_col_detail": "細節", + "tr_col_fired": "已觸發", + "tr_col_enabled": "啟用", + "tr_yes": "是", + "tr_no": "否", + "tr_dialog_select_script": "選擇腳本", + "tr_dialog_select_image": "選擇影像", + "tr_dialog_select_file": "選擇要監看的檔案", + + # Plugins Tab + "pl_dir_label": "外掛目錄：", + "pl_load": "載入並註冊", + "pl_no_loaded": "尚未載入外掛", + "pl_registered_label": "已註冊指令：", + "pl_dialog_plugin_dir": "外掛目錄", + + # Scheduler Tab + "sch_script_label": "腳本：", + "sch_interval_label": "每 (秒)：", + "sch_repeat": "重複", + "sch_add": "新增", + "sch_remove_selected": "移除所選", + "sch_start": "啟動排程器", + "sch_stop": "停止排程器", + "sch_status_running": "排程器運行中", + "sch_status_stopped": "排程器已停止", + "sch_col_job_id": "任務 ID", + "sch_col_script": "腳本", + "sch_col_interval": "間隔 (秒)", + "sch_col_runs": "執行次數", + "sch_col_enabled": "啟用", + "sch_dialog_select_script": "選擇腳本", + + # Socket / REST Tab + "ss_tcp_group": "TCP Socket 伺服器", + "ss_rest_group": "REST API 伺服器", + "ss_host_label": "主機：", + "ss_port_label": "埠號：", + "ss_tcp_any_check": "綁定 TCP 至 0.0.0.0（暴露於網路）", + "ss_rest_any_check": "綁定 REST 至 0.0.0.0（暴露於網路）", + "ss_tcp_stopped": "TCP 已停止", + "ss_rest_stopped": "REST 已停止", + "ss_tcp_start": "啟動 TCP", + "ss_tcp_stop": "停止 TCP", + "ss_rest_start": "啟動 REST", + "ss_rest_stop": "停止 REST", + + # Live HUD Tab + "hud_watchers_group": "監看器", + "hud_start": "啟動 HUD", + "hud_stop": "停止 HUD", + "hud_clear": "清除日誌", + "hud_recent_log": "最近日誌：", + "hud_mouse_prefix": "滑鼠：", + "hud_pixel_prefix": "像素：", + + # Window Manager Tab + "win_refresh": "重新整理", + "win_filter_placeholder": "以標題子字串過濾", + "win_col_hwnd": "HWND", + "win_col_title": "標題", + "win_focus_selected": "聚焦所選", + "win_close_selected": "關閉所選", + "win_status_count": "{n} 個視窗", + + # Recording Editor Tab + "re_file_label": "檔案：", + "re_browse": "瀏覽", + "re_load": "載入", + "re_save_as": "另存為", + "re_trim_start": "裁剪起點：", + "re_trim_end": "終點：", + "re_apply_trim": "套用裁剪", + "re_remove_selected": "移除所選", + "re_delay_x": "延遲倍數", + "re_floor_ms": "下限 ms：", + "re_apply_delays": "套用延遲", + "re_scale_x": "縮放 x：", + "re_scale_y": "y：", + "re_apply_scale": "套用縮放", + "re_keep_mouse": "僅保留滑鼠", + "re_keep_keyboard": "僅保留鍵盤", + "re_preview": "預覽：", + "re_dialog_open": "開啟錄製", + "re_dialog_save": "儲存錄製", + + # Script Builder Tab + "sb_delete": "刪除", + "sb_up": "上移", + "sb_down": "下移", + "sb_load_json": "載入 JSON", + "sb_save_json": "儲存 JSON", + "sb_run": "執行", + "sb_add_step": "新增步驟", + "sb_dialog_save": "儲存腳本", + "sb_dialog_load": "載入腳本", + "sb_no_step_selected": "尚未選取步驟", + + # Run History Tab + "rh_filter_label": "來源：", + "rh_source_all": "全部", + "rh_source_scheduler": "排程器", + "rh_source_trigger": "觸發器", + "rh_source_hotkey": "熱鍵", + "rh_source_manual": "手動", + "rh_source_rest": "REST", + "rh_refresh": "重新整理", + "rh_clear": "清除", + "rh_confirm_clear": "要刪除全部執行紀錄嗎？此動作無法復原。", + "rh_count_label": "{n} 筆紀錄", + "rh_col_id": "ID", + "rh_col_source": "來源", + "rh_col_target": "目標", + "rh_col_script": "腳本", + "rh_col_started": "開始時間", + "rh_col_duration": "耗時", + "rh_col_status": "狀態", + "rh_status_ok": "成功", + "rh_status_error": "錯誤", + "rh_status_running": "執行中", + "rh_col_artifact": "截圖", + "rh_open_artifact": "開啟截圖", + "rh_no_artifact": "所選紀錄沒有截圖。", + "rh_artifact_missing": "截圖檔案已不存在。", + + # Accessibility Tab + "a11y_app_label": "應用程式：", + "a11y_app_placeholder": "例如：小算盤", + "a11y_name_label": "名稱包含：", + "a11y_name_placeholder": "部分比對", + "a11y_refresh": "重新整理", + "a11y_click_selected": "點擊所選", + "a11y_col_app": "應用程式", + "a11y_col_role": "角色", + "a11y_col_name": "名稱", + "a11y_col_bounds": "邊界", + "a11y_col_center": "中心", + "a11y_count_label": "{n} 個元素", + "a11y_no_selection": "請先選擇一列", + "a11y_click_not_found": "找不到可點擊的元素", + + # VLM (AI Locator) Tab + "vlm_desc_label": "描述：", + "vlm_desc_placeholder": "例如：綠色的送出按鈕", + "vlm_model_label": "模型：", + "vlm_model_placeholder": "選填（例如 claude-opus-4-7）", + "vlm_locate": "定位", + "vlm_click": "定位並點擊", + "vlm_result": "位置：({x}, {y})", + "vlm_ok": "完成", + "vlm_not_found": "找不到符合的元素", + "vlm_error": "錯誤", + "vlm_desc_required": "請先輸入目標描述", + # Menu bar "menu_file": "檔案", "menu_file_open_script": "開啟腳本...", diff --git a/je_auto_control/gui/live_hud_tab.py b/je_auto_control/gui/live_hud_tab.py index fb1d71a8..95649a22 100644 --- a/je_auto_control/gui/live_hud_tab.py +++ b/je_auto_control/gui/live_hud_tab.py @@ -6,20 +6,32 @@ QGroupBox, QHBoxLayout, QLabel, QPushButton, QTextEdit, QVBoxLayout, QWidget, ) +from je_auto_control.gui._i18n_helpers import TranslatableMixin +from je_auto_control.gui.language_wrapper.multi_language_wrapper import ( + language_wrapper, +) from je_auto_control.utils.logging.logging_instance import autocontrol_logger from je_auto_control.utils.watcher.watcher import LogTail, MouseWatcher, PixelWatcher -class LiveHUDTab(QWidget): +def _t(key: str) -> str: + return language_wrapper.translate(key, key) + + +class LiveHUDTab(TranslatableMixin, QWidget): """Poll watchers and render a readable HUD.""" def __init__(self, parent: Optional[QWidget] = None) -> None: super().__init__(parent) + self._tr_init() self._mouse = MouseWatcher() self._pixel = PixelWatcher() self._log_tail = LogTail(capacity=400) - self._pos_label = QLabel("Mouse: --") - self._color_label = QLabel("Pixel: --") + self._pos_suffix = " --" + self._color_suffix = " --" + self._pos_label = QLabel() + self._color_label = QLabel() + self._apply_position_labels() self._log_view = QTextEdit() self._log_view.setReadOnly(True) self._timer = QTimer(self) @@ -27,9 +39,17 @@ def __init__(self, parent: Optional[QWidget] = None) -> None: self._timer.timeout.connect(self._tick) self._build_layout() + def _apply_position_labels(self) -> None: + self._pos_label.setText(_t("hud_mouse_prefix") + self._pos_suffix) + self._color_label.setText(_t("hud_pixel_prefix") + self._color_suffix) + + def retranslate(self) -> None: + TranslatableMixin.retranslate(self) + self._apply_position_labels() + def _build_layout(self) -> None: root = QVBoxLayout(self) - status_group = QGroupBox("Watchers") + status_group = self._tr(QGroupBox(), "hud_watchers_group") status_layout = QVBoxLayout() status_layout.addWidget(self._pos_label) status_layout.addWidget(self._color_label) @@ -37,18 +57,18 @@ def _build_layout(self) -> None: root.addWidget(status_group) ctl = QHBoxLayout() - start_btn = QPushButton("Start HUD") + start_btn = self._tr(QPushButton(), "hud_start") start_btn.clicked.connect(self._start) - stop_btn = QPushButton("Stop HUD") + stop_btn = self._tr(QPushButton(), "hud_stop") stop_btn.clicked.connect(self._stop) - clear_btn = QPushButton("Clear log") + clear_btn = self._tr(QPushButton(), "hud_clear") clear_btn.clicked.connect(self._log_view.clear) for btn in (start_btn, stop_btn, clear_btn): ctl.addWidget(btn) ctl.addStretch() root.addLayout(ctl) - root.addWidget(QLabel("Recent log:")) + root.addWidget(self._tr(QLabel(), "hud_recent_log")) root.addWidget(self._log_view, stretch=1) def _start(self) -> None: @@ -63,11 +83,13 @@ def _tick(self) -> None: try: x, y = self._mouse.sample() except RuntimeError as error: - self._pos_label.setText(f"Mouse: {error}") + self._pos_suffix = f" {error}" + self._apply_position_labels() return - self._pos_label.setText(f"Mouse: ({x}, {y})") + self._pos_suffix = f" ({x}, {y})" rgb = self._pixel.sample(x, y) - self._color_label.setText(f"Pixel: {rgb}" if rgb is not None else "Pixel: n/a") + self._color_suffix = f" {rgb}" if rgb is not None else " n/a" + self._apply_position_labels() lines = self._log_tail.snapshot() self._log_view.setPlainText("\n".join(lines)) scrollbar = self._log_view.verticalScrollBar() diff --git a/je_auto_control/gui/main_widget.py b/je_auto_control/gui/main_widget.py index 05d48477..dca598cb 100644 --- a/je_auto_control/gui/main_widget.py +++ b/je_auto_control/gui/main_widget.py @@ -10,17 +10,20 @@ ) from je_auto_control.gui._auto_click_tab import AutoClickTabMixin -from je_auto_control.gui._shell_report_tabs import ShellReportTabsMixin +from je_auto_control.gui._i18n_helpers import TranslatableMixin +from je_auto_control.gui.accessibility_tab import AccessibilityTab +from je_auto_control.gui._report_tab import ReportTabMixin from je_auto_control.gui.hotkeys_tab import HotkeysTab from je_auto_control.gui.language_wrapper.multi_language_wrapper import language_wrapper from je_auto_control.gui.live_hud_tab import LiveHUDTab from je_auto_control.gui.plugins_tab import PluginsTab from je_auto_control.gui.recording_editor_tab import RecordingEditorTab +from je_auto_control.gui.run_history_tab import RunHistoryTab from je_auto_control.gui.scheduler_tab import SchedulerTab from je_auto_control.gui.script_builder import ScriptBuilderTab from je_auto_control.gui.selector import crop_template_to_file, open_region_selector -from je_auto_control.gui.socket_server_tab import SocketServerTab from je_auto_control.gui.triggers_tab import TriggersTab +from je_auto_control.gui.vlm_tab import VLMTab from je_auto_control.gui.window_tab import WindowManagerTab from je_auto_control.wrapper.auto_control_screen import screen_size, screenshot, get_pixel from je_auto_control.wrapper.auto_control_image import locate_all_image, locate_image_center, locate_and_click @@ -28,7 +31,9 @@ from je_auto_control.utils.executor.action_executor import execute_action, execute_files from je_auto_control.utils.json.json_file import read_action_json, write_action_json from je_auto_control.utils.file_process.get_dir_file_list import get_dir_files_as_list -from je_auto_control.utils.cv2_utils.screen_record import ScreenRecorder + + +_JSON_FILE_FILTER = "JSON (*.json)" def _t(key: str) -> str: @@ -51,13 +56,16 @@ class _TabEntry: # ============================================================================= # Main Widget # ============================================================================= -class AutoControlGUIWidget(AutoClickTabMixin, ShellReportTabsMixin, QWidget): +class AutoControlGUIWidget( + TranslatableMixin, AutoClickTabMixin, ReportTabMixin, QWidget, +): """Owns the QTabWidget and exposes show/hide/list APIs for the menu bar.""" tabs_changed = Signal() def __init__(self, parent=None): super().__init__(parent) + self._tr_init() layout = QVBoxLayout() self._tab_entries: list = [] @@ -75,13 +83,13 @@ def __init__(self, parent=None): self._add_tab("recording_editor", "tab_recording_editor", RecordingEditorTab()) self._add_tab("window_manager", "tab_window_manager", WindowManagerTab()) self._add_tab("scheduler", "tab_scheduler", SchedulerTab()) - self._add_tab("socket_server", "tab_socket_server", SocketServerTab()) self._add_tab("live_hud", "tab_live_hud", LiveHUDTab()) - self._add_tab("screen_record", "tab_screen_record", self._build_screen_record_tab()) - self._add_tab("shell", "tab_shell", self._build_shell_tab()) self._add_tab("report", "tab_report", self._build_report_tab()) self._add_tab("hotkeys", "tab_hotkeys", HotkeysTab()) self._add_tab("triggers", "tab_triggers", TriggersTab()) + self._add_tab("run_history", "tab_run_history", RunHistoryTab()) + self._add_tab("accessibility", "tab_accessibility", AccessibilityTab()) + self._add_tab("vlm", "tab_vlm", VLMTab()) self._add_tab("plugins", "tab_plugins", PluginsTab()) layout.addWidget(self.tabs) @@ -90,7 +98,6 @@ def __init__(self, parent=None): self.timer = QTimer() self.repeat_count = 0 self.repeat_max = 0 - self.screen_recorder = ScreenRecorder() self._record_data = [] # --- tab registry API ---------------------------------------------------- @@ -147,14 +154,34 @@ def _on_tab_close_requested(self, index: int) -> None: self.hide_tab(entry.key) return + def _translate(self, key: str) -> str: + return language_wrapper.translate(key, key) + def retranslate(self) -> None: - """Relabel visible tabs after a language change.""" + """Relabel tab titles and propagate into every child tab.""" for entry in self._tab_entries: index = self.tabs.indexOf(entry.widget) if index != -1: self.tabs.setTabText( index, language_wrapper.translate(entry.title_key, entry.title_key), ) + # Widgets registered via TranslatableMixin on this widget (screenshot, + # image-detect, record, script, screen-record, shell, report tabs). + TranslatableMixin.retranslate(self) + if hasattr(self, "_auto_click_retranslate"): + self._auto_click_retranslate() + if hasattr(self, "_screenshot_retranslate"): + self._screenshot_retranslate() + if hasattr(self, "_record_retranslate"): + self._record_retranslate() + # Child class tabs get their own retranslate if they implement one. + for entry in self._tab_entries: + callback = getattr(entry.widget, "retranslate", None) + if callable(callback) and entry.widget is not self: + try: + callback() + except (RuntimeError, AttributeError): + continue def open_script_file(self, path: str) -> None: """Load a JSON script into the Script Executor tab and focus it.""" @@ -179,10 +206,10 @@ def _build_screenshot_tab(self) -> QWidget: layout = QVBoxLayout() # Screen size - size_group = QGroupBox(_t("screen_size_label")) + size_group = self._tr(QGroupBox(), "screen_size_label") sg = QHBoxLayout() self.screen_size_label = QLabel("--") - self.screen_size_btn = QPushButton(_t("get_screen_size")) + self.screen_size_btn = self._tr(QPushButton(), "get_screen_size") self.screen_size_btn.clicked.connect(self._get_screen_size) sg.addWidget(self.screen_size_label) sg.addWidget(self.screen_size_btn) @@ -190,25 +217,25 @@ def _build_screenshot_tab(self) -> QWidget: layout.addWidget(size_group) # Screenshot - ss_group = QGroupBox(_t("take_screenshot")) + ss_group = self._tr(QGroupBox(), "take_screenshot") ss_grid = QGridLayout() - ss_grid.addWidget(QLabel(_t("file_path_label")), 0, 0) + ss_grid.addWidget(self._tr(QLabel(), "file_path_label"), 0, 0) self.ss_path_input = QLineEdit() ss_grid.addWidget(self.ss_path_input, 0, 1) - self.ss_browse_btn = QPushButton(_t("browse")) + self.ss_browse_btn = self._tr(QPushButton(), "browse") self.ss_browse_btn.clicked.connect(self._browse_ss_path) ss_grid.addWidget(self.ss_browse_btn, 0, 2) - ss_grid.addWidget(QLabel(_t("region_label")), 1, 0) + ss_grid.addWidget(self._tr(QLabel(), "region_label"), 1, 0) self.ss_region_input = QLineEdit() self.ss_region_input.setPlaceholderText("0, 0, 800, 600") ss_grid.addWidget(self.ss_region_input, 1, 1) - self.ss_pick_region_btn = QPushButton(_t("pick_region")) + self.ss_pick_region_btn = self._tr(QPushButton(), "pick_region") self.ss_pick_region_btn.clicked.connect(self._pick_ss_region) ss_grid.addWidget(self.ss_pick_region_btn, 1, 2) btn_h = QHBoxLayout() - self.ss_take_btn = QPushButton(_t("take_screenshot")) + self.ss_take_btn = self._tr(QPushButton(), "take_screenshot") self.ss_take_btn.clicked.connect(self._take_screenshot) btn_h.addWidget(self.ss_take_btn) ss_grid.addLayout(btn_h, 2, 0, 1, 3) @@ -216,20 +243,24 @@ def _build_screenshot_tab(self) -> QWidget: layout.addWidget(ss_group) # Get pixel - px_group = QGroupBox(_t("get_pixel_label")) + px_group = self._tr(QGroupBox(), "get_pixel_label") px_grid = QGridLayout() - px_grid.addWidget(QLabel(_t("pixel_x")), 0, 0) + px_grid.addWidget(self._tr(QLabel(), "pixel_x"), 0, 0) self.pixel_x_input = QLineEdit("0") self.pixel_x_input.setValidator(QIntValidator()) px_grid.addWidget(self.pixel_x_input, 0, 1) - px_grid.addWidget(QLabel(_t("pixel_y")), 0, 2) + px_grid.addWidget(self._tr(QLabel(), "pixel_y"), 0, 2) self.pixel_y_input = QLineEdit("0") self.pixel_y_input.setValidator(QIntValidator()) px_grid.addWidget(self.pixel_y_input, 0, 3) - self.pixel_btn = QPushButton(_t("get_pixel_label")) + self.pixel_btn = self._tr(QPushButton(), "get_pixel_label") self.pixel_btn.clicked.connect(self._get_pixel_color) px_grid.addWidget(self.pixel_btn, 1, 0, 1, 2) - self.pixel_result_label = QLabel(_t("pixel_result") + " --") + self.pixel_result_label = QLabel() + self._pixel_result_suffix = " --" + self.pixel_result_label.setText( + self._translate("pixel_result") + self._pixel_result_suffix, + ) px_grid.addWidget(self.pixel_result_label, 1, 2, 1, 2) px_group.setLayout(px_grid) layout.addWidget(px_group) @@ -278,10 +309,19 @@ def _get_pixel_color(self): x = int(self.pixel_x_input.text()) y = int(self.pixel_y_input.text()) color = get_pixel(x, y) - self.pixel_result_label.setText(_t("pixel_result") + f" {color}") + self._pixel_result_suffix = f" {color}" + self.pixel_result_label.setText( + self._translate("pixel_result") + self._pixel_result_suffix, + ) except (OSError, ValueError, TypeError, RuntimeError) as error: self.pixel_result_label.setText(f"Error: {error}") + def _screenshot_retranslate(self) -> None: + if hasattr(self, "pixel_result_label"): + self.pixel_result_label.setText( + self._translate("pixel_result") + self._pixel_result_suffix, + ) + # ========================================================================= # Tab 3: Image Detection # ========================================================================= @@ -290,38 +330,38 @@ def _build_image_detect_tab(self) -> QWidget: layout = QVBoxLayout() grid = QGridLayout() - grid.addWidget(QLabel(_t("template_image")), 0, 0) + grid.addWidget(self._tr(QLabel(), "template_image"), 0, 0) self.img_path_input = QLineEdit() grid.addWidget(self.img_path_input, 0, 1) - self.img_browse_btn = QPushButton(_t("browse")) + self.img_browse_btn = self._tr(QPushButton(), "browse") self.img_browse_btn.clicked.connect(self._browse_img) grid.addWidget(self.img_browse_btn, 0, 2) - self.img_crop_btn = QPushButton(_t("crop_template")) + self.img_crop_btn = self._tr(QPushButton(), "crop_template") self.img_crop_btn.clicked.connect(self._crop_template) grid.addWidget(self.img_crop_btn, 0, 3) - grid.addWidget(QLabel(_t("threshold_label")), 1, 0) + grid.addWidget(self._tr(QLabel(), "threshold_label"), 1, 0) self.threshold_input = QLineEdit("0.8") self.threshold_input.setValidator(QDoubleValidator(0.0, 1.0, 2)) grid.addWidget(self.threshold_input, 1, 1) - self.draw_check = QCheckBox(_t("draw_image_check")) + self.draw_check = self._tr(QCheckBox(), "draw_image_check") grid.addWidget(self.draw_check, 1, 2) layout.addLayout(grid) btn_h = QHBoxLayout() - self.locate_btn = QPushButton(_t("locate_image")) + self.locate_btn = self._tr(QPushButton(), "locate_image") self.locate_btn.clicked.connect(self._locate_image) - self.locate_all_btn = QPushButton(_t("locate_all")) + self.locate_all_btn = self._tr(QPushButton(), "locate_all") self.locate_all_btn.clicked.connect(self._locate_all) - self.locate_click_btn = QPushButton(_t("locate_click")) + self.locate_click_btn = self._tr(QPushButton(), "locate_click") self.locate_click_btn.clicked.connect(self._locate_click) btn_h.addWidget(self.locate_btn) btn_h.addWidget(self.locate_all_btn) btn_h.addWidget(self.locate_click_btn) layout.addLayout(btn_h) - layout.addWidget(QLabel(_t("detection_result"))) + layout.addWidget(self._tr(QLabel(), "detection_result")) self.detect_result_text = QTextEdit() self.detect_result_text.setReadOnly(True) layout.addWidget(self.detect_result_text) @@ -388,15 +428,17 @@ def _build_record_tab(self) -> QWidget: tab = QWidget() layout = QVBoxLayout() - self.record_status_label = QLabel(_t("record_status") + " " + _t("record_idle")) + self._record_status_key = "record_idle" + self.record_status_label = QLabel() + self._apply_record_status_label() layout.addWidget(self.record_status_label) btn_h = QHBoxLayout() - self.rec_start_btn = QPushButton(_t("start_record")) + self.rec_start_btn = self._tr(QPushButton(), "start_record") self.rec_start_btn.clicked.connect(self._start_record) - self.rec_stop_btn = QPushButton(_t("stop_record")) + self.rec_stop_btn = self._tr(QPushButton(), "stop_record") self.rec_stop_btn.clicked.connect(self._stop_record) - self.rec_play_btn = QPushButton(_t("playback")) + self.rec_play_btn = self._tr(QPushButton(), "playback") self.rec_play_btn.clicked.connect(self._playback_record) btn_h.addWidget(self.rec_start_btn) btn_h.addWidget(self.rec_stop_btn) @@ -404,32 +446,44 @@ def _build_record_tab(self) -> QWidget: layout.addLayout(btn_h) btn_h2 = QHBoxLayout() - self.rec_save_btn = QPushButton(_t("save_record")) + self.rec_save_btn = self._tr(QPushButton(), "save_record") self.rec_save_btn.clicked.connect(self._save_record) - self.rec_load_btn = QPushButton(_t("load_record")) + self.rec_load_btn = self._tr(QPushButton(), "load_record") self.rec_load_btn.clicked.connect(self._load_record) btn_h2.addWidget(self.rec_save_btn) btn_h2.addWidget(self.rec_load_btn) layout.addLayout(btn_h2) - layout.addWidget(QLabel(_t("record_list_label"))) + layout.addWidget(self._tr(QLabel(), "record_list_label")) self.record_list_text = QTextEdit() self.record_list_text.setReadOnly(True) layout.addWidget(self.record_list_text) tab.setLayout(layout) return tab + def _apply_record_status_label(self) -> None: + if hasattr(self, "record_status_label"): + self.record_status_label.setText( + self._translate("record_status") + " " + + self._translate(self._record_status_key), + ) + + def _record_retranslate(self) -> None: + self._apply_record_status_label() + def _start_record(self): try: record() - self.record_status_label.setText(_t("record_status") + " " + _t("record_recording")) + self._record_status_key = "record_recording" + self._apply_record_status_label() except (OSError, ValueError, TypeError, RuntimeError) as error: QMessageBox.warning(self, "Error", str(error)) def _stop_record(self): try: self._record_data = stop_record() or [] - self.record_status_label.setText(_t("record_status") + " " + _t("record_idle")) + self._record_status_key = "record_idle" + self._apply_record_status_label() self.record_list_text.setText(json.dumps(self._record_data, indent=2, ensure_ascii=False)) except (OSError, ValueError, TypeError, RuntimeError) as error: QMessageBox.warning(self, "Error", str(error)) @@ -448,7 +502,7 @@ def _save_record(self): if not self._record_data: QMessageBox.warning(self, "Warning", "No recorded data") return - path, _ = QFileDialog.getSaveFileName(self, _t("save_record"), "", "JSON (*.json)") + path, _ = QFileDialog.getSaveFileName(self, _t("save_record"), "", _JSON_FILE_FILTER) if path: write_action_json(path, self._record_data) except (OSError, ValueError, TypeError, RuntimeError) as error: @@ -456,7 +510,7 @@ def _save_record(self): def _load_record(self): try: - path, _ = QFileDialog.getOpenFileName(self, _t("load_record"), "", "JSON (*.json)") + path, _ = QFileDialog.getOpenFileName(self, _t("load_record"), "", _JSON_FILE_FILTER) if path: self._record_data = read_action_json(path) self.record_list_text.setText(json.dumps(self._record_data, indent=2, ensure_ascii=False)) @@ -470,41 +524,38 @@ def _build_script_tab(self) -> QWidget: tab = QWidget() layout = QVBoxLayout() - # Load / execute single file file_h = QHBoxLayout() self.script_path_input = QLineEdit() - self.script_browse_btn = QPushButton(_t("load_script")) + self.script_browse_btn = self._tr(QPushButton(), "load_script") self.script_browse_btn.clicked.connect(self._browse_script) - self.script_exec_btn = QPushButton(_t("execute_script")) + self.script_exec_btn = self._tr(QPushButton(), "execute_script") self.script_exec_btn.clicked.connect(self._execute_script) file_h.addWidget(self.script_path_input) file_h.addWidget(self.script_browse_btn) file_h.addWidget(self.script_exec_btn) layout.addLayout(file_h) - # Execute directory dir_h = QHBoxLayout() self.script_dir_input = QLineEdit() - self.script_dir_browse_btn = QPushButton(_t("execute_dir_label")) + self.script_dir_browse_btn = self._tr(QPushButton(), "execute_dir_label") self.script_dir_browse_btn.clicked.connect(self._browse_script_dir) - self.script_dir_exec_btn = QPushButton(_t("execute_dir")) + self.script_dir_exec_btn = self._tr(QPushButton(), "execute_dir") self.script_dir_exec_btn.clicked.connect(self._execute_dir) dir_h.addWidget(self.script_dir_input) dir_h.addWidget(self.script_dir_browse_btn) dir_h.addWidget(self.script_dir_exec_btn) layout.addLayout(dir_h) - # Manual JSON input - layout.addWidget(QLabel(_t("script_content"))) + layout.addWidget(self._tr(QLabel(), "script_content")) self.script_editor = QTextEdit() self.script_editor.setPlaceholderText('[["AC_type_keyboard", {"keycode": "a"}]]') layout.addWidget(self.script_editor) - exec_btn = QPushButton(_t("execute_script")) + exec_btn = self._tr(QPushButton(), "execute_script") exec_btn.clicked.connect(self._execute_manual_script) layout.addWidget(exec_btn) - layout.addWidget(QLabel(_t("execution_result"))) + layout.addWidget(self._tr(QLabel(), "execution_result")) self.script_result_text = QTextEdit() self.script_result_text.setReadOnly(True) layout.addWidget(self.script_result_text) @@ -512,7 +563,7 @@ def _build_script_tab(self) -> QWidget: return tab def _browse_script(self): - path, _ = QFileDialog.getOpenFileName(self, _t("load_script"), "", "JSON (*.json)") + path, _ = QFileDialog.getOpenFileName(self, _t("load_script"), "", _JSON_FILE_FILTER) if path: self.script_path_input.setText(path) try: @@ -559,89 +610,6 @@ def _execute_manual_script(self): except (OSError, ValueError, TypeError, RuntimeError) as error: self.script_result_text.setText(f"Error: {error}") - # ========================================================================= - # Tab 6: Screen Recording - # ========================================================================= - def _build_screen_record_tab(self) -> QWidget: - tab = QWidget() - layout = QVBoxLayout() - - grid = QGridLayout() - row = 0 - - grid.addWidget(QLabel(_t("recorder_name")), row, 0) - self.sr_name_input = QLineEdit("default") - grid.addWidget(self.sr_name_input, row, 1) - - row += 1 - grid.addWidget(QLabel(_t("output_file")), row, 0) - self.sr_file_input = QLineEdit("output.avi") - grid.addWidget(self.sr_file_input, row, 1) - self.sr_file_browse_btn = QPushButton(_t("browse")) - self.sr_file_browse_btn.clicked.connect(self._browse_sr_file) - grid.addWidget(self.sr_file_browse_btn, row, 2) - - row += 1 - grid.addWidget(QLabel(_t("codec_label")), row, 0) - self.sr_codec_input = QLineEdit("XVID") - grid.addWidget(self.sr_codec_input, row, 1) - - row += 1 - grid.addWidget(QLabel(_t("fps_label")), row, 0) - self.sr_fps_input = QLineEdit("30") - self.sr_fps_input.setValidator(QIntValidator(1, 120)) - grid.addWidget(self.sr_fps_input, row, 1) - - row += 1 - grid.addWidget(QLabel(_t("resolution_label")), row, 0) - self.sr_res_input = QLineEdit("1920x1080") - grid.addWidget(self.sr_res_input, row, 1) - - layout.addLayout(grid) - - btn_h = QHBoxLayout() - self.sr_start_btn = QPushButton(_t("start_screen_record")) - self.sr_start_btn.clicked.connect(self._start_screen_record) - self.sr_stop_btn = QPushButton(_t("stop_screen_record")) - self.sr_stop_btn.clicked.connect(self._stop_screen_record) - btn_h.addWidget(self.sr_start_btn) - btn_h.addWidget(self.sr_stop_btn) - layout.addLayout(btn_h) - - self.sr_status_label = QLabel(_t("screen_record_status") + " " + _t("record_idle")) - layout.addWidget(self.sr_status_label) - - layout.addStretch() - tab.setLayout(layout) - return tab - - def _browse_sr_file(self): - path, _ = QFileDialog.getSaveFileName(self, _t("output_file"), "", "AVI (*.avi);;MP4 (*.mp4);;All (*)") - if path: - self.sr_file_input.setText(path) - - def _start_screen_record(self): - try: - name = self.sr_name_input.text() or "default" - output = self.sr_file_input.text() or "output.avi" - codec = self.sr_codec_input.text() or "XVID" - fps = int(self.sr_fps_input.text() or "30") - res_text = self.sr_res_input.text() or "1920x1080" - w, h = res_text.lower().split("x") - resolution = (int(w), int(h)) - self.screen_recorder.start_new_record(name, output, codec, fps, resolution) - self.sr_status_label.setText(_t("screen_record_status") + " " + _t("record_recording")) - except (OSError, ValueError, TypeError, RuntimeError) as error: - QMessageBox.warning(self, "Error", str(error)) - - def _stop_screen_record(self): - try: - name = self.sr_name_input.text() or "default" - self.screen_recorder.stop_record(name) - self.sr_status_label.setText(_t("screen_record_status") + " " + _t("record_idle")) - except (OSError, ValueError, TypeError, RuntimeError) as error: - QMessageBox.warning(self, "Error", str(error)) - # ========================================================================= # Global keyboard shortcut: Ctrl+4 to stop # ========================================================================= diff --git a/je_auto_control/gui/plugins_tab.py b/je_auto_control/gui/plugins_tab.py index 0a89c340..dd4f0a57 100644 --- a/je_auto_control/gui/plugins_tab.py +++ b/je_auto_control/gui/plugins_tab.py @@ -6,39 +6,55 @@ QPushButton, QVBoxLayout, QWidget, ) +from je_auto_control.gui._i18n_helpers import TranslatableMixin +from je_auto_control.gui.language_wrapper.multi_language_wrapper import ( + language_wrapper, +) from je_auto_control.utils.plugin_loader.plugin_loader import ( load_plugin_directory, register_plugin_commands, ) -class PluginsTab(QWidget): +def _t(key: str) -> str: + return language_wrapper.translate(key, key) + + +class PluginsTab(TranslatableMixin, QWidget): """Pick a directory of plugins, register their ``AC_*`` callables.""" def __init__(self, parent: Optional[QWidget] = None) -> None: super().__init__(parent) + self._tr_init() self._dir_input = QLineEdit() self._list = QListWidget() - self._status = QLabel("No plugins loaded") + self._status_text = _t("pl_no_loaded") + self._status = QLabel(self._status_text) + self._status_is_translatable = True self._build_layout() def _build_layout(self) -> None: root = QVBoxLayout(self) form = QHBoxLayout() - form.addWidget(QLabel("Plugin dir:")) + form.addWidget(self._tr(QLabel(), "pl_dir_label")) form.addWidget(self._dir_input, stretch=1) - browse = QPushButton("Browse") + browse = self._tr(QPushButton(), "browse") browse.clicked.connect(self._browse) form.addWidget(browse) - load = QPushButton("Load + register") + load = self._tr(QPushButton(), "pl_load") load.clicked.connect(self._on_load) form.addWidget(load) root.addLayout(form) - root.addWidget(QLabel("Registered commands:")) + root.addWidget(self._tr(QLabel(), "pl_registered_label")) root.addWidget(self._list, stretch=1) root.addWidget(self._status) + def retranslate(self) -> None: + TranslatableMixin.retranslate(self) + if self._status_is_translatable: + self._status.setText(_t("pl_no_loaded")) + def _browse(self) -> None: - path = QFileDialog.getExistingDirectory(self, "Plugin directory") + path = QFileDialog.getExistingDirectory(self, _t("pl_dialog_plugin_dir")) if path: self._dir_input.setText(path) @@ -48,14 +64,16 @@ def _on_load(self) -> None: return try: commands = load_plugin_directory(path) - except (OSError, NotADirectoryError) as error: + except OSError as error: QMessageBox.warning(self, "Error", str(error)) return if not commands: self._status.setText(f"No AC_* callables found in {path}") + self._status_is_translatable = False return registered = register_plugin_commands(commands) self._list.clear() for name in registered: self._list.addItem(name) self._status.setText(f"Registered {len(registered)} commands from {path}") + self._status_is_translatable = False diff --git a/je_auto_control/gui/recording_editor_tab.py b/je_auto_control/gui/recording_editor_tab.py index 599d8988..339deaf8 100644 --- a/je_auto_control/gui/recording_editor_tab.py +++ b/je_auto_control/gui/recording_editor_tab.py @@ -7,17 +7,26 @@ QPushButton, QTextEdit, QVBoxLayout, QWidget, ) +from je_auto_control.gui._i18n_helpers import TranslatableMixin +from je_auto_control.gui.language_wrapper.multi_language_wrapper import ( + language_wrapper, +) from je_auto_control.utils.json.json_file import read_action_json, write_action_json from je_auto_control.utils.recording_edit.editor import ( adjust_delays, filter_actions, remove_action, scale_coordinates, trim_actions, ) -class RecordingEditorTab(QWidget): +def _t(key: str) -> str: + return language_wrapper.translate(key, key) + + +class RecordingEditorTab(TranslatableMixin, QWidget): """Load a recording JSON and apply non-destructive edits.""" def __init__(self, parent: Optional[QWidget] = None) -> None: super().__init__(parent) + self._tr_init() self._actions: list = [] self._path_input = QLineEdit() self._list = QListWidget() @@ -32,17 +41,20 @@ def __init__(self, parent: Optional[QWidget] = None) -> None: self._scale_y = QLineEdit("1.0") self._build_layout() + def retranslate(self) -> None: + TranslatableMixin.retranslate(self) + def _build_layout(self) -> None: root = QVBoxLayout(self) top = QHBoxLayout() - top.addWidget(QLabel("File:")) + top.addWidget(self._tr(QLabel(), "re_file_label")) top.addWidget(self._path_input, stretch=1) - for label, handler in ( - ("Browse", self._browse), - ("Load", self._load), - ("Save As", self._save_as), + for key, handler in ( + ("re_browse", self._browse), + ("re_load", self._load), + ("re_save_as", self._save_as), ): - btn = QPushButton(label) + btn = self._tr(QPushButton(), key) btn.clicked.connect(handler) top.addWidget(btn) root.addLayout(top) @@ -50,41 +62,41 @@ def _build_layout(self) -> None: root.addWidget(self._list, stretch=1) ops1 = QHBoxLayout() - ops1.addWidget(QLabel("Trim start:")) + ops1.addWidget(self._tr(QLabel(), "re_trim_start")) ops1.addWidget(self._trim_start) - ops1.addWidget(QLabel("end:")) + ops1.addWidget(self._tr(QLabel(), "re_trim_end")) ops1.addWidget(self._trim_end) - trim_btn = QPushButton("Apply trim") + trim_btn = self._tr(QPushButton(), "re_apply_trim") trim_btn.clicked.connect(self._apply_trim) ops1.addWidget(trim_btn) - remove_btn = QPushButton("Remove selected") + remove_btn = self._tr(QPushButton(), "re_remove_selected") remove_btn.clicked.connect(self._remove_selected) ops1.addWidget(remove_btn) ops1.addStretch() root.addLayout(ops1) ops2 = QHBoxLayout() - ops2.addWidget(QLabel("Delay x")) + ops2.addWidget(self._tr(QLabel(), "re_delay_x")) ops2.addWidget(self._delay_factor) - ops2.addWidget(QLabel("floor ms:")) + ops2.addWidget(self._tr(QLabel(), "re_floor_ms")) ops2.addWidget(self._delay_clamp) - delay_btn = QPushButton("Apply delays") + delay_btn = self._tr(QPushButton(), "re_apply_delays") delay_btn.clicked.connect(self._apply_delays) ops2.addWidget(delay_btn) - ops2.addWidget(QLabel("Scale x:")) + ops2.addWidget(self._tr(QLabel(), "re_scale_x")) ops2.addWidget(self._scale_x) - ops2.addWidget(QLabel("y:")) + ops2.addWidget(self._tr(QLabel(), "re_scale_y")) ops2.addWidget(self._scale_y) - scale_btn = QPushButton("Apply scale") + scale_btn = self._tr(QPushButton(), "re_apply_scale") scale_btn.clicked.connect(self._apply_scale) ops2.addWidget(scale_btn) ops2.addStretch() root.addLayout(ops2) ops3 = QHBoxLayout() - keep_mouse = QPushButton("Keep mouse only") + keep_mouse = self._tr(QPushButton(), "re_keep_mouse") keep_mouse.clicked.connect(lambda: self._filter_prefix("AC_mouse")) - keep_keyboard = QPushButton("Keep keyboard only") + keep_keyboard = self._tr(QPushButton(), "re_keep_keyboard") keep_keyboard.clicked.connect( lambda: self._filter_prefix(("AC_type_keyboard", "AC_press_keyboard_key", "AC_release_keyboard_key", "AC_hotkey", "AC_write")) @@ -94,12 +106,14 @@ def _build_layout(self) -> None: ops3.addStretch() root.addLayout(ops3) - root.addWidget(QLabel("Preview:")) + root.addWidget(self._tr(QLabel(), "re_preview")) root.addWidget(self._preview, stretch=1) root.addWidget(self._status) def _browse(self) -> None: - path, _ = QFileDialog.getOpenFileName(self, "Open recording", "", "JSON (*.json)") + path, _ = QFileDialog.getOpenFileName( + self, _t("re_dialog_open"), "", "JSON (*.json)", + ) if path: self._path_input.setText(path) @@ -117,7 +131,9 @@ def _load(self) -> None: def _save_as(self) -> None: if not self._actions: return - path, _ = QFileDialog.getSaveFileName(self, "Save recording", "", "JSON (*.json)") + path, _ = QFileDialog.getSaveFileName( + self, _t("re_dialog_save"), "", "JSON (*.json)", + ) if not path: return try: diff --git a/je_auto_control/gui/run_history_tab.py b/je_auto_control/gui/run_history_tab.py new file mode 100644 index 00000000..aec57621 --- /dev/null +++ b/je_auto_control/gui/run_history_tab.py @@ -0,0 +1,217 @@ +"""Run History tab: browse past scheduler / trigger / hotkey fires.""" +import datetime as _dt +from pathlib import Path +from typing import Optional + +from PySide6.QtCore import QTimer, Qt, QUrl +from PySide6.QtGui import QDesktopServices +from PySide6.QtWidgets import ( + QAbstractItemView, QComboBox, QHBoxLayout, QHeaderView, QLabel, + QMessageBox, QPushButton, QTableWidget, QTableWidgetItem, + QVBoxLayout, QWidget, +) + +from je_auto_control.gui._i18n_helpers import TranslatableMixin +from je_auto_control.gui.language_wrapper.multi_language_wrapper import ( + language_wrapper, +) +from je_auto_control.utils.run_history.history_store import ( + SOURCE_HOTKEY, SOURCE_MANUAL, SOURCE_REST, SOURCE_SCHEDULER, + SOURCE_TRIGGER, STATUS_ERROR, STATUS_OK, STATUS_RUNNING, + default_history_store, +) + +_COLUMN_COUNT = 8 +_REFRESH_INTERVAL_MS = 2000 +_SOURCES = ( + ("rh_source_all", None), + ("rh_source_scheduler", SOURCE_SCHEDULER), + ("rh_source_trigger", SOURCE_TRIGGER), + ("rh_source_hotkey", SOURCE_HOTKEY), + ("rh_source_manual", SOURCE_MANUAL), + ("rh_source_rest", SOURCE_REST), +) +_STATUS_LABEL_KEYS = { + STATUS_OK: "rh_status_ok", + STATUS_ERROR: "rh_status_error", + STATUS_RUNNING: "rh_status_running", +} + + +def _t(key: str) -> str: + return language_wrapper.translate(key, key) + + +def _format_time(epoch: float) -> str: + try: + return _dt.datetime.fromtimestamp(epoch).strftime("%Y-%m-%d %H:%M:%S") + except (OSError, ValueError, OverflowError): + return str(epoch) + + +def _format_duration(seconds: Optional[float]) -> str: + if seconds is None: + return "-" + if seconds < 1.0: + return f"{int(seconds * 1000)} ms" + return f"{seconds:.2f} s" + + +class RunHistoryTab(TranslatableMixin, QWidget): + """Timeline view of every fired scheduler job / trigger / hotkey.""" + + def __init__(self, parent: Optional[QWidget] = None) -> None: + super().__init__(parent) + self._tr_init() + self._filter = QComboBox() + self._populate_filter() + self._filter.currentIndexChanged.connect(self._refresh) + self._table = QTableWidget(0, _COLUMN_COUNT) + self._table.setEditTriggers(QAbstractItemView.NoEditTriggers) + self._table.setSelectionBehavior(QAbstractItemView.SelectRows) + self._table.verticalHeader().setVisible(False) + self._apply_table_headers() + header = self._table.horizontalHeader() + header.setSectionResizeMode(QHeaderView.Interactive) + header.setStretchLastSection(True) + self._count_label = QLabel() + self._timer = QTimer(self) + self._timer.setInterval(_REFRESH_INTERVAL_MS) + self._timer.timeout.connect(self._refresh) + self._auto_refresh = True + self._build_layout() + self._refresh() + self._timer.start() + + def retranslate(self) -> None: + TranslatableMixin.retranslate(self) + self._apply_table_headers() + self._repopulate_filter_labels() + self._refresh() + + def _populate_filter(self) -> None: + self._filter.blockSignals(True) + for label_key, source_value in _SOURCES: + self._filter.addItem(_t(label_key), source_value) + self._filter.blockSignals(False) + + def _repopulate_filter_labels(self) -> None: + self._filter.blockSignals(True) + current = self._filter.currentIndex() + for row, (label_key, _source) in enumerate(_SOURCES): + self._filter.setItemText(row, _t(label_key)) + self._filter.setCurrentIndex(max(0, current)) + self._filter.blockSignals(False) + + def _apply_table_headers(self) -> None: + self._table.setHorizontalHeaderLabels([ + _t("rh_col_id"), _t("rh_col_source"), _t("rh_col_target"), + _t("rh_col_script"), _t("rh_col_started"), + _t("rh_col_duration"), _t("rh_col_status"), + _t("rh_col_artifact"), + ]) + + def _build_layout(self) -> None: + root = QVBoxLayout(self) + top = QHBoxLayout() + top.addWidget(self._tr(QLabel(), "rh_filter_label")) + top.addWidget(self._filter) + top.addStretch() + refresh_btn = self._tr(QPushButton(), "rh_refresh") + refresh_btn.clicked.connect(self._refresh) + top.addWidget(refresh_btn) + clear_btn = self._tr(QPushButton(), "rh_clear") + clear_btn.clicked.connect(self._on_clear) + top.addWidget(clear_btn) + root.addLayout(top) + root.addWidget(self._table, stretch=1) + self._table.cellDoubleClicked.connect(self._on_cell_double_clicked) + open_row = QHBoxLayout() + self._open_artifact_btn = self._tr(QPushButton(), "rh_open_artifact") + self._open_artifact_btn.clicked.connect(self._open_selected_artifact) + open_row.addWidget(self._open_artifact_btn) + open_row.addStretch() + root.addLayout(open_row) + root.addWidget(self._count_label) + + def _on_clear(self) -> None: + reply = QMessageBox.question( + self, _t("rh_clear"), _t("rh_confirm_clear"), + QMessageBox.Yes | QMessageBox.No, QMessageBox.No, + ) + if reply == QMessageBox.Yes: + default_history_store.clear() + self._refresh() + + def _refresh(self) -> None: + source = self._filter.currentData() + try: + runs = default_history_store.list_runs(limit=500, source_type=source) + except ValueError: + runs = [] + self._table.setRowCount(len(runs)) + for row, record in enumerate(runs): + self._set_row(row, record) + self._count_label.setText( + _t("rh_count_label").replace("{n}", str(len(runs))), + ) + + def _set_row(self, row: int, record) -> None: + status_key = _STATUS_LABEL_KEYS.get(record.status, record.status) + status_text = _t(status_key) if record.error_text is None \ + else f"{_t(status_key)}: {record.error_text}" + artifact_text = record.artifact_path or "-" + values = ( + str(record.id), + record.source_type, + record.source_id, + record.script_path, + _format_time(record.started_at), + _format_duration(record.duration_seconds), + status_text, + artifact_text, + ) + for col, text in enumerate(values): + item = QTableWidgetItem(text) + item.setFlags(item.flags() & ~Qt.ItemIsEditable) + self._table.setItem(row, col, item) + + def _selected_artifact_path(self) -> Optional[str]: + row = self._table.currentRow() + if row < 0: + return None + item = self._table.item(row, _COLUMN_COUNT - 1) + if item is None: + return None + text = item.text() + if not text or text == "-": + return None + return text + + def _open_selected_artifact(self) -> None: + path = self._selected_artifact_path() + if path is None: + QMessageBox.information( + self, _t("rh_open_artifact"), _t("rh_no_artifact"), + ) + return + self._open_path(path) + + def _on_cell_double_clicked(self, row: int, column: int) -> None: + if column != _COLUMN_COUNT - 1: + return + item = self._table.item(row, column) + if item is None: + return + text = item.text() + if text and text != "-": + self._open_path(text) + + def _open_path(self, path: str) -> None: + resolved = Path(path) + if not resolved.exists(): + QMessageBox.warning( + self, _t("rh_open_artifact"), _t("rh_artifact_missing"), + ) + return + QDesktopServices.openUrl(QUrl.fromLocalFile(str(resolved))) diff --git a/je_auto_control/gui/scheduler_tab.py b/je_auto_control/gui/scheduler_tab.py index af75ff21..3d0bb8e5 100644 --- a/je_auto_control/gui/scheduler_tab.py +++ b/je_auto_control/gui/scheduler_tab.py @@ -7,40 +7,66 @@ QPushButton, QTableWidget, QTableWidgetItem, QVBoxLayout, QWidget, ) +from je_auto_control.gui._i18n_helpers import TranslatableMixin +from je_auto_control.gui.language_wrapper.multi_language_wrapper import ( + language_wrapper, +) from je_auto_control.utils.scheduler import default_scheduler -class SchedulerTab(QWidget): +def _t(key: str) -> str: + return language_wrapper.translate(key, key) + + +class SchedulerTab(TranslatableMixin, QWidget): """Add / remove / start / stop scheduler jobs.""" def __init__(self, parent: Optional[QWidget] = None) -> None: super().__init__(parent) + self._tr_init() self._path_input = QLineEdit() self._interval_input = QLineEdit("60") - self._repeat_check = QCheckBox("Repeat") + self._repeat_check = self._tr(QCheckBox(), "sch_repeat") self._repeat_check.setChecked(True) self._table = QTableWidget(0, 5) - self._table.setHorizontalHeaderLabels( - ["Job ID", "Script", "Interval (s)", "Runs", "Enabled"] - ) - self._status = QLabel("Scheduler stopped") + self._apply_table_headers() + self._running = False + self._status = QLabel() + self._apply_status() self._timer = QTimer(self) self._timer.setInterval(1000) self._timer.timeout.connect(self._refresh_table) self._build_layout() + def _apply_table_headers(self) -> None: + self._table.setHorizontalHeaderLabels([ + _t("sch_col_job_id"), _t("sch_col_script"), + _t("sch_col_interval"), _t("sch_col_runs"), + _t("sch_col_enabled"), + ]) + + def _apply_status(self) -> None: + key = "sch_status_running" if self._running else "sch_status_stopped" + self._status.setText(_t(key)) + + def retranslate(self) -> None: + TranslatableMixin.retranslate(self) + self._apply_table_headers() + self._apply_status() + self._refresh_table() + def _build_layout(self) -> None: root = QVBoxLayout(self) form = QHBoxLayout() - form.addWidget(QLabel("Script:")) + form.addWidget(self._tr(QLabel(), "sch_script_label")) form.addWidget(self._path_input, stretch=1) - browse = QPushButton("Browse") + browse = self._tr(QPushButton(), "browse") browse.clicked.connect(self._browse) form.addWidget(browse) - form.addWidget(QLabel("Every (s):")) + form.addWidget(self._tr(QLabel(), "sch_interval_label")) form.addWidget(self._interval_input) form.addWidget(self._repeat_check) - add_btn = QPushButton("Add") + add_btn = self._tr(QPushButton(), "sch_add") add_btn.clicked.connect(self._on_add) form.addWidget(add_btn) root.addLayout(form) @@ -48,12 +74,12 @@ def _build_layout(self) -> None: root.addWidget(self._table, stretch=1) ctl = QHBoxLayout() - for label, handler in ( - ("Remove selected", self._on_remove), - ("Start scheduler", self._on_start), - ("Stop scheduler", self._on_stop), + for key, handler in ( + ("sch_remove_selected", self._on_remove), + ("sch_start", self._on_start), + ("sch_stop", self._on_stop), ): - btn = QPushButton(label) + btn = self._tr(QPushButton(), key) btn.clicked.connect(handler) ctl.addWidget(btn) ctl.addStretch() @@ -61,7 +87,9 @@ def _build_layout(self) -> None: root.addWidget(self._status) def _browse(self) -> None: - path, _ = QFileDialog.getOpenFileName(self, "Select script", "", "JSON (*.json)") + path, _ = QFileDialog.getOpenFileName( + self, _t("sch_dialog_select_script"), "", "JSON (*.json)", + ) if path: self._path_input.setText(path) @@ -93,19 +121,23 @@ def _on_remove(self) -> None: def _on_start(self) -> None: default_scheduler.start() self._timer.start() - self._status.setText("Scheduler running") + self._running = True + self._apply_status() def _on_stop(self) -> None: default_scheduler.stop() self._timer.stop() - self._status.setText("Scheduler stopped") + self._running = False + self._apply_status() def _refresh_table(self) -> None: jobs = default_scheduler.list_jobs() self._table.setRowCount(len(jobs)) + yes = _t("tr_yes") + no = _t("tr_no") for row, job in enumerate(jobs): for col, value in enumerate(( job.job_id, job.script_path, f"{job.interval_seconds:g}", - str(job.runs), "Yes" if job.enabled else "No", + str(job.runs), yes if job.enabled else no, )): self._table.setItem(row, col, QTableWidgetItem(value)) diff --git a/je_auto_control/gui/script_builder/builder_tab.py b/je_auto_control/gui/script_builder/builder_tab.py index 12ca1efe..904a7804 100644 --- a/je_auto_control/gui/script_builder/builder_tab.py +++ b/je_auto_control/gui/script_builder/builder_tab.py @@ -9,6 +9,10 @@ QTextEdit, QToolButton, QVBoxLayout, QWidget, ) +from je_auto_control.gui._i18n_helpers import TranslatableMixin +from je_auto_control.gui.language_wrapper.multi_language_wrapper import ( + language_wrapper, +) from je_auto_control.gui.script_builder.command_schema import ( CATEGORIES, COMMAND_SPECS, specs_in_category, ) @@ -21,19 +25,32 @@ from je_auto_control.utils.json.json_file import read_action_json, write_action_json -class ScriptBuilderTab(QWidget): +def _t(key: str) -> str: + return language_wrapper.translate(key, key) + + +class ScriptBuilderTab(TranslatableMixin, QWidget): """Visual editor for composing AC_* scripts.""" def __init__(self, parent: Optional[QWidget] = None) -> None: super().__init__(parent) + self._tr_init() self._tree = StepTreeView() self._form = StepFormView() self._result = QTextEdit() self._result.setReadOnly(True) self._result.setMaximumHeight(140) + self._add_btn: Optional[QToolButton] = None self._build_layout() self._wire_signals() + def retranslate(self) -> None: + TranslatableMixin.retranslate(self) + if self._add_btn is not None: + self._add_btn.setText(_t("sb_add_step")) + if hasattr(self._form, "retranslate"): + self._form.retranslate() + def _build_layout(self) -> None: root = QVBoxLayout(self) root.addLayout(self._build_toolbar()) @@ -47,28 +64,28 @@ def _build_layout(self) -> None: def _build_toolbar(self) -> QHBoxLayout: bar = QHBoxLayout() bar.addWidget(self._add_button()) - for label, handler in ( - ("Delete", self._on_delete), - ("Up", lambda: self._tree.move_selected(-1)), - ("Down", lambda: self._tree.move_selected(1)), + for key, handler in ( + ("sb_delete", self._on_delete), + ("sb_up", lambda: self._tree.move_selected(-1)), + ("sb_down", lambda: self._tree.move_selected(1)), ): - btn = QPushButton(label) + btn = self._tr(QPushButton(), key) btn.clicked.connect(handler) bar.addWidget(btn) bar.addStretch() - for label, handler in ( - ("Load JSON", self._on_load), - ("Save JSON", self._on_save), - ("Run", self._on_run), + for key, handler in ( + ("sb_load_json", self._on_load), + ("sb_save_json", self._on_save), + ("sb_run", self._on_run), ): - btn = QPushButton(label) + btn = self._tr(QPushButton(), key) btn.clicked.connect(handler) bar.addWidget(btn) return bar def _add_button(self) -> QToolButton: button = QToolButton() - button.setText("Add Step") + button.setText(_t("sb_add_step")) button.setPopupMode(QToolButton.ToolButtonPopupMode.InstantPopup) menu = QMenu(button) for category in CATEGORIES: @@ -80,6 +97,7 @@ def _add_button(self) -> QToolButton: ) submenu.addAction(action) button.setMenu(menu) + self._add_btn = button return button def _wire_signals(self) -> None: @@ -102,7 +120,9 @@ def _on_delete(self) -> None: self._form.load_step(None) def _on_save(self) -> None: - path, _ = QFileDialog.getSaveFileName(self, "Save script", "", "JSON (*.json)") + path, _ = QFileDialog.getSaveFileName( + self, _t("sb_dialog_save"), "", "JSON (*.json)", + ) if not path: return try: @@ -113,7 +133,9 @@ def _on_save(self) -> None: QMessageBox.warning(self, "Error", str(error)) def _on_load(self) -> None: - path, _ = QFileDialog.getOpenFileName(self, "Load script", "", "JSON (*.json)") + path, _ = QFileDialog.getOpenFileName( + self, _t("sb_dialog_load"), "", "JSON (*.json)", + ) if not path: return try: diff --git a/je_auto_control/gui/script_builder/step_form_view.py b/je_auto_control/gui/script_builder/step_form_view.py index 4a0a98b7..f336d4fd 100644 --- a/je_auto_control/gui/script_builder/step_form_view.py +++ b/je_auto_control/gui/script_builder/step_form_view.py @@ -8,6 +8,9 @@ QLineEdit, QPushButton, QWidget, ) +from je_auto_control.gui.language_wrapper.multi_language_wrapper import ( + language_wrapper, +) from je_auto_control.gui.script_builder.command_schema import ( COMMAND_SPECS, CommandSpec, FieldSpec, FieldType, ) @@ -17,6 +20,10 @@ _EDITOR_BUILDERS: Dict[FieldType, Callable[["StepFormView", FieldSpec], QWidget]] +def _t(key: str) -> str: + return language_wrapper.translate(key, key) + + class StepFormView(QWidget): """Right-pane editor for a single Step.""" @@ -27,15 +34,22 @@ def __init__(self, parent: Optional[QWidget] = None) -> None: self._step: Optional[Step] = None self._editors: Dict[str, QWidget] = {} self._layout = QFormLayout(self) - self._title = QLabel("No step selected") + self._title = QLabel(_t("sb_no_step_selected")) self._layout.addRow(self._title) + def retranslate(self) -> None: + """Re-apply translated title and reload current step for label refresh.""" + if self._step is None: + self._title.setText(_t("sb_no_step_selected")) + else: + self.load_step(self._step) + def load_step(self, step: Optional[Step]) -> None: """Populate the form with fields for ``step``.""" self._clear() self._step = step if step is None: - self._title.setText("No step selected") + self._title.setText(_t("sb_no_step_selected")) return spec = COMMAND_SPECS.get(step.command) if spec is None: diff --git a/je_auto_control/gui/socket_server_tab.py b/je_auto_control/gui/socket_server_tab.py deleted file mode 100644 index dce68ed0..00000000 --- a/je_auto_control/gui/socket_server_tab.py +++ /dev/null @@ -1,141 +0,0 @@ -"""Socket + REST server control panel.""" -from typing import Optional - -from PySide6.QtGui import QIntValidator -from PySide6.QtWidgets import ( - QCheckBox, QGroupBox, QHBoxLayout, QLabel, QLineEdit, QMessageBox, - QPushButton, QVBoxLayout, QWidget, -) - -from je_auto_control.utils.rest_api.rest_server import RestApiServer -from je_auto_control.utils.socket_server.auto_control_socket_server import ( - start_autocontrol_socket_server, -) - - -class SocketServerTab(QWidget): - """Start / stop the TCP socket server and the REST API server.""" - - def __init__(self, parent: Optional[QWidget] = None) -> None: - super().__init__(parent) - self._tcp_server = None - self._rest_server: Optional[RestApiServer] = None - self._tcp_host = QLineEdit("127.0.0.1") - self._tcp_port = QLineEdit("9938") - self._tcp_port.setValidator(QIntValidator(1, 65535)) - self._tcp_any = QCheckBox("Bind TCP to 0.0.0.0 (exposes to network)") - self._tcp_status = QLabel("TCP stopped") - self._tcp_start_btn = QPushButton("Start TCP") - self._tcp_stop_btn = QPushButton("Stop TCP") - self._tcp_stop_btn.setEnabled(False) - - self._rest_host = QLineEdit("127.0.0.1") - self._rest_port = QLineEdit("9939") - self._rest_port.setValidator(QIntValidator(1, 65535)) - self._rest_any = QCheckBox("Bind REST to 0.0.0.0 (exposes to network)") - self._rest_status = QLabel("REST stopped") - self._rest_start_btn = QPushButton("Start REST") - self._rest_stop_btn = QPushButton("Stop REST") - self._rest_stop_btn.setEnabled(False) - - self._build_layout() - - def _build_layout(self) -> None: - root = QVBoxLayout(self) - - tcp_group = QGroupBox("TCP socket server") - tcp_layout = QVBoxLayout(tcp_group) - tcp_form = QHBoxLayout() - tcp_form.addWidget(QLabel("Host:")) - tcp_form.addWidget(self._tcp_host) - tcp_form.addWidget(QLabel("Port:")) - tcp_form.addWidget(self._tcp_port) - tcp_layout.addLayout(tcp_form) - tcp_layout.addWidget(self._tcp_any) - tcp_btns = QHBoxLayout() - self._tcp_start_btn.clicked.connect(self._start_tcp) - self._tcp_stop_btn.clicked.connect(self._stop_tcp) - tcp_btns.addWidget(self._tcp_start_btn) - tcp_btns.addWidget(self._tcp_stop_btn) - tcp_btns.addStretch() - tcp_layout.addLayout(tcp_btns) - tcp_layout.addWidget(self._tcp_status) - root.addWidget(tcp_group) - - rest_group = QGroupBox("REST API server") - rest_layout = QVBoxLayout(rest_group) - rest_form = QHBoxLayout() - rest_form.addWidget(QLabel("Host:")) - rest_form.addWidget(self._rest_host) - rest_form.addWidget(QLabel("Port:")) - rest_form.addWidget(self._rest_port) - rest_layout.addLayout(rest_form) - rest_layout.addWidget(self._rest_any) - rest_btns = QHBoxLayout() - self._rest_start_btn.clicked.connect(self._start_rest) - self._rest_stop_btn.clicked.connect(self._stop_rest) - rest_btns.addWidget(self._rest_start_btn) - rest_btns.addWidget(self._rest_stop_btn) - rest_btns.addStretch() - rest_layout.addLayout(rest_btns) - rest_layout.addWidget(self._rest_status) - root.addWidget(rest_group) - - root.addStretch() - - def _resolved_host(self, input_field: QLineEdit, any_addr: QCheckBox) -> str: - if any_addr.isChecked(): - return "0.0.0.0" # noqa: S104 # nosec B104 # reason: explicit opt-in via checkbox - return input_field.text().strip() or "127.0.0.1" - - def _start_tcp(self) -> None: - if self._tcp_server is not None: - return - host = self._resolved_host(self._tcp_host, self._tcp_any) - try: - port = int(self._tcp_port.text() or "9938") - self._tcp_server = start_autocontrol_socket_server(host, port) - except (OSError, ValueError) as error: - QMessageBox.warning(self, "Error", str(error)) - return - self._tcp_status.setText(f"Listening on {host}:{port}") - self._tcp_start_btn.setEnabled(False) - self._tcp_stop_btn.setEnabled(True) - - def _stop_tcp(self) -> None: - if self._tcp_server is None: - return - try: - self._tcp_server.shutdown() - self._tcp_server.server_close() - except OSError as error: - QMessageBox.warning(self, "Error", str(error)) - self._tcp_server = None - self._tcp_status.setText("TCP stopped") - self._tcp_start_btn.setEnabled(True) - self._tcp_stop_btn.setEnabled(False) - - def _start_rest(self) -> None: - if self._rest_server is not None: - return - host = self._resolved_host(self._rest_host, self._rest_any) - try: - port = int(self._rest_port.text() or "9939") - self._rest_server = RestApiServer(host=host, port=port) - self._rest_server.start() - except (OSError, ValueError) as error: - QMessageBox.warning(self, "Error", str(error)) - self._rest_server = None - return - self._rest_status.setText(f"Listening on {host}:{port}") - self._rest_start_btn.setEnabled(False) - self._rest_stop_btn.setEnabled(True) - - def _stop_rest(self) -> None: - if self._rest_server is None: - return - self._rest_server.stop() - self._rest_server = None - self._rest_status.setText("REST stopped") - self._rest_start_btn.setEnabled(True) - self._rest_stop_btn.setEnabled(False) diff --git a/je_auto_control/gui/triggers_tab.py b/je_auto_control/gui/triggers_tab.py index 78cd2166..cd8d51fe 100644 --- a/je_auto_control/gui/triggers_tab.py +++ b/je_auto_control/gui/triggers_tab.py @@ -8,51 +8,90 @@ QVBoxLayout, QWidget, ) +from je_auto_control.gui._i18n_helpers import TranslatableMixin +from je_auto_control.gui.language_wrapper.multi_language_wrapper import ( + language_wrapper, +) from je_auto_control.utils.triggers.trigger_engine import ( FilePathTrigger, ImageAppearsTrigger, PixelColorTrigger, WindowAppearsTrigger, default_trigger_engine, ) -class TriggersTab(QWidget): +def _t(key: str) -> str: + return language_wrapper.translate(key, key) + + +_TYPE_KEYS = ( + "tr_type_image", "tr_type_window", "tr_type_pixel", "tr_type_file", +) + + +class TriggersTab(TranslatableMixin, QWidget): """Build triggers, run the engine, inspect the table.""" def __init__(self, parent: Optional[QWidget] = None) -> None: super().__init__(parent) + self._tr_init() self._script_input = QLineEdit() - self._repeat_check = QCheckBox("Repeat") + self._repeat_check = self._tr(QCheckBox(), "tr_repeat") self._repeat_check.setChecked(True) self._type_combo = QComboBox() - self._type_combo.addItems(["Image appears", "Window appears", - "Pixel matches", "File changed"]) + self._apply_type_combo() self._stack = QStackedWidget() self._image_widgets = self._build_image_form() self._window_widgets = self._build_window_form() self._pixel_widgets = self._build_pixel_form() self._file_widgets = self._build_file_form() - self._status = QLabel("Engine stopped") + self._running = False + self._status = QLabel() + self._apply_status() self._table = QTableWidget(0, 5) - self._table.setHorizontalHeaderLabels( - ["ID", "Type", "Detail", "Fired", "Enabled"] - ) + self._apply_table_headers() self._timer = QTimer(self) self._timer.setInterval(1000) self._timer.timeout.connect(self._refresh) self._build_layout() self._type_combo.currentIndexChanged.connect(self._stack.setCurrentIndex) + def _apply_type_combo(self) -> None: + current = self._type_combo.currentIndex() + self._type_combo.blockSignals(True) + self._type_combo.clear() + self._type_combo.addItems([_t(k) for k in _TYPE_KEYS]) + if 0 <= current < self._type_combo.count(): + self._type_combo.setCurrentIndex(current) + self._type_combo.blockSignals(False) + + def _apply_table_headers(self) -> None: + self._table.setHorizontalHeaderLabels([ + _t("tr_col_id"), _t("tr_col_type"), + _t("tr_col_detail"), _t("tr_col_fired"), _t("tr_col_enabled"), + ]) + + def _apply_status(self) -> None: + key = "tr_engine_running" if self._running else "tr_engine_stopped" + self._status.setText(_t(key)) + + def retranslate(self) -> None: + TranslatableMixin.retranslate(self) + self._apply_type_combo() + self._apply_table_headers() + self._apply_status() + self._refresh() + def _build_layout(self) -> None: root = QVBoxLayout(self) form_top = QHBoxLayout() - form_top.addWidget(QLabel("Script:")) + form_top.addWidget(self._tr(QLabel(), "tr_script_label")) form_top.addWidget(self._script_input, stretch=1) - browse = QPushButton("Browse") + browse = self._tr(QPushButton(), "browse") browse.clicked.connect(self._browse_script) form_top.addWidget(browse) form_top.addWidget(self._repeat_check) - form_top.addWidget(QLabel("Type:")) + form_top.addWidget(self._tr(QLabel(), "tr_type_label")) form_top.addWidget(self._type_combo) - add_btn = QPushButton("Add trigger") + add_btn = self._tr(QPushButton(), "tr_add") add_btn.clicked.connect(self._on_add) form_top.addWidget(add_btn) root.addLayout(form_top) @@ -60,12 +99,12 @@ def _build_layout(self) -> None: root.addWidget(self._table, stretch=1) ctl = QHBoxLayout() - for label, handler in ( - ("Remove selected", self._on_remove), - ("Start engine", self._on_start), - ("Stop engine", self._on_stop), + for key, handler in ( + ("tr_remove_selected", self._on_remove), + ("tr_start_engine", self._on_start), + ("tr_stop_engine", self._on_stop), ): - btn = QPushButton(label) + btn = self._tr(QPushButton(), key) btn.clicked.connect(handler) ctl.addWidget(btn) ctl.addStretch() @@ -77,12 +116,12 @@ def _build_image_form(self) -> dict: layout = QHBoxLayout(widget) path_input = QLineEdit() threshold_input = QLineEdit("0.8") - browse = QPushButton("Browse") + browse = self._tr(QPushButton(), "browse") browse.clicked.connect(lambda: self._browse_image(path_input)) - layout.addWidget(QLabel("Image:")) + layout.addWidget(self._tr(QLabel(), "tr_image_label")) layout.addWidget(path_input, stretch=1) layout.addWidget(browse) - layout.addWidget(QLabel("Threshold:")) + layout.addWidget(self._tr(QLabel(), "tr_threshold_label")) layout.addWidget(threshold_input) self._stack.addWidget(widget) return {"path": path_input, "threshold": threshold_input} @@ -91,7 +130,7 @@ def _build_window_form(self) -> dict: widget = QWidget() layout = QHBoxLayout(widget) title_input = QLineEdit() - layout.addWidget(QLabel("Title contains:")) + layout.addWidget(self._tr(QLabel(), "tr_title_contains_label")) layout.addWidget(title_input, stretch=1) self._stack.addWidget(widget) return {"title": title_input} @@ -107,7 +146,7 @@ def _build_pixel_form(self) -> dict: tol_input = QLineEdit("8") for label, field in (("X:", x_input), ("Y:", y_input), ("R:", r_input), ("G:", g_input), ("B:", b_input), - ("±:", tol_input)): + ("\u00b1:", tol_input)): layout.addWidget(QLabel(label)) layout.addWidget(field) self._stack.addWidget(widget) @@ -118,27 +157,30 @@ def _build_file_form(self) -> dict: widget = QWidget() layout = QHBoxLayout(widget) path_input = QLineEdit() - browse = QPushButton("Browse") + browse = self._tr(QPushButton(), "browse") browse.clicked.connect(lambda: self._browse_watch(path_input)) - layout.addWidget(QLabel("Watch path:")) + layout.addWidget(self._tr(QLabel(), "tr_watch_label")) layout.addWidget(path_input, stretch=1) layout.addWidget(browse) self._stack.addWidget(widget) return {"path": path_input} def _browse_script(self) -> None: - path, _ = QFileDialog.getOpenFileName(self, "Select script", "", "JSON (*.json)") + path, _ = QFileDialog.getOpenFileName( + self, _t("tr_dialog_select_script"), "", "JSON (*.json)", + ) if path: self._script_input.setText(path) def _browse_image(self, target: QLineEdit) -> None: - path, _ = QFileDialog.getOpenFileName(self, "Select image", "", - "Images (*.png *.jpg *.bmp)") + path, _ = QFileDialog.getOpenFileName( + self, _t("tr_dialog_select_image"), "", "Images (*.png *.jpg *.bmp)", + ) if path: target.setText(path) def _browse_watch(self, target: QLineEdit) -> None: - path, _ = QFileDialog.getOpenFileName(self, "Select file to watch", "") + path, _ = QFileDialog.getOpenFileName(self, _t("tr_dialog_select_file"), "") if path: target.setText(path) @@ -196,21 +238,25 @@ def _on_remove(self) -> None: def _on_start(self) -> None: default_trigger_engine.start() self._timer.start() - self._status.setText("Engine running") + self._running = True + self._apply_status() def _on_stop(self) -> None: default_trigger_engine.stop() self._timer.stop() - self._status.setText("Engine stopped") + self._running = False + self._apply_status() def _refresh(self) -> None: triggers = default_trigger_engine.list_triggers() self._table.setRowCount(len(triggers)) + yes = _t("tr_yes") + no = _t("tr_no") for row, trigger in enumerate(triggers): detail = _describe(trigger) for col, value in enumerate(( trigger.trigger_id, type(trigger).__name__, detail, - str(trigger.fired), "Yes" if trigger.enabled else "No", + str(trigger.fired), yes if trigger.enabled else no, )): self._table.setItem(row, col, QTableWidgetItem(value)) @@ -221,7 +267,7 @@ def _describe(trigger) -> str: if isinstance(trigger, WindowAppearsTrigger): return f"title~{trigger.title_substring!r}" if isinstance(trigger, PixelColorTrigger): - return f"({trigger.x},{trigger.y})={trigger.target_rgb} ±{trigger.tolerance}" + return f"({trigger.x},{trigger.y})={trigger.target_rgb} \u00b1{trigger.tolerance}" if isinstance(trigger, FilePathTrigger): return f"watch={trigger.watch_path}" return "?" diff --git a/je_auto_control/gui/vlm_tab.py b/je_auto_control/gui/vlm_tab.py new file mode 100644 index 00000000..82aa41d5 --- /dev/null +++ b/je_auto_control/gui/vlm_tab.py @@ -0,0 +1,111 @@ +"""VLM tab: describe a UI element in words and have a model find it.""" +from typing import Optional + +from PySide6.QtWidgets import ( + QHBoxLayout, QLabel, QLineEdit, QMessageBox, QPushButton, + QVBoxLayout, QWidget, +) + +from je_auto_control.gui._i18n_helpers import TranslatableMixin +from je_auto_control.gui.language_wrapper.multi_language_wrapper import ( + language_wrapper, +) +from je_auto_control.utils.vision.backends.base import VLMNotAvailableError +from je_auto_control.utils.vision.vlm_api import ( + click_by_description, locate_by_description, +) + + +def _t(key: str) -> str: + return language_wrapper.translate(key, key) + + +class VLMTab(TranslatableMixin, QWidget): + """Drive a vision-language model to locate or click described elements.""" + + def __init__(self, parent: Optional[QWidget] = None) -> None: + super().__init__(parent) + self._tr_init() + self._description = QLineEdit() + self._model = QLineEdit() + self._status = QLabel() + self._last_result = QLabel() + self._apply_placeholders() + self._build_layout() + + def retranslate(self) -> None: + TranslatableMixin.retranslate(self) + self._apply_placeholders() + + def _apply_placeholders(self) -> None: + self._description.setPlaceholderText(_t("vlm_desc_placeholder")) + self._model.setPlaceholderText(_t("vlm_model_placeholder")) + + def _build_layout(self) -> None: + root = QVBoxLayout(self) + desc_row = QHBoxLayout() + desc_row.addWidget(self._tr(QLabel(), "vlm_desc_label")) + desc_row.addWidget(self._description, stretch=1) + root.addLayout(desc_row) + model_row = QHBoxLayout() + model_row.addWidget(self._tr(QLabel(), "vlm_model_label")) + model_row.addWidget(self._model, stretch=1) + root.addLayout(model_row) + btn_row = QHBoxLayout() + locate_btn = self._tr(QPushButton(), "vlm_locate") + locate_btn.clicked.connect(self._on_locate) + click_btn = self._tr(QPushButton(), "vlm_click") + click_btn.clicked.connect(self._on_click) + btn_row.addWidget(locate_btn) + btn_row.addWidget(click_btn) + btn_row.addStretch() + root.addLayout(btn_row) + root.addWidget(self._last_result) + root.addWidget(self._status) + root.addStretch() + + def _collect_inputs(self): + description = self._description.text().strip() + if not description: + self._status.setText(_t("vlm_desc_required")) + return None + model = self._model.text().strip() or None + return description, model + + def _on_locate(self) -> None: + inputs = self._collect_inputs() + if inputs is None: + return + description, model = inputs + try: + coords = locate_by_description(description, model=model) + except VLMNotAvailableError as error: + QMessageBox.warning(self, _t("vlm_locate"), str(error)) + return + except (OSError, ValueError, RuntimeError) as error: + self._status.setText(f"{_t('vlm_error')}: {error}") + return + if coords is None: + self._status.setText(_t("vlm_not_found")) + self._last_result.setText("") + return + self._last_result.setText( + _t("vlm_result").replace("{x}", str(coords[0])) + .replace("{y}", str(coords[1])), + ) + self._status.setText(_t("vlm_ok")) + + def _on_click(self) -> None: + inputs = self._collect_inputs() + if inputs is None: + return + description, model = inputs + try: + ok = click_by_description(description, model=model) + except VLMNotAvailableError as error: + QMessageBox.warning(self, _t("vlm_click"), str(error)) + return + except (OSError, ValueError, RuntimeError) as error: + self._status.setText(f"{_t('vlm_error')}: {error}") + return + self._status.setText(_t("vlm_ok") if ok else _t("vlm_not_found")) diff --git a/je_auto_control/gui/window_tab.py b/je_auto_control/gui/window_tab.py index ac5852e1..716fbdb0 100644 --- a/je_auto_control/gui/window_tab.py +++ b/je_auto_control/gui/window_tab.py @@ -7,40 +7,71 @@ QTableWidgetItem, QVBoxLayout, QWidget, ) +from je_auto_control.gui._i18n_helpers import TranslatableMixin +from je_auto_control.gui.language_wrapper.multi_language_wrapper import ( + language_wrapper, +) from je_auto_control.wrapper.auto_control_window import ( close_window_by_title, focus_window, list_windows, ) -class WindowManagerTab(QWidget): +def _t(key: str) -> str: + return language_wrapper.translate(key, key) + + +class WindowManagerTab(TranslatableMixin, QWidget): """Browse top-level windows and trigger focus / close actions.""" def __init__(self, parent: Optional[QWidget] = None) -> None: super().__init__(parent) + self._tr_init() self._table = QTableWidget(0, 2) - self._table.setHorizontalHeaderLabels(["HWND", "Title"]) + self._apply_table_headers() self._filter = QLineEdit() - self._filter.setPlaceholderText("Filter by title substring") + self._apply_filter_placeholder() self._filter.textChanged.connect(self._apply_filter) + self._status_count: Optional[int] = None + self._status_error: Optional[str] = None self._status = QLabel("") self._build_layout() self.refresh() + def _apply_table_headers(self) -> None: + self._table.setHorizontalHeaderLabels([_t("win_col_hwnd"), _t("win_col_title")]) + + def _apply_filter_placeholder(self) -> None: + self._filter.setPlaceholderText(_t("win_filter_placeholder")) + + def _apply_status(self) -> None: + if self._status_error is not None: + self._status.setText(self._status_error) + elif self._status_count is not None: + self._status.setText(_t("win_status_count").replace("{n}", str(self._status_count))) + else: + self._status.setText("") + + def retranslate(self) -> None: + TranslatableMixin.retranslate(self) + self._apply_table_headers() + self._apply_filter_placeholder() + self._apply_status() + def _build_layout(self) -> None: root = QVBoxLayout(self) top = QHBoxLayout() - refresh = QPushButton("Refresh") + refresh = self._tr(QPushButton(), "win_refresh") refresh.clicked.connect(self.refresh) top.addWidget(refresh) top.addWidget(self._filter, stretch=1) root.addLayout(top) root.addWidget(self._table, stretch=1) actions = QHBoxLayout() - for label, handler in ( - ("Focus selected", self._on_focus), - ("Close selected", self._on_close), + for key, handler in ( + ("win_focus_selected", self._on_focus), + ("win_close_selected", self._on_close), ): - btn = QPushButton(label) + btn = self._tr(QPushButton(), key) btn.clicked.connect(handler) actions.addWidget(btn) actions.addStretch() @@ -51,14 +82,18 @@ def refresh(self) -> None: try: windows = list_windows() except NotImplementedError as error: - self._status.setText(str(error)) + self._status_error = str(error) + self._status_count = None + self._apply_status() self._table.setRowCount(0) return + self._status_error = None self._table.setRowCount(len(windows)) for row, (hwnd, title) in enumerate(windows): self._table.setItem(row, 0, QTableWidgetItem(str(hwnd))) self._table.setItem(row, 1, QTableWidgetItem(title)) - self._status.setText(f"{len(windows)} windows") + self._status_count = len(windows) + self._apply_status() QTimer.singleShot(0, self._apply_filter) def _apply_filter(self) -> None: @@ -81,7 +116,7 @@ def _on_focus(self) -> None: return try: focus_window(title, case_sensitive=True) - except (NotImplementedError, RuntimeError, OSError) as error: + except (RuntimeError, OSError) as error: QMessageBox.warning(self, "Error", str(error)) def _on_close(self) -> None: @@ -91,5 +126,5 @@ def _on_close(self) -> None: try: close_window_by_title(title, case_sensitive=True) self.refresh() - except (NotImplementedError, RuntimeError, OSError) as error: + except (RuntimeError, OSError) as error: QMessageBox.warning(self, "Error", str(error)) diff --git a/je_auto_control/linux_with_x11/keyboard/x11_linux_keyboard_control.py b/je_auto_control/linux_with_x11/keyboard/x11_linux_keyboard_control.py index fbd036a4..9fe6febc 100644 --- a/je_auto_control/linux_with_x11/keyboard/x11_linux_keyboard_control.py +++ b/je_auto_control/linux_with_x11/keyboard/x11_linux_keyboard_control.py @@ -13,6 +13,8 @@ from Xlib.ext.xtest import fake_input from Xlib import X, protocol +_KEYCODE_INT_ERROR = "Keycode must be an integer 鍵盤代碼必須是整數" + def press_key(keycode: int) -> None: """ @@ -22,7 +24,7 @@ def press_key(keycode: int) -> None: :param keycode: (int) The keycode to press 要按下的鍵盤代碼 """ if not isinstance(keycode, int): - raise ValueError("Keycode must be an integer 鍵盤代碼必須是整數") + raise ValueError(_KEYCODE_INT_ERROR) time.sleep(0.01) # Small delay to ensure event stability 確保事件穩定的小延遲 fake_input(display, X.KeyPress, keycode) @@ -37,7 +39,7 @@ def release_key(keycode: int) -> None: :param keycode: (int) The keycode to release 要釋放的鍵盤代碼 """ if not isinstance(keycode, int): - raise ValueError("Keycode must be an integer 鍵盤代碼必須是整數") + raise ValueError(_KEYCODE_INT_ERROR) time.sleep(0.01) fake_input(display, X.KeyRelease, keycode) @@ -55,7 +57,7 @@ def send_key_event_to_window(window_id: int, keycode: int) -> None: if not isinstance(window_id, int): raise ValueError("Window ID must be an integer 視窗 ID 必須是整數") if not isinstance(keycode, int): - raise ValueError("Keycode must be an integer 鍵盤代碼必須是整數") + raise ValueError(_KEYCODE_INT_ERROR) # 建立目標視窗物件 Create target window object window = display.create_resource_object("window", window_id) diff --git a/je_auto_control/linux_with_x11/listener/x11_linux_listener.py b/je_auto_control/linux_with_x11/listener/x11_linux_listener.py index c02f5fa2..52590c59 100644 --- a/je_auto_control/linux_with_x11/listener/x11_linux_listener.py +++ b/je_auto_control/linux_with_x11/listener/x11_linux_listener.py @@ -19,19 +19,22 @@ current_display = Display() -class KeypressHandler(Thread): +class KeypressHandler: """ KeypressHandler 鍵盤事件處理器 - 負責解析 X11 事件 - 可選擇記錄事件到 Queue + + Plain handler (not a Thread): instances are passed as the + ``record_enable_context`` callback, never started as their own thread. """ def __init__(self, default_daemon: bool = True): """ - :param default_daemon: 是否設為守護執行緒 (程式結束時自動停止) + :param default_daemon: kept for backwards compatibility; the handler + is no longer a Thread, so this flag is informational only. """ - super().__init__() self.daemon = default_daemon self.still_listener = True self.record_flag = False @@ -49,7 +52,7 @@ def check_is_press(self, keycode: int) -> bool: return True return False - def run(self, reply) -> None: + def handle_reply(self, reply) -> None: """ 處理 X11 回傳的事件資料 Handle X11 reply data and parse events @@ -137,7 +140,7 @@ def run(self) -> None: }] ) # 啟用事件監聽 - current_display.record_enable_context(self.context, self.handler.run) + current_display.record_enable_context(self.context, self.handler.handle_reply) current_display.record_free_context(self.context) # 持續等待事件 diff --git a/je_auto_control/linux_with_x11/record/x11_linux_record.py b/je_auto_control/linux_with_x11/record/x11_linux_record.py index 90e8e1fe..01744b4a 100644 --- a/je_auto_control/linux_with_x11/record/x11_linux_record.py +++ b/je_auto_control/linux_with_x11/record/x11_linux_record.py @@ -56,7 +56,7 @@ def stop_record(self) -> Queue[Any]: action_queue = Queue() # 將原始事件轉換成可讀格式 - for details in list(self.result_queue.queue): + for details in self.result_queue.queue: if details[0] == 5: # 滑鼠事件 action_queue.put( (detail_dict.get(details[1]), details[2], details[3]) diff --git a/je_auto_control/osx/listener/osx_listener.py b/je_auto_control/osx/listener/osx_listener.py index 4ff3af90..f6fe4e61 100644 --- a/je_auto_control/osx/listener/osx_listener.py +++ b/je_auto_control/osx/listener/osx_listener.py @@ -33,7 +33,7 @@ class AppDelegate(NSObject): - 負責在應用程式啟動後註冊全域事件監聽器 """ - def applicationDidFinishLaunching_(self, aNotification): + def applicationDidFinishLaunching_(self, notification): # noqa: N802 # reason: ObjC selector signature """ 註冊全域事件監聽器 Register global event monitors diff --git a/je_auto_control/osx/pid/pid_control.py b/je_auto_control/osx/pid/pid_control.py index d1f4e25e..52385c5b 100644 --- a/je_auto_control/osx/pid/pid_control.py +++ b/je_auto_control/osx/pid/pid_control.py @@ -1,5 +1,5 @@ import objc -import subprocess +import subprocess # nosec B404 # reason: required to invoke osascript with argv list from ctypes import cdll, c_void_p from Quartz import CGEventCreateKeyboardEvent @@ -54,10 +54,11 @@ def get_pid_by_window_title(title: str) -> int | None: end tell ''' try: - pid_str = subprocess.check_output( + pid_str = subprocess.check_output( # nosec B603 B607 # reason: argv list, osascript on PATH; title is escaped ["osascript", "-e", script], - stderr=subprocess.DEVNULL + stderr=subprocess.DEVNULL, + timeout=5, ).decode().strip() return int(pid_str) if pid_str else None - except (subprocess.CalledProcessError, ValueError): + except (subprocess.CalledProcessError, subprocess.TimeoutExpired, ValueError): return None \ No newline at end of file diff --git a/je_auto_control/osx/screen/osx_screen.py b/je_auto_control/osx/screen/osx_screen.py index 85a2f994..bb966cef 100644 --- a/je_auto_control/osx/screen/osx_screen.py +++ b/je_auto_control/osx/screen/osx_screen.py @@ -41,10 +41,10 @@ def get_pixel(x: int, y: int) -> Tuple[int, int, int, int]: cf = ctypes.CDLL("/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation") # 定義 CGRect 結構 (x, y, width, height) - CGRect = ctypes.c_double * 4 + cg_rect_t = ctypes.c_double * 4 # 設定函式簽名 Function signatures - cg.CGWindowListCreateImage.argtypes = [CGRect, c_uint32, c_uint32, c_uint32] + cg.CGWindowListCreateImage.argtypes = [cg_rect_t, c_uint32, c_uint32, c_uint32] cg.CGWindowListCreateImage.restype = c_void_p cg.CGImageGetDataProvider.argtypes = [c_void_p] @@ -62,20 +62,21 @@ def get_pixel(x: int, y: int) -> Tuple[int, int, int, int]: cf.CFRelease.argtypes = [c_void_p] cf.CFRelease.restype = None - # 常數 Constants - kCGWindowListOptionOnScreenOnly = 1 - kCGNullWindowID = 0 - kCGWindowImageDefault = 0 + # 常數 Constants (Apple names: kCGWindowListOptionOnScreenOnly, + # kCGNullWindowID, kCGWindowImageDefault) + window_list_option_on_screen_only = 1 + null_window_id = 0 + window_image_default = 0 # 建立擷取範圍 Create capture rect - rect = CGRect(x, y, 1.0, 1.0) + rect = cg_rect_t(x, y, 1.0, 1.0) # 擷取螢幕影像 Capture screen image img = cg.CGWindowListCreateImage( rect, - kCGWindowListOptionOnScreenOnly, - kCGNullWindowID, - kCGWindowImageDefault + window_list_option_on_screen_only, + null_window_id, + window_image_default ) if not img: raise RuntimeError( diff --git a/je_auto_control/utils/accessibility/__init__.py b/je_auto_control/utils/accessibility/__init__.py new file mode 100644 index 00000000..414f4e79 --- /dev/null +++ b/je_auto_control/utils/accessibility/__init__.py @@ -0,0 +1,12 @@ +"""Cross-platform accessibility-tree widget location.""" +from je_auto_control.utils.accessibility.accessibility_api import ( + AccessibilityElement, AccessibilityNotAvailableError, + click_accessibility_element, find_accessibility_element, + list_accessibility_elements, +) + +__all__ = [ + "AccessibilityElement", "AccessibilityNotAvailableError", + "list_accessibility_elements", "find_accessibility_element", + "click_accessibility_element", +] diff --git a/je_auto_control/utils/accessibility/accessibility_api.py b/je_auto_control/utils/accessibility/accessibility_api.py new file mode 100644 index 00000000..c7de788f --- /dev/null +++ b/je_auto_control/utils/accessibility/accessibility_api.py @@ -0,0 +1,63 @@ +"""Public cross-platform accessibility API. + +Target GUI elements by role / name / owning-app rather than pixel +coordinates. The backend is chosen by :func:`get_backend` per platform +and can be swapped out in tests via ``reset_backend_cache``. +""" +from typing import List, Optional + +from je_auto_control.utils.accessibility.backends import get_backend +from je_auto_control.utils.accessibility.element import ( + AccessibilityElement, AccessibilityNotAvailableError, element_matches, +) + + +def list_accessibility_elements(app_name: Optional[str] = None, + max_results: int = 200, + ) -> List[AccessibilityElement]: + """Return a flat list of accessibility elements, optionally filtered.""" + return get_backend().list_elements( + app_name=app_name, max_results=int(max_results), + ) + + +def find_accessibility_element(name: Optional[str] = None, + role: Optional[str] = None, + app_name: Optional[str] = None, + ) -> Optional[AccessibilityElement]: + """First element matching all provided filters, or ``None``.""" + for element in list_accessibility_elements(app_name=app_name): + if element_matches(element, name=name, role=role, app_name=app_name): + return element + return None + + +def click_accessibility_element(name: Optional[str] = None, + role: Optional[str] = None, + app_name: Optional[str] = None, + ) -> bool: + """Click the center of the first element matching the filters. + + Returns ``True`` on success, ``False`` if nothing matched. Raises + :class:`AccessibilityNotAvailableError` if the platform backend is + missing. + """ + element = find_accessibility_element( + name=name, role=role, app_name=app_name, + ) + if element is None: + return False + cx, cy = element.center + from je_auto_control.wrapper.auto_control_mouse import ( + click_mouse, set_mouse_position, + ) + set_mouse_position(cx, cy) + click_mouse("mouse_left", cx, cy) + return True + + +__all__ = [ + "AccessibilityElement", "AccessibilityNotAvailableError", + "list_accessibility_elements", "find_accessibility_element", + "click_accessibility_element", +] diff --git a/je_auto_control/utils/accessibility/backends/__init__.py b/je_auto_control/utils/accessibility/backends/__init__.py new file mode 100644 index 00000000..4f39ffea --- /dev/null +++ b/je_auto_control/utils/accessibility/backends/__init__.py @@ -0,0 +1,59 @@ +"""Platform backends for the accessibility API.""" +import sys +from typing import Optional + +from je_auto_control.utils.accessibility.backends.base import ( + AccessibilityBackend, +) +from je_auto_control.utils.accessibility.backends.null_backend import ( + NullAccessibilityBackend, +) + +_cached_backend: Optional[AccessibilityBackend] = None + + +def get_backend() -> AccessibilityBackend: + """Return (and cache) the best backend for the current platform.""" + global _cached_backend + if _cached_backend is not None: + return _cached_backend + _cached_backend = _build_backend() + return _cached_backend + + +def reset_backend_cache() -> None: + """Force the next ``get_backend()`` call to re-detect.""" + global _cached_backend + _cached_backend = None + + +def _build_backend() -> AccessibilityBackend: + if sys.platform.startswith("win"): + from je_auto_control.utils.accessibility.backends.windows_backend import ( + WindowsAccessibilityBackend, + ) + backend = WindowsAccessibilityBackend() + if backend.available: + return backend + return NullAccessibilityBackend( + "install comtypes to enable Windows UIAutomation support", + ) + if sys.platform == "darwin": + from je_auto_control.utils.accessibility.backends.macos_backend import ( + MacOSAccessibilityBackend, + ) + backend = MacOSAccessibilityBackend() + if backend.available: + return backend + return NullAccessibilityBackend( + "pyobjc (ApplicationServices, AppKit) is required on macOS", + ) + return NullAccessibilityBackend( + f"no accessibility backend for platform {sys.platform!r}", + ) + + +__all__ = [ + "AccessibilityBackend", "NullAccessibilityBackend", + "get_backend", "reset_backend_cache", +] diff --git a/je_auto_control/utils/accessibility/backends/base.py b/je_auto_control/utils/accessibility/backends/base.py new file mode 100644 index 00000000..9af687c1 --- /dev/null +++ b/je_auto_control/utils/accessibility/backends/base.py @@ -0,0 +1,16 @@ +"""Abstract accessibility backend.""" +from typing import List, Optional + +from je_auto_control.utils.accessibility.element import AccessibilityElement + + +class AccessibilityBackend: + """Each backend exposes the platform's accessibility tree as flat lists.""" + + name: str = "abstract" + available: bool = False + + def list_elements(self, app_name: Optional[str] = None, + max_results: int = 200, + ) -> List[AccessibilityElement]: + raise NotImplementedError diff --git a/je_auto_control/utils/accessibility/backends/macos_backend.py b/je_auto_control/utils/accessibility/backends/macos_backend.py new file mode 100644 index 00000000..20086c98 --- /dev/null +++ b/je_auto_control/utils/accessibility/backends/macos_backend.py @@ -0,0 +1,120 @@ +"""macOS accessibility backend via pyobjc's ``ApplicationServices``. + +Requires Accessibility permission for the Python interpreter (System +Settings → Privacy & Security → Accessibility). Enumerates the frontmost +application's window tree, or a specific ``app_name``. +""" +from typing import List, Optional + +from je_auto_control.utils.accessibility.backends.base import ( + AccessibilityBackend, +) +from je_auto_control.utils.accessibility.element import ( + AccessibilityElement, AccessibilityNotAvailableError, +) +from je_auto_control.utils.logging.logging_instance import autocontrol_logger + + +def _is_available() -> bool: + try: + import ApplicationServices # noqa: F401 # reason: probe import + import AppKit # noqa: F401 # reason: probe import + return True + except ImportError: + return False + + +class MacOSAccessibilityBackend(AccessibilityBackend): + """Accessibility walker using ``AXUIElement*`` APIs.""" + + name = "macos-ax" + + def __init__(self) -> None: + self.available = _is_available() + + def list_elements(self, app_name: Optional[str] = None, + max_results: int = 200, + ) -> List[AccessibilityElement]: + if not self.available: + raise AccessibilityNotAvailableError( + "pyobjc (ApplicationServices, AppKit) is required for " + "macOS accessibility", + ) + import ApplicationServices as ax_module + import AppKit + + workspace = AppKit.NSWorkspace.sharedWorkspace() + running_apps = list(workspace.runningApplications()) + results: List[AccessibilityElement] = [] + for app in running_apps: + if not app.isActive() and app_name is None: + continue + name = str(app.localizedName() or "") + if app_name is not None and name != app_name: + continue + pid = int(app.processIdentifier()) + try: + root = ax_module.AXUIElementCreateApplication(pid) + self._walk(ax_module, root, name, pid, results, max_results) + except Exception as error: # noqa: BLE001 # reason: AX errors vary + autocontrol_logger.warning( + "AX walk failed for %s (%d): %r", name, pid, error, + ) + if len(results) >= max_results: + break + return results[:max_results] + + def _walk(self, ax_module, element, app_name: str, pid: int, + results: List[AccessibilityElement], max_results: int) -> None: + if len(results) >= max_results: + return + converted = _convert_ax(ax_module, element, app_name, pid) + if converted is not None: + results.append(converted) + err, children = ax_module.AXUIElementCopyAttributeValue( + element, "AXChildren", None, + ) + if err or children is None: + return + for child in children: + if len(results) >= max_results: + return + self._walk(ax_module, child, app_name, pid, results, max_results) + + +def _convert_ax(ax_module, element, app_name: str, pid: int, + ) -> Optional[AccessibilityElement]: + try: + _err, role = ax_module.AXUIElementCopyAttributeValue( + element, "AXRole", None, + ) + _err, title = ax_module.AXUIElementCopyAttributeValue( + element, "AXTitle", None, + ) + _err, position = ax_module.AXUIElementCopyAttributeValue( + element, "AXPosition", None, + ) + _err, size = ax_module.AXUIElementCopyAttributeValue( + element, "AXSize", None, + ) + except Exception: # noqa: BLE001 # reason: AX errors vary + return None + if role is None and title is None: + return None + bounds = _extract_bounds(position, size) + return AccessibilityElement( + name=str(title or ""), + role=str(role or ""), + bounds=bounds, app_name=app_name, process_id=pid, + ) + + +def _extract_bounds(position, size) -> tuple: + try: + if position is None or size is None: + return (0, 0, 0, 0) + x, y = position + w, h = size + return (int(x), int(y), int(w), int(h)) + except (TypeError, ValueError): + return (0, 0, 0, 0) diff --git a/je_auto_control/utils/accessibility/backends/null_backend.py b/je_auto_control/utils/accessibility/backends/null_backend.py new file mode 100644 index 00000000..691e3b0e --- /dev/null +++ b/je_auto_control/utils/accessibility/backends/null_backend.py @@ -0,0 +1,24 @@ +"""Fallback backend used when no real backend can be loaded.""" +from typing import List, Optional + +from je_auto_control.utils.accessibility.backends.base import ( + AccessibilityBackend, +) +from je_auto_control.utils.accessibility.element import ( + AccessibilityElement, AccessibilityNotAvailableError, +) + + +class NullAccessibilityBackend(AccessibilityBackend): + """Backend that always reports an unavailability error.""" + + name = "null" + available = False + + def __init__(self, reason: str = "no accessibility backend available"): + self._reason = reason + + def list_elements(self, app_name: Optional[str] = None, + max_results: int = 200, + ) -> List[AccessibilityElement]: + raise AccessibilityNotAvailableError(self._reason) diff --git a/je_auto_control/utils/accessibility/backends/windows_backend.py b/je_auto_control/utils/accessibility/backends/windows_backend.py new file mode 100644 index 00000000..41b1c417 --- /dev/null +++ b/je_auto_control/utils/accessibility/backends/windows_backend.py @@ -0,0 +1,132 @@ +"""Windows UIAutomation backend via ``comtypes``. + +Requires ``pip install comtypes``. If the module is absent, ``available`` is +``False`` and the facade falls back to the Null backend. + +Flattens the UIAutomation tree into ``AccessibilityElement`` records one +level at a time starting from the root desktop, filtered by app if needed. +Only ``is_control_element=True`` nodes are surfaced to avoid millions of +decorative text children. +""" +from typing import List, Optional + +from je_auto_control.utils.accessibility.backends.base import ( + AccessibilityBackend, +) +from je_auto_control.utils.accessibility.element import ( + AccessibilityElement, AccessibilityNotAvailableError, +) +from je_auto_control.utils.logging.logging_instance import autocontrol_logger + +_TREE_SCOPE_DESCENDANTS = 4 +_UIA_IS_CONTROL_ELEMENT_PROPERTY = 30016 +_UIA_NAME_PROPERTY = 30005 + + +def _is_available() -> bool: + try: + import comtypes.client # noqa: F401 # reason: probe import + return True + except ImportError: + return False + + +class WindowsAccessibilityBackend(AccessibilityBackend): + """UIAutomation-based flat element listing.""" + + name = "windows-uia" + + def __init__(self) -> None: + self.available = _is_available() + self._automation = None + + def _ensure_automation(self): + if self._automation is not None: + return self._automation + if not self.available: + raise AccessibilityNotAvailableError( + "comtypes is required for Windows accessibility; " + "install it with: pip install comtypes", + ) + import comtypes.client # noqa: F401 + from comtypes import CoCreateInstance, GUID + try: + uia_module = comtypes.client.GetModule("UIAutomationCore.dll") + except OSError as error: + raise AccessibilityNotAvailableError( + f"UIAutomationCore.dll unavailable: {error!r}", + ) from error + automation = CoCreateInstance( + GUID("{ff48dba4-60ef-4201-aa87-54103eef594e}"), + interface=uia_module.IUIAutomation, + ) + self._automation = automation + return automation + + def list_elements(self, app_name: Optional[str] = None, + max_results: int = 200, + ) -> List[AccessibilityElement]: + automation = self._ensure_automation() + try: + root = automation.GetRootElement() + condition = automation.CreatePropertyCondition( + _UIA_IS_CONTROL_ELEMENT_PROPERTY, True, + ) + found = root.FindAll(_TREE_SCOPE_DESCENDANTS, condition) + except (OSError, AttributeError) as error: + autocontrol_logger.error("UIA FindAll failed: %r", error) + return [] + results: List[AccessibilityElement] = [] + count = min(max(0, int(max_results)), int(found.Length or 0)) + for idx in range(count): + element = _convert_uia(found.GetElement(idx)) + if element is None: + continue + if app_name is not None and element.app_name != app_name: + continue + results.append(element) + return results + + +def _convert_uia(raw) -> Optional[AccessibilityElement]: + try: + name = str(raw.CurrentName or "") + control_type = int(raw.CurrentControlType or 0) + rect = raw.CurrentBoundingRectangle + process_id = int(raw.CurrentProcessId or 0) + automation_id = str(raw.CurrentAutomationId or "") + except (OSError, AttributeError): + return None + width = max(0, int(rect.right - rect.left)) + height = max(0, int(rect.bottom - rect.top)) + return AccessibilityElement( + name=name, role=f"ControlType_{control_type}", + bounds=(int(rect.left), int(rect.top), width, height), + app_name=_process_name(process_id), + process_id=process_id, + native_id=automation_id, + ) + + +def _process_name(process_id: int) -> str: + if process_id <= 0: + return "" + try: + import ctypes + from ctypes import wintypes + kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) + process_query_information = 0x0400 | 0x0010 + handle = kernel32.OpenProcess(process_query_information, False, process_id) + if not handle: + return "" + try: + buf = ctypes.create_unicode_buffer(260) + size = wintypes.DWORD(len(buf)) + get_image = kernel32.QueryFullProcessImageNameW + if not get_image(handle, 0, buf, ctypes.byref(size)): + return "" + return buf.value.rsplit("\\", 1)[-1] + finally: + kernel32.CloseHandle(handle) + except OSError: + return "" diff --git a/je_auto_control/utils/accessibility/element.py b/je_auto_control/utils/accessibility/element.py new file mode 100644 index 00000000..3d286ae0 --- /dev/null +++ b/je_auto_control/utils/accessibility/element.py @@ -0,0 +1,50 @@ +"""Shared dataclasses and exceptions for the accessibility API.""" +from dataclasses import dataclass +from typing import Optional, Tuple + + +@dataclass(frozen=True) +class AccessibilityElement: + """A GUI element exposed through the platform's accessibility tree. + + Coordinates are in screen pixels; ``(left, top, width, height)``. + ``app_name`` / ``process_id`` identify the owning application. + """ + name: str + role: str + bounds: Tuple[int, int, int, int] + app_name: str = "" + process_id: int = 0 + native_id: str = "" + + @property + def center(self) -> Tuple[int, int]: + left, top, width, height = self.bounds + return (left + width // 2, top + height // 2) + + def to_dict(self) -> dict: + return { + "name": self.name, "role": self.role, + "bounds": list(self.bounds), + "app_name": self.app_name, "process_id": self.process_id, + "native_id": self.native_id, + "center": list(self.center), + } + + +class AccessibilityNotAvailableError(RuntimeError): + """Raised when the platform backend cannot be initialised.""" + + +def element_matches(element: AccessibilityElement, + name: Optional[str] = None, + role: Optional[str] = None, + app_name: Optional[str] = None) -> bool: + """Return True if ``element`` matches all non-None filters.""" + if name is not None and element.name != name: + return False + if role is not None and element.role.lower() != role.lower(): + return False + if app_name is not None and element.app_name != app_name: + return False + return True diff --git a/je_auto_control/utils/clipboard/clipboard.py b/je_auto_control/utils/clipboard/clipboard.py index ca90a597..bf289804 100644 --- a/je_auto_control/utils/clipboard/clipboard.py +++ b/je_auto_control/utils/clipboard/clipboard.py @@ -8,7 +8,7 @@ callers can degrade gracefully. """ import shutil -import subprocess +import subprocess # nosec B404 # reason: required for pbcopy/pbpaste/xclip/xsel import sys from typing import Optional @@ -115,14 +115,14 @@ def _win_set(text: str) -> None: # === macOS backend =========================================================== def _mac_get() -> str: - result = subprocess.run( + result = subprocess.run( # nosec B603 B607 # reason: hard-coded macOS clipboard tool, argv list ["pbpaste"], capture_output=True, check=True, timeout=5, ) return result.stdout.decode("utf-8", errors="replace") def _mac_set(text: str) -> None: - subprocess.run( + subprocess.run( # nosec B603 B607 # reason: hard-coded macOS clipboard tool, argv list ["pbcopy"], input=text.encode("utf-8"), check=True, timeout=5, ) @@ -143,7 +143,7 @@ def _linux_get() -> str: if cmd is None: raise RuntimeError("Install xclip or xsel for Linux clipboard support") read_cmd = cmd + ["-o"] if cmd[0] == "xclip" else cmd + ["--output"] - result = subprocess.run( + result = subprocess.run( # nosec B603 # reason: argv from allowlist (xclip/xsel) discovered via shutil.which read_cmd, capture_output=True, check=True, timeout=5, ) return result.stdout.decode("utf-8", errors="replace") @@ -154,7 +154,7 @@ def _linux_set(text: str) -> None: if cmd is None: raise RuntimeError("Install xclip or xsel for Linux clipboard support") write_cmd = cmd + ["-i"] if cmd[0] == "xclip" else cmd + ["--input"] - subprocess.run( + subprocess.run( # nosec B603 # reason: argv from allowlist (xclip/xsel) discovered via shutil.which write_cmd, input=text.encode("utf-8"), check=True, timeout=5, ) diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index c810d47b..a96bd76d 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -9,6 +9,12 @@ AutoControlActionException, AutoControlAddCommandException, AutoControlActionNullException ) +from je_auto_control.utils.accessibility.accessibility_api import ( + click_accessibility_element, find_accessibility_element, +) +from je_auto_control.utils.vision.vlm_api import ( + click_by_description, locate_by_description, +) from je_auto_control.utils.clipboard.clipboard import ( get_clipboard, set_clipboard, ) @@ -21,6 +27,7 @@ locate_text_center as ocr_locate_text_center, wait_for_text as ocr_wait_for_text, ) +from je_auto_control.utils.run_history.history_store import default_history_store from je_auto_control.utils.script_vars.interpolate import interpolate_actions from je_auto_control.utils.generate_report.generate_html_report import generate_html, generate_html_report from je_auto_control.utils.generate_report.generate_json_report import generate_json, generate_json_report @@ -48,6 +55,58 @@ ) +def _a11y_list_as_dicts(app_name: Optional[str] = None, + max_results: int = 200) -> List[dict]: + """Executor adapter: list accessibility elements as plain dicts.""" + from je_auto_control.utils.accessibility.accessibility_api import ( + list_accessibility_elements, + ) + return [ + element.to_dict() + for element in list_accessibility_elements( + app_name=app_name, max_results=int(max_results), + ) + ] + + +def _a11y_find_as_dict(name: Optional[str] = None, + role: Optional[str] = None, + app_name: Optional[str] = None) -> Optional[dict]: + """Executor adapter: find an accessibility element, return its dict.""" + element = find_accessibility_element( + name=name, role=role, app_name=app_name, + ) + return None if element is None else element.to_dict() + + +def _vlm_locate_as_list(description: str, + screen_region: Optional[List[int]] = None, + model: Optional[str] = None) -> Optional[List[int]]: + """Executor adapter: return VLM-located coords as a JSON-safe list.""" + coords = locate_by_description( + description, screen_region=screen_region, model=model, + ) + return None if coords is None else [coords[0], coords[1]] + + +def _history_list_as_dicts(limit: int = 100, + source_type: Optional[str] = None) -> List[dict]: + """Executor adapter: list run history as plain dicts (JSON-friendly).""" + rows = default_history_store.list_runs( + limit=int(limit), source_type=source_type, + ) + return [ + { + "id": r.id, "source_type": r.source_type, + "source_id": r.source_id, "script_path": r.script_path, + "started_at": r.started_at, "finished_at": r.finished_at, + "status": r.status, "error_text": r.error_text, + "duration_seconds": r.duration_seconds, + } + for r in rows + ] + + class Executor: """ Executor @@ -136,6 +195,19 @@ def __init__(self): # Clipboard "AC_clipboard_get": get_clipboard, "AC_clipboard_set": set_clipboard, + + # Run history + "AC_history_list": _history_list_as_dicts, + "AC_history_clear": default_history_store.clear, + + # Accessibility-tree widget location + "AC_a11y_list": _a11y_list_as_dicts, + "AC_a11y_find": _a11y_find_as_dict, + "AC_a11y_click": click_accessibility_element, + + # VLM-based element locator + "AC_vlm_locate": _vlm_locate_as_list, + "AC_vlm_click": click_by_description, } def known_commands(self) -> set: diff --git a/je_auto_control/utils/generate_report/generate_xml_report.py b/je_auto_control/utils/generate_report/generate_xml_report.py index c92afb2f..aac64dfc 100644 --- a/je_auto_control/utils/generate_report/generate_xml_report.py +++ b/je_auto_control/utils/generate_report/generate_xml_report.py @@ -1,6 +1,7 @@ from threading import Lock from typing import Tuple, Union -from xml.dom.minidom import parseString + +from defusedxml.minidom import parseString # nosec B405 # nosemgrep: python.lang.security.use-defused-xml.use-defused-xml # reason: defusedxml is the safe replacement from je_auto_control.utils.generate_report.generate_json_report import generate_json from je_auto_control.utils.logging.logging_instance import autocontrol_logger diff --git a/je_auto_control/utils/hotkey/backends/__init__.py b/je_auto_control/utils/hotkey/backends/__init__.py new file mode 100644 index 00000000..5e97542d --- /dev/null +++ b/je_auto_control/utils/hotkey/backends/__init__.py @@ -0,0 +1,29 @@ +"""Platform-specific hotkey daemon backends (Strategy pattern).""" +import sys + +from je_auto_control.utils.hotkey.backends.base import HotkeyBackend + + +def get_backend() -> HotkeyBackend: + """Return the backend for the current platform.""" + if sys.platform.startswith("win"): + from je_auto_control.utils.hotkey.backends.windows_backend import ( + WindowsHotkeyBackend, + ) + return WindowsHotkeyBackend() + if sys.platform == "darwin": + from je_auto_control.utils.hotkey.backends.macos_backend import ( + MacOSHotkeyBackend, + ) + return MacOSHotkeyBackend() + if sys.platform.startswith("linux"): + from je_auto_control.utils.hotkey.backends.linux_backend import ( + LinuxHotkeyBackend, + ) + return LinuxHotkeyBackend() + raise NotImplementedError( + f"HotkeyDaemon has no backend for platform {sys.platform!r}" + ) + + +__all__ = ["HotkeyBackend", "get_backend"] diff --git a/je_auto_control/utils/hotkey/backends/base.py b/je_auto_control/utils/hotkey/backends/base.py new file mode 100644 index 00000000..8952cb64 --- /dev/null +++ b/je_auto_control/utils/hotkey/backends/base.py @@ -0,0 +1,19 @@ +"""Abstract hotkey backend contract.""" +from je_auto_control.utils.hotkey.hotkey_daemon import BackendContext + + +class HotkeyBackend: + """Each backend owns a thread and listens for OS-level hotkey presses. + + Implementations must: + * poll ``context.get_bindings()`` so added / removed bindings take + effect without restarting the daemon; + * call ``context.fire(binding_id)`` from the listener thread whenever + a registered hotkey is observed; + * return as soon as ``context.stop_event`` is set. + """ + + name: str = "abstract" + + def run_forever(self, context: BackendContext) -> None: # pragma: no cover + raise NotImplementedError diff --git a/je_auto_control/utils/hotkey/backends/linux_backend.py b/je_auto_control/utils/hotkey/backends/linux_backend.py new file mode 100644 index 00000000..65e6fd55 --- /dev/null +++ b/je_auto_control/utils/hotkey/backends/linux_backend.py @@ -0,0 +1,194 @@ +"""Linux X11 hotkey backend using python-Xlib's ``XGrabKey``. + +Requires an X11 display — Wayland is not supported. The grab consumes the +key, matching Windows ``RegisterHotKey`` semantics. NumLock / CapsLock are +masked so the hotkey still fires with those toggles active. +""" +from typing import Dict, List, Optional, Tuple + +from je_auto_control.utils.hotkey.backends.base import HotkeyBackend +from je_auto_control.utils.hotkey.hotkey_daemon import ( + BackendContext, HotkeyBinding, split_combo, +) +from je_auto_control.utils.logging.logging_instance import autocontrol_logger + +_LINUX_POLL_SECONDS = 0.1 + +_KEY_NAME_TO_KEYSYM = { + "space": "space", "enter": "Return", "return": "Return", + "tab": "Tab", "escape": "Escape", "esc": "Escape", + "left": "Left", "right": "Right", "up": "Up", "down": "Down", + "home": "Home", "end": "End", "pageup": "Page_Up", + "pagedown": "Page_Down", "insert": "Insert", "delete": "Delete", + "f1": "F1", "f2": "F2", "f3": "F3", "f4": "F4", "f5": "F5", + "f6": "F6", "f7": "F7", "f8": "F8", "f9": "F9", "f10": "F10", + "f11": "F11", "f12": "F12", +} + + +def _combo_to_x11(combo: str) -> Tuple[int, int]: + """Return ``(x11_modifier_mask, keycode)`` for ``combo`` on this display.""" + from Xlib import X, XK + from Xlib import display as xdisplay + + mods, primary = split_combo(combo) + mask = 0 + if "ctrl" in mods: + mask |= X.ControlMask + if "shift" in mods: + mask |= X.ShiftMask + if "alt" in mods: + mask |= X.Mod1Mask + if "win" in mods: + mask |= X.Mod4Mask + + keysym_name = _KEY_NAME_TO_KEYSYM.get(primary.lower()) + if keysym_name is None: + if len(primary) == 1: + keysym_name = primary.lower() + else: + raise ValueError(f"unsupported hotkey key: {primary!r}") + keysym = XK.string_to_keysym(keysym_name) + if keysym == 0: + raise ValueError(f"unknown X keysym for key: {primary!r}") + disp = xdisplay.Display() + try: + return mask, disp.keysym_to_keycode(keysym) + finally: + disp.close() + + +class LinuxHotkeyBackend(HotkeyBackend): + """Grab keys via X11 and dispatch KeyPress events.""" + + name = "linux-x11" + + def __init__(self) -> None: + # binding_id -> (combo, modifier_mask, keycode) + self._registered: Dict[str, Tuple[str, int, int]] = {} + + def run_forever(self, context: BackendContext) -> None: + from Xlib import X + from Xlib import display as xdisplay + + try: + disp = xdisplay.Display() + except Exception as error: + autocontrol_logger.error("open X display failed: %r", error) + return + + root = disp.screen().root + root.change_attributes(event_mask=X.KeyPressMask) + try: + while not context.stop_event.is_set(): + self._sync(disp, root, context.get_bindings()) + self._drain(disp, context.fire) + context.stop_event.wait(_LINUX_POLL_SECONDS) + finally: + self._ungrab_all(disp, root) + disp.close() + + def _sync(self, disp, root, bindings: List[HotkeyBinding]) -> None: + current_ids = {b.binding_id for b in bindings} + self._ungrab_stale(root, current_ids) + for binding in bindings: + self._sync_one(root, binding) + disp.sync() + + def _ungrab_stale(self, root, current_ids: set) -> None: + stale_ids = [bid for bid in self._registered if bid not in current_ids] + for stale in stale_ids: + _combo, mask, keycode = self._registered.pop(stale) + self._ungrab_masked(root, keycode, mask) + + def _sync_one(self, root, binding: HotkeyBinding) -> None: + prior = self._registered.get(binding.binding_id) + if prior is not None and prior[0] == binding.combo: + return + if prior is not None: + self._registered.pop(binding.binding_id, None) + try: + mask, keycode = _combo_to_x11(binding.combo) + except ValueError as error: + autocontrol_logger.error( + "hotkey parse failed for %s: %r", binding.combo, error, + ) + return + if self._grab_masked(root, binding, mask, keycode): + self._registered[binding.binding_id] = ( + binding.combo, mask, keycode, + ) + + @staticmethod + def _ungrab_masked(root, keycode: int, mask: int) -> None: + for extra_mask in _lock_mask_variants(): + try: + root.ungrab_key(keycode, mask | extra_mask) + except Exception: # nosec B110 # noqa: BLE001 # reason: X11 ungrab races are non-fatal + pass + + @staticmethod + def _grab_masked(root, binding: HotkeyBinding, + mask: int, keycode: int) -> bool: + from Xlib import X + + for extra_mask in _lock_mask_variants(): + try: + root.grab_key( + keycode, mask | extra_mask, True, + X.GrabModeAsync, X.GrabModeAsync, + ) + except Exception as error: # noqa: BLE001 # reason: X errors + autocontrol_logger.error( + "XGrabKey failed for %s: %r", binding.combo, error, + ) + return False + return True + + def _drain(self, disp, fire: "callable") -> None: + from Xlib import X + + while disp.pending_events(): + event = disp.next_event() + if event.type != X.KeyPress: + continue + match = self._find_binding(event.detail, event.state) + if match is not None: + fire(match) + + def _find_binding(self, keycode: int, state: int) -> Optional[str]: + effective_state = state & ~_lock_all_mask() + for bid, (_combo, mask, kc) in self._registered.items(): + if kc == keycode and (state & mask) == mask \ + and effective_state == mask: + return bid + return None + + def _ungrab_all(self, disp, root) -> None: + for _combo, mask, keycode in self._registered.values(): + for extra_mask in _lock_mask_variants(): + try: + root.ungrab_key(keycode, mask | extra_mask) + except Exception: # nosec B110 # noqa: BLE001 # reason: X11 ungrab races are non-fatal + pass + self._registered.clear() + disp.sync() + + +def _lock_mask_variants() -> List[int]: + """Masks to re-register each hotkey with NumLock / CapsLock combos.""" + try: + from Xlib import X + except ImportError: + return [0] + num_lock = X.Mod2Mask + caps_lock = X.LockMask + return [0, num_lock, caps_lock, num_lock | caps_lock] + + +def _lock_all_mask() -> int: + try: + from Xlib import X + except ImportError: + return 0 + return X.Mod2Mask | X.LockMask diff --git a/je_auto_control/utils/hotkey/backends/macos_backend.py b/je_auto_control/utils/hotkey/backends/macos_backend.py new file mode 100644 index 00000000..f980a5ec --- /dev/null +++ b/je_auto_control/utils/hotkey/backends/macos_backend.py @@ -0,0 +1,166 @@ +"""macOS hotkey backend using ``CGEventTap`` from Quartz. + +Requires Accessibility permission (System Settings → Privacy & Security → +Accessibility) for the Python interpreter or the host application. Without +it ``CGEventTapCreate`` silently returns ``None`` and we log + exit. +""" +from typing import Dict, Optional, Tuple + +from je_auto_control.utils.hotkey.backends.base import HotkeyBackend +from je_auto_control.utils.hotkey.hotkey_daemon import ( + BackendContext, split_combo, +) +from je_auto_control.utils.logging.logging_instance import autocontrol_logger + +_MACOS_POLL_SECONDS = 0.1 + +# Carbon virtual key codes for non-character keys. +_KEY_NAME_TO_KEYCODE = { + "return": 36, "enter": 36, "tab": 48, "space": 49, "escape": 53, "esc": 53, + "left": 123, "right": 124, "down": 125, "up": 126, + "home": 115, "end": 119, "pageup": 116, "pagedown": 121, + "insert": 114, "delete": 117, + "f1": 122, "f2": 120, "f3": 99, "f4": 118, "f5": 96, "f6": 97, + "f7": 98, "f8": 100, "f9": 101, "f10": 109, "f11": 103, "f12": 111, +} + +# a-z -> Carbon virtual key codes. +_LETTER_KEYCODES = { + "a": 0, "s": 1, "d": 2, "f": 3, "h": 4, "g": 5, "z": 6, "x": 7, + "c": 8, "v": 9, "b": 11, "q": 12, "w": 13, "e": 14, "r": 15, + "y": 16, "t": 17, "o": 31, "u": 32, "i": 34, "p": 35, "l": 37, + "j": 38, "k": 40, "n": 45, "m": 46, +} + +# 0-9 -> Carbon virtual key codes (note the unusual ordering). +_DIGIT_KEYCODES = { + "1": 18, "2": 19, "3": 20, "4": 21, "5": 23, "6": 22, + "7": 26, "8": 28, "9": 25, "0": 29, +} + +_FLAG_SHIFT = 1 << 17 +_FLAG_CONTROL = 1 << 18 +_FLAG_ALT = 1 << 19 +_FLAG_CMD = 1 << 20 +_FLAG_MASK_ALL = _FLAG_SHIFT | _FLAG_CONTROL | _FLAG_ALT | _FLAG_CMD + + +def _primary_key_to_keycode(primary: str) -> int: + lowered = primary.lower() + if lowered in _KEY_NAME_TO_KEYCODE: + return _KEY_NAME_TO_KEYCODE[lowered] + if lowered in _LETTER_KEYCODES: + return _LETTER_KEYCODES[lowered] + if lowered in _DIGIT_KEYCODES: + return _DIGIT_KEYCODES[lowered] + raise ValueError(f"unsupported hotkey key: {primary!r}") + + +def _combo_to_macos(combo: str) -> Tuple[int, int]: + """Return ``(flags_mask, keycode)`` for ``combo`` on macOS.""" + mods, primary = split_combo(combo) + mask = 0 + if "ctrl" in mods: + mask |= _FLAG_CONTROL + if "shift" in mods: + mask |= _FLAG_SHIFT + if "alt" in mods: + mask |= _FLAG_ALT + if "win" in mods: + mask |= _FLAG_CMD + return mask, _primary_key_to_keycode(primary) + + +class MacOSHotkeyBackend(HotkeyBackend): + """Carbon CGEventTap-based listener. Consumes key events that match.""" + + name = "macos" + + def __init__(self) -> None: + # binding_id -> (combo, flags_mask, keycode) + self._registered: Dict[str, Tuple[str, int, int]] = {} + self._pending_fires: list = [] + + def run_forever(self, context: BackendContext) -> None: + try: + import Quartz # noqa: F401 # reason: verifying pyobjc present + except ImportError as error: + autocontrol_logger.error("Quartz unavailable: %r", error) + return + self._loop(context) + + def _loop(self, context: BackendContext) -> None: + import Quartz + from CoreFoundation import ( + CFRunLoopRunInMode, kCFRunLoopDefaultMode, + ) + + def tap_callback(_proxy, _event_type, event, _refcon): + keycode = Quartz.CGEventGetIntegerValueField( + event, Quartz.kCGKeyboardEventKeycode, + ) + flags = int(Quartz.CGEventGetFlags(event)) & _FLAG_MASK_ALL + hit = self._match(keycode, flags) + if hit is None: + return event + self._pending_fires.append(hit) + return None # consume + + mask = 1 << Quartz.kCGEventKeyDown + tap = Quartz.CGEventTapCreate( + Quartz.kCGHIDEventTap, Quartz.kCGHeadInsertEventTap, + Quartz.kCGEventTapOptionDefault, mask, tap_callback, None, + ) + if tap is None: + autocontrol_logger.error( + "CGEventTapCreate returned None — enable Accessibility perms", + ) + return + source = Quartz.CFMachPortCreateRunLoopSource(None, tap, 0) + run_loop = Quartz.CFRunLoopGetCurrent() + Quartz.CFRunLoopAddSource( + run_loop, source, kCFRunLoopDefaultMode, + ) + Quartz.CGEventTapEnable(tap, True) + + try: + while not context.stop_event.is_set(): + self._sync(context.get_bindings()) + CFRunLoopRunInMode( + kCFRunLoopDefaultMode, _MACOS_POLL_SECONDS, True, + ) + self._drain_fires(context.fire) + finally: + Quartz.CGEventTapEnable(tap, False) + Quartz.CFRunLoopRemoveSource( + run_loop, source, kCFRunLoopDefaultMode, + ) + + def _sync(self, bindings) -> None: + current_ids = {b.binding_id for b in bindings} + for stale in [bid for bid in self._registered if bid not in current_ids]: + self._registered.pop(stale, None) + for binding in bindings: + prior = self._registered.get(binding.binding_id) + if prior is not None and prior[0] == binding.combo: + continue + try: + mask, keycode = _combo_to_macos(binding.combo) + except ValueError as error: + autocontrol_logger.error( + "hotkey parse failed for %s: %r", binding.combo, error, + ) + continue + self._registered[binding.binding_id] = ( + binding.combo, mask, keycode, + ) + + def _match(self, keycode: int, flags: int) -> Optional[str]: + for bid, (_combo, mask, kc) in self._registered.items(): + if kc == keycode and (flags & _FLAG_MASK_ALL) == mask: + return bid + return None + + def _drain_fires(self, fire) -> None: + while self._pending_fires: + fire(self._pending_fires.pop(0)) diff --git a/je_auto_control/utils/hotkey/backends/windows_backend.py b/je_auto_control/utils/hotkey/backends/windows_backend.py new file mode 100644 index 00000000..68a6f669 --- /dev/null +++ b/je_auto_control/utils/hotkey/backends/windows_backend.py @@ -0,0 +1,95 @@ +"""Windows hotkey backend: ``RegisterHotKey`` + a message-pump thread.""" +from typing import Dict, List, Optional, Tuple + +from je_auto_control.utils.hotkey.backends.base import HotkeyBackend +from je_auto_control.utils.hotkey.hotkey_daemon import ( + BackendContext, HotkeyBinding, parse_combo, +) +from je_auto_control.utils.logging.logging_instance import autocontrol_logger + + +class WindowsHotkeyBackend(HotkeyBackend): + """Win32 backend using user32's ``RegisterHotKey``.""" + + name = "windows" + + def __init__(self) -> None: + self._id_counter = 100 + # binding_id -> (os_registration_id, combo) + self._registered: Dict[str, Tuple[int, str]] = {} + + def run_forever(self, context: BackendContext) -> None: + import ctypes + from ctypes import wintypes + + user32 = ctypes.WinDLL("user32", use_last_error=True) + user32.RegisterHotKey.argtypes = [ + wintypes.HWND, ctypes.c_int, wintypes.UINT, wintypes.UINT, + ] + user32.RegisterHotKey.restype = wintypes.BOOL + user32.UnregisterHotKey.argtypes = [wintypes.HWND, ctypes.c_int] + user32.PeekMessageW.argtypes = [ + ctypes.POINTER(wintypes.MSG), wintypes.HWND, + wintypes.UINT, wintypes.UINT, wintypes.UINT, + ] + user32.PeekMessageW.restype = wintypes.BOOL + + msg = wintypes.MSG() + wm_hotkey = 0x0312 + pm_remove = 0x0001 + try: + while not context.stop_event.is_set(): + self._sync(user32, context.get_bindings()) + while user32.PeekMessageW( + ctypes.byref(msg), None, 0, 0, pm_remove, + ): + if msg.message == wm_hotkey: + self._dispatch(msg.wParam, context.fire) + context.stop_event.wait(0.05) + finally: + for reg_id, _ in self._registered.values(): + user32.UnregisterHotKey(None, reg_id) + self._registered.clear() + + def _sync(self, user32, bindings: List[HotkeyBinding]) -> None: + """Add new bindings + drop removed ones vs. ``self._registered``.""" + current_ids = {b.binding_id for b in bindings} + for stale_id in [bid for bid in self._registered if bid not in current_ids]: + reg_id, _combo = self._registered.pop(stale_id) + user32.UnregisterHotKey(None, reg_id) + for binding in bindings: + existing = self._registered.get(binding.binding_id) + if existing is not None and existing[1] == binding.combo: + continue + if existing is not None: + user32.UnregisterHotKey(None, existing[0]) + self._registered.pop(binding.binding_id, None) + self._try_register(user32, binding) + + def _try_register(self, user32, binding: HotkeyBinding) -> None: + try: + modifiers, vk = parse_combo(binding.combo) + except ValueError as error: + autocontrol_logger.error( + "hotkey parse failed for %s: %r", binding.combo, error, + ) + return + self._id_counter += 1 + reg_id = self._id_counter + if user32.RegisterHotKey(None, reg_id, modifiers, vk): + self._registered[binding.binding_id] = (reg_id, binding.combo) + else: + autocontrol_logger.error( + "RegisterHotKey failed for %s (%s)", + binding.combo, binding.binding_id, + ) + + def _dispatch(self, registered_id: int, + fire: "callable") -> None: + match: Optional[str] = None + for bid, (reg_id, _combo) in self._registered.items(): + if reg_id == registered_id: + match = bid + break + if match is not None: + fire(match) diff --git a/je_auto_control/utils/hotkey/hotkey_daemon.py b/je_auto_control/utils/hotkey/hotkey_daemon.py index 574f664b..8ce838b4 100644 --- a/je_auto_control/utils/hotkey/hotkey_daemon.py +++ b/je_auto_control/utils/hotkey/hotkey_daemon.py @@ -1,8 +1,9 @@ -"""Global hotkey daemon. +"""Global hotkey daemon with pluggable platform backends. -Windows implementation uses ``RegisterHotKey`` + a dedicated message pump -thread. macOS / Linux raise ``NotImplementedError`` for now — the Strategy -pattern keeps the public API stable so backends can be added later. +Public API (``bind``, ``unbind``, ``start``, ``stop``, ``list_bindings``) is +unchanged. Backends register their OS listener in ``backends/`` and the +daemon picks one via ``get_backend()``. Windows uses ``RegisterHotKey``, +Linux uses X11 ``XGrabKey``, macOS uses ``CGEventTap``. Usage:: @@ -10,15 +11,19 @@ default_hotkey_daemon.bind("ctrl+alt+1", "scripts/greet.json") default_hotkey_daemon.start() """ -import sys import threading import uuid from dataclasses import dataclass -from typing import Callable, Dict, List, Optional, Tuple +from typing import Callable, Dict, FrozenSet, List, Optional, Tuple from je_auto_control.utils.json.json_file import read_action_json from je_auto_control.utils.logging.logging_instance import autocontrol_logger - +from je_auto_control.utils.run_history.artifact_manager import ( + capture_error_snapshot, +) +from je_auto_control.utils.run_history.history_store import ( + SOURCE_HOTKEY, STATUS_ERROR, STATUS_OK, default_history_store, +) MOD_ALT = 0x0001 MOD_CONTROL = 0x0002 @@ -33,6 +38,8 @@ "win": MOD_WIN, "super": MOD_WIN, "meta": MOD_WIN, } +_MODIFIER_NAMES = frozenset(_MODIFIER_MAP.keys()) + @dataclass class HotkeyBinding: @@ -44,25 +51,42 @@ class HotkeyBinding: fired: int = 0 -def parse_combo(combo: str) -> Tuple[int, int]: - """Parse ``"ctrl+alt+1"`` into ``(modifiers, virtual_key_code)``.""" +def split_combo(combo: str) -> Tuple[FrozenSet[str], str]: + """Return ``(canonical modifier names, primary key name)``. + + Modifier names are canonical (``ctrl``/``alt``/``shift``/``win``) — + aliases like ``control``/``super``/``meta`` are normalised. + """ if not combo or not combo.strip(): raise ValueError("hotkey combo is empty") parts = [p.strip().lower() for p in combo.split("+") if p.strip()] if not parts: raise ValueError(f"invalid hotkey combo: {combo!r}") - modifiers = 0 + mods: set = set() key_part: Optional[str] = None for part in parts: - if part in _MODIFIER_MAP: - modifiers |= _MODIFIER_MAP[part] - else: - if key_part is not None: - raise ValueError(f"hotkey {combo!r} has multiple non-modifier keys") - key_part = part + if part in _MODIFIER_NAMES: + mods.add(_canonical_mod(part)) + continue + if key_part is not None: + raise ValueError(f"hotkey {combo!r} has multiple non-modifier keys") + key_part = part if key_part is None: raise ValueError(f"hotkey {combo!r} is missing a primary key") - return modifiers | MOD_NOREPEAT, _key_to_vk(key_part) + return frozenset(mods), key_part + + +def _canonical_mod(name: str) -> str: + return {"control": "ctrl", "super": "win", "meta": "win"}.get(name, name) + + +def parse_combo(combo: str) -> Tuple[int, int]: + """Windows-flavoured parser: ``(modifiers_bitmask, virtual_key)``.""" + mods, key = split_combo(combo) + modifiers = MOD_NOREPEAT + for mod in mods: + modifiers |= _MODIFIER_MAP[mod] + return modifiers, _key_to_vk(key) def _key_to_vk(key: str) -> int: @@ -84,6 +108,14 @@ def _key_to_vk(key: str) -> int: raise ValueError(f"unsupported hotkey key: {key!r}") +@dataclass +class BackendContext: + """Data the daemon hands to a backend thread.""" + stop_event: threading.Event + get_bindings: Callable[[], List[HotkeyBinding]] + fire: Callable[[str], None] + + class HotkeyDaemon: """Register OS-level hotkeys and run their action JSON on trigger.""" @@ -95,21 +127,17 @@ def __init__(self, self._lock = threading.Lock() self._thread: Optional[threading.Thread] = None self._stop = threading.Event() - self._pending_register: List[HotkeyBinding] = [] - self._id_counter = 100 - self._registered_ids: Dict[str, int] = {} def bind(self, combo: str, script_path: str, binding_id: Optional[str] = None) -> HotkeyBinding: """Register a hotkey → script binding. Safe to call before/after start.""" - parse_combo(combo) + split_combo(combo) bid = binding_id or uuid.uuid4().hex[:8] binding = HotkeyBinding( binding_id=bid, combo=combo, script_path=script_path, ) with self._lock: self._bindings[bid] = binding - self._pending_register.append(binding) return binding def unbind(self, binding_id: str) -> bool: @@ -120,16 +148,22 @@ def list_bindings(self) -> List[HotkeyBinding]: with self._lock: return list(self._bindings.values()) + _snapshot = list_bindings + def start(self) -> None: if self._thread is not None and self._thread.is_alive(): return - if not sys.platform.startswith("win"): - raise NotImplementedError( - "HotkeyDaemon currently supports Windows only" - ) + from je_auto_control.utils.hotkey.backends import get_backend + backend = get_backend() + context = BackendContext( + stop_event=self._stop, + get_bindings=self._snapshot, + fire=self._fire_binding, + ) self._stop.clear() self._thread = threading.Thread( - target=self._run_win, daemon=True, name="AutoControlHotkey", + target=backend.run_forever, args=(context,), + daemon=True, name=f"AutoControlHotkey-{backend.name}", ) self._thread.start() @@ -139,72 +173,30 @@ def stop(self, timeout: float = 2.0) -> None: self._thread.join(timeout=timeout) self._thread = None - # --- Windows backend ----------------------------------------------------- - - def _run_win(self) -> None: - import ctypes - from ctypes import wintypes - - user32 = ctypes.WinDLL("user32", use_last_error=True) - user32.RegisterHotKey.argtypes = [ - wintypes.HWND, ctypes.c_int, wintypes.UINT, wintypes.UINT, - ] - user32.RegisterHotKey.restype = wintypes.BOOL - user32.UnregisterHotKey.argtypes = [wintypes.HWND, ctypes.c_int] - user32.PeekMessageW.argtypes = [ - ctypes.POINTER(wintypes.MSG), wintypes.HWND, - wintypes.UINT, wintypes.UINT, wintypes.UINT, - ] - user32.PeekMessageW.restype = wintypes.BOOL - - msg = wintypes.MSG() - wm_hotkey = 0x0312 - pm_remove = 0x0001 - - self._drain_pending(user32) - while not self._stop.is_set(): - self._drain_pending(user32) - while user32.PeekMessageW(ctypes.byref(msg), None, 0, 0, pm_remove): - if msg.message == wm_hotkey: - self._handle_win_hotkey(msg.wParam) - self._stop.wait(0.05) - - # cleanup registrations - for registered_id in list(self._registered_ids.values()): - user32.UnregisterHotKey(None, registered_id) - self._registered_ids.clear() - - def _drain_pending(self, user32) -> None: - with self._lock: - pending = list(self._pending_register) - self._pending_register.clear() - for binding in pending: - modifiers, vk = parse_combo(binding.combo) - self._id_counter += 1 - reg_id = self._id_counter - if user32.RegisterHotKey(None, reg_id, modifiers, vk): - self._registered_ids[binding.binding_id] = reg_id - else: - autocontrol_logger.error( - "RegisterHotKey failed for %s (%s)", - binding.combo, binding.binding_id, - ) - - def _handle_win_hotkey(self, registered_id: int) -> None: - match: Optional[HotkeyBinding] = None + def _fire_binding(self, binding_id: str) -> None: with self._lock: - for bid, reg_id in self._registered_ids.items(): - if reg_id == registered_id: - match = self._bindings.get(bid) - break + match = self._bindings.get(binding_id) if match is None or not match.enabled: return + run_id = default_history_store.start_run( + SOURCE_HOTKEY, match.binding_id, match.script_path, + ) + status = STATUS_OK + error_text: Optional[str] = None try: actions = read_action_json(match.script_path) self._execute(actions) except (OSError, ValueError, RuntimeError) as error: + status = STATUS_ERROR + error_text = repr(error) autocontrol_logger.error("hotkey %s failed: %r", match.combo, error) + finally: + artifact = (capture_error_snapshot(run_id) + if status == STATUS_ERROR else None) + default_history_store.finish_run( + run_id, status, error_text, artifact_path=artifact, + ) match.fired += 1 diff --git a/je_auto_control/utils/package_manager/package_manager_class.py b/je_auto_control/utils/package_manager/package_manager_class.py index 9e145383..c1a28df3 100644 --- a/je_auto_control/utils/package_manager/package_manager_class.py +++ b/je_auto_control/utils/package_manager/package_manager_class.py @@ -1,4 +1,5 @@ import importlib +import re from importlib.util import find_spec from inspect import getmembers, isfunction, isbuiltin, isclass from types import ModuleType @@ -6,6 +7,8 @@ from je_auto_control.utils.logging.logging_instance import autocontrol_logger +_PACKAGE_NAME_RE = re.compile(r"^[A-Za-z_]\w*(\.[A-Za-z_]\w*)*$") + class PackageManager: """ @@ -25,13 +28,17 @@ def check_package(self, package: str) -> Optional[ModuleType]: 檢查並載入套件 Check and import package - :param package: 套件名稱 Package name + :param package: 套件名稱 Package name (must match a Python dotted identifier) :return: 套件模組 ModuleType 或 None """ + if not isinstance(package, str) or not _PACKAGE_NAME_RE.match(package): + autocontrol_logger.error("rejected invalid package name: %r", package) + return None if package not in self.installed_package_dict: found_spec = find_spec(package) if found_spec is not None: try: + # nosemgrep: python.lang.security.audit.non-literal-import.non-literal-import installed_package = importlib.import_module(found_spec.name) self.installed_package_dict[found_spec.name] = installed_package except ModuleNotFoundError as error: diff --git a/je_auto_control/utils/project/create_project_structure.py b/je_auto_control/utils/project/create_project_structure.py index 26f1474b..59e9bcc3 100644 --- a/je_auto_control/utils/project/create_project_structure.py +++ b/je_auto_control/utils/project/create_project_structure.py @@ -12,6 +12,7 @@ ) _project_lock = Lock() +_TEMPLATE_PLACEHOLDER = "{temp}" # noqa: S101 # reason: literal sentinel substituted in templates def create_dir(dir_name: str) -> None: @@ -61,15 +62,15 @@ def create_template(parent_name: str, project_path: str = None) -> None: if executor_dir_path.exists() and executor_dir_path.is_dir(): _write_file( executor_dir_path / "executor_one_file.py", - executor_template_1.replace("{temp}", str(keyword_dir_path / "keyword1.json")) + executor_template_1.replace(_TEMPLATE_PLACEHOLDER, str(keyword_dir_path / "keyword1.json")) ) _write_file( executor_dir_path / "executor_bad_file.py", - bad_executor_template_1.replace("{temp}", str(keyword_dir_path / "bad_keyword_1.json")) + bad_executor_template_1.replace(_TEMPLATE_PLACEHOLDER, str(keyword_dir_path / "bad_keyword_1.json")) ) _write_file( executor_dir_path / "executor_folder.py", - executor_template_2.replace("{temp}", str(keyword_dir_path)) + executor_template_2.replace(_TEMPLATE_PLACEHOLDER, str(keyword_dir_path)) ) diff --git a/je_auto_control/utils/rest_api/rest_server.py b/je_auto_control/utils/rest_api/rest_server.py index bd59f07a..237fd080 100644 --- a/je_auto_control/utils/rest_api/rest_server.py +++ b/je_auto_control/utils/rest_api/rest_server.py @@ -12,9 +12,11 @@ import json import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple +from urllib.parse import parse_qs, urlparse from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.run_history.history_store import default_history_store class _JSONHandler(BaseHTTPRequestHandler): @@ -28,17 +30,26 @@ def log_message(self, fmt: str, *args: Any) -> None: self.address_string(), fmt % args) def do_GET(self) -> None: # noqa: N802 # reason: stdlib API - if self.path == "/health": + parsed = urlparse(self.path) + if parsed.path == "/health": self._send_json({"status": "ok"}) return - if self.path == "/jobs": + if parsed.path == "/jobs": self._send_json({"jobs": self._serialize_jobs()}) return - self._send_json({"error": f"unknown path: {self.path}"}, status=404) + if parsed.path == "/history": + self._send_json( + {"runs": self._serialize_history(parsed.query)}, + default=str, + ) + return + autocontrol_logger.info("rest-api unknown GET path: %r", self.path) + self._send_json({"error": "unknown path"}, status=404) def do_POST(self) -> None: # noqa: N802 # reason: stdlib API if self.path != "/execute": - self._send_json({"error": f"unknown path: {self.path}"}, status=404) + autocontrol_logger.info("rest-api unknown POST path: %r", self.path) + self._send_json({"error": "unknown path"}, status=404) return payload = self._read_json_body() if payload is None: @@ -51,7 +62,8 @@ def do_POST(self) -> None: # noqa: N802 # reason: stdlib API from je_auto_control.utils.executor.action_executor import execute_action result = execute_action(actions) except (OSError, RuntimeError, ValueError, TypeError) as error: - self._send_json({"error": repr(error)}, status=500) + autocontrol_logger.error("rest-api execute_action failed: %r", error) + self._send_json({"error": "execute_action failed"}, status=500) return self._send_json({"result": result}, default=str) @@ -65,8 +77,9 @@ def _read_json_body(self) -> Optional[Any]: raw = self.rfile.read(length) try: return json.loads(raw.decode("utf-8")) - except (ValueError, UnicodeDecodeError) as error: - self._send_json({"error": f"invalid JSON: {error}"}, status=400) + except ValueError as error: + autocontrol_logger.info("rest-api invalid JSON body: %r", error) + self._send_json({"error": "invalid JSON"}, status=400) return None def _send_json(self, payload: Dict[str, Any], status: int = 200, @@ -91,6 +104,31 @@ def _serialize_jobs() -> list: for job in default_scheduler.list_jobs() ] + @staticmethod + def _serialize_history(query: str) -> List[Dict[str, Any]]: + params = parse_qs(query) + try: + limit = int(params.get("limit", ["100"])[0]) + except ValueError: + limit = 100 + source_type = params.get("source_type", [None])[0] or None + try: + rows = default_history_store.list_runs( + limit=limit, source_type=source_type, + ) + except ValueError: + return [] + return [ + { + "id": r.id, "source_type": r.source_type, + "source_id": r.source_id, "script_path": r.script_path, + "started_at": r.started_at, "finished_at": r.finished_at, + "status": r.status, "error_text": r.error_text, + "duration_seconds": r.duration_seconds, + } + for r in rows + ] + class RestApiServer: """Thin wrapper that owns the HTTP server + its background thread.""" diff --git a/je_auto_control/utils/run_history/__init__.py b/je_auto_control/utils/run_history/__init__.py new file mode 100644 index 00000000..67825bd0 --- /dev/null +++ b/je_auto_control/utils/run_history/__init__.py @@ -0,0 +1,6 @@ +"""Run history package.""" +from je_auto_control.utils.run_history.history_store import ( + HistoryStore, RunRecord, default_history_store, +) + +__all__ = ["HistoryStore", "RunRecord", "default_history_store"] diff --git a/je_auto_control/utils/run_history/artifact_manager.py b/je_auto_control/utils/run_history/artifact_manager.py new file mode 100644 index 00000000..ee5fb46e --- /dev/null +++ b/je_auto_control/utils/run_history/artifact_manager.py @@ -0,0 +1,54 @@ +"""Capture error-time screenshots and attach them to run-history rows. + +The snapshot is stored under ``~/.je_auto_control/artifacts/`` and its path +is written back into the ``runs.artifact_path`` column so the GUI / REST +surfaces can surface it alongside the failure reason. +""" +import time +from pathlib import Path +from typing import Optional + +from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.run_history.history_store import ( + HistoryStore, default_history_store, +) + +_ARTIFACTS_DIRNAME = "artifacts" + + +def default_artifacts_dir() -> Path: + """Return the per-user directory that holds error-time snapshots.""" + return Path.home() / ".je_auto_control" / _ARTIFACTS_DIRNAME + + +def capture_error_snapshot(run_id: int, + artifacts_dir: Optional[Path] = None, + store: Optional[HistoryStore] = None, + ) -> Optional[str]: + """Screenshot the full screen and attach the file to ``run_id``. + + Returns the absolute file path on success, ``None`` if the capture + failed (no display, missing backend, disk error). Errors are + swallowed and logged — a crashing artifact step must not mask the + original failure the caller is trying to record. + """ + target_dir = Path(artifacts_dir) if artifacts_dir is not None \ + else default_artifacts_dir() + target = target_dir / f"run_{int(run_id)}_{int(time.time() * 1000)}.png" + try: + target_dir.mkdir(parents=True, exist_ok=True) + from je_auto_control.wrapper.auto_control_screen import screenshot + screenshot(str(target)) + except (OSError, ValueError, RuntimeError) as error: + autocontrol_logger.warning( + "error-snapshot for run %d failed: %r", int(run_id), error, + ) + return None + if not target.exists(): + return None + bound_store = store if store is not None else default_history_store + bound_store.attach_artifact(int(run_id), str(target)) + return str(target) + + +__all__ = ["capture_error_snapshot", "default_artifacts_dir"] diff --git a/je_auto_control/utils/run_history/history_store.py b/je_auto_control/utils/run_history/history_store.py new file mode 100644 index 00000000..b02db3a9 --- /dev/null +++ b/je_auto_control/utils/run_history/history_store.py @@ -0,0 +1,281 @@ +"""Thread-safe SQLite-backed run history. + +Records every fire of a scheduled job, trigger, or hotkey binding so the GUI +and REST API can show "what ran, when, and whether it succeeded". The store +is intentionally minimal — no retention policy, no analytics — callers drive +pruning via :meth:`clear` or :meth:`prune`. +""" +import os +import sqlite3 +import threading +import time +from dataclasses import dataclass +from pathlib import Path +from typing import List, Optional, Union + +from je_auto_control.utils.logging.logging_instance import autocontrol_logger + +SOURCE_SCHEDULER = "scheduler" +SOURCE_TRIGGER = "trigger" +SOURCE_HOTKEY = "hotkey" +SOURCE_MANUAL = "manual" +SOURCE_REST = "rest" + +STATUS_RUNNING = "running" +STATUS_OK = "ok" +STATUS_ERROR = "error" + +_IN_MEMORY_DB = ":memory:" + +_VALID_SOURCES = frozenset({ + SOURCE_SCHEDULER, SOURCE_TRIGGER, SOURCE_HOTKEY, + SOURCE_MANUAL, SOURCE_REST, +}) +_VALID_STATUSES = frozenset({STATUS_RUNNING, STATUS_OK, STATUS_ERROR}) + +_SCHEMA = """ +CREATE TABLE IF NOT EXISTS runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_type TEXT NOT NULL, + source_id TEXT NOT NULL, + script_path TEXT NOT NULL, + started_at REAL NOT NULL, + finished_at REAL, + status TEXT NOT NULL, + error_text TEXT, + artifact_path TEXT +); +CREATE INDEX IF NOT EXISTS idx_runs_started_at ON runs(started_at DESC); +CREATE INDEX IF NOT EXISTS idx_runs_source ON runs(source_type, source_id); +""" + + +@dataclass +class RunRecord: + """One row of the ``runs`` table.""" + id: int + source_type: str + source_id: str + script_path: str + started_at: float + finished_at: Optional[float] + status: str + error_text: Optional[str] + artifact_path: Optional[str] = None + + @property + def duration_seconds(self) -> Optional[float]: + """End-to-end duration or ``None`` if the run is still in-flight.""" + if self.finished_at is None: + return None + return max(0.0, self.finished_at - self.started_at) + + +def _default_history_path() -> Path: + """Return the per-user cache path for the default store.""" + return Path.home() / ".je_auto_control" / "run_history.sqlite" + + +def _validate_source(source_type: str) -> None: + if source_type not in _VALID_SOURCES: + raise ValueError( + f"invalid source_type {source_type!r}; " + f"expected one of {sorted(_VALID_SOURCES)}" + ) + + +def _validate_status(status: str) -> None: + if status not in _VALID_STATUSES: + raise ValueError( + f"invalid status {status!r}; expected one of {sorted(_VALID_STATUSES)}" + ) + + +class HistoryStore: + """SQLite-backed run log. Safe to share across threads.""" + + def __init__(self, path: Union[str, Path] = _IN_MEMORY_DB) -> None: + self._path = str(path) if path == _IN_MEMORY_DB else str(Path(path)) + if self._path != _IN_MEMORY_DB: + os.makedirs(os.path.dirname(self._path) or ".", exist_ok=True) + self._lock = threading.Lock() + self._conn = sqlite3.connect( + self._path, check_same_thread=False, isolation_level=None, + ) + self._conn.row_factory = sqlite3.Row + self._conn.executescript(_SCHEMA) + self._migrate_schema() + + def _migrate_schema(self) -> None: + """Add columns that older store files are missing.""" + cols = {row["name"] for row in self._conn.execute( + "PRAGMA table_info(runs)", + ).fetchall()} + if "artifact_path" not in cols: + self._conn.execute( + "ALTER TABLE runs ADD COLUMN artifact_path TEXT", + ) + + @property + def path(self) -> str: + return self._path + + def start_run(self, source_type: str, source_id: str, + script_path: str, started_at: Optional[float] = None, + ) -> int: + """Record a run that has just begun; return its row id.""" + _validate_source(source_type) + ts = float(started_at) if started_at is not None else time.time() + with self._lock: + cursor = self._conn.execute( + "INSERT INTO runs (source_type, source_id, script_path," + " started_at, status) VALUES (?, ?, ?, ?, ?)", + (source_type, source_id, script_path, ts, STATUS_RUNNING), + ) + return int(cursor.lastrowid) + + def finish_run(self, run_id: int, status: str, + error_text: Optional[str] = None, + finished_at: Optional[float] = None, + artifact_path: Optional[str] = None) -> bool: + """Update a pending run with its final status; return False if unknown.""" + _validate_status(status) + if status == STATUS_RUNNING: + raise ValueError("cannot finish a run with status=running") + ts = float(finished_at) if finished_at is not None else time.time() + with self._lock: + cursor = self._conn.execute( + "UPDATE runs SET finished_at = ?, status = ?, error_text = ?," + " artifact_path = ? WHERE id = ?", + (ts, status, error_text, artifact_path, int(run_id)), + ) + return cursor.rowcount > 0 + + def attach_artifact(self, run_id: int, artifact_path: str) -> bool: + """Attach or replace the artifact path on a finished run.""" + with self._lock: + cursor = self._conn.execute( + "UPDATE runs SET artifact_path = ? WHERE id = ?", + (artifact_path, int(run_id)), + ) + return cursor.rowcount > 0 + + def list_runs(self, limit: int = 100, + source_type: Optional[str] = None, + ) -> List[RunRecord]: + """Return the most recent runs (newest first).""" + if limit <= 0: + return [] + bound_limit = int(limit) + if source_type is None: + with self._lock: + rows = self._conn.execute( + "SELECT * FROM runs " + "ORDER BY started_at DESC LIMIT ?", + (bound_limit,), + ).fetchall() + else: + _validate_source(source_type) + with self._lock: + rows = self._conn.execute( + "SELECT * FROM runs WHERE source_type = ? " + "ORDER BY started_at DESC LIMIT ?", + (source_type, bound_limit), + ).fetchall() + return [_row_to_record(row) for row in rows] + + def get_run(self, run_id: int) -> Optional[RunRecord]: + """Return a specific row or ``None`` if absent.""" + with self._lock: + row = self._conn.execute( + "SELECT * FROM runs WHERE id = ?", (int(run_id),), + ).fetchone() + return _row_to_record(row) if row is not None else None + + def count(self, source_type: Optional[str] = None) -> int: + """Return the number of rows, optionally filtered by source.""" + if source_type is not None: + _validate_source(source_type) + with self._lock: + row = self._conn.execute( + "SELECT COUNT(*) FROM runs WHERE source_type = ?", + (source_type,), + ).fetchone() + else: + with self._lock: + row = self._conn.execute("SELECT COUNT(*) FROM runs").fetchone() + return int(row[0]) + + def clear(self) -> int: + """Delete every row (and its artifact file); return rows removed.""" + with self._lock: + paths = [r[0] for r in self._conn.execute( + "SELECT artifact_path FROM runs WHERE artifact_path IS NOT NULL", + ).fetchall()] + cursor = self._conn.execute("DELETE FROM runs") + removed = int(cursor.rowcount) + _remove_artifact_files(paths) + return removed + + def prune(self, keep_latest: int) -> int: + """Keep only the newest ``keep_latest`` rows; delete the rest.""" + if keep_latest < 0: + raise ValueError("keep_latest must be >= 0") + with self._lock: + paths = [r[0] for r in self._conn.execute( + "SELECT artifact_path FROM runs WHERE artifact_path IS NOT NULL" + " AND id NOT IN (" + "SELECT id FROM runs ORDER BY started_at DESC LIMIT ?)", + (int(keep_latest),), + ).fetchall()] + cursor = self._conn.execute( + "DELETE FROM runs WHERE id NOT IN (" + "SELECT id FROM runs ORDER BY started_at DESC LIMIT ?" + ")", + (int(keep_latest),), + ) + removed = int(cursor.rowcount) + _remove_artifact_files(paths) + return removed + + def close(self) -> None: + with self._lock: + try: + self._conn.close() + except sqlite3.Error as error: + autocontrol_logger.warning("history close failed: %r", error) + + +def _row_to_record(row: sqlite3.Row) -> RunRecord: + artifact = row["artifact_path"] if "artifact_path" in row.keys() else None + return RunRecord( + id=int(row["id"]), + source_type=str(row["source_type"]), + source_id=str(row["source_id"]), + script_path=str(row["script_path"]), + started_at=float(row["started_at"]), + finished_at=(float(row["finished_at"]) + if row["finished_at"] is not None else None), + status=str(row["status"]), + error_text=(str(row["error_text"]) + if row["error_text"] is not None else None), + artifact_path=str(artifact) if artifact is not None else None, + ) + + +def _remove_artifact_files(paths) -> None: + """Best-effort delete of artifact files; ignore missing entries.""" + for raw in paths: + if not raw: + continue + try: + Path(raw).unlink() + except FileNotFoundError: + continue + except OSError as error: + autocontrol_logger.warning( + "failed to remove artifact %r: %r", raw, error, + ) + + +default_history_store = HistoryStore(path=_default_history_path()) diff --git a/je_auto_control/utils/scheduler/scheduler.py b/je_auto_control/utils/scheduler/scheduler.py index df7db6ee..abf9dd7b 100644 --- a/je_auto_control/utils/scheduler/scheduler.py +++ b/je_auto_control/utils/scheduler/scheduler.py @@ -12,6 +12,12 @@ from je_auto_control.utils.json.json_file import read_action_json from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.run_history.artifact_manager import ( + capture_error_snapshot, +) +from je_auto_control.utils.run_history.history_store import ( + SOURCE_SCHEDULER, STATUS_ERROR, STATUS_OK, default_history_store, +) from je_auto_control.utils.scheduler.cron import ( CronExpression, next_match, parse_cron, ) @@ -151,12 +157,25 @@ def _tick_once(self) -> None: self._fire(job, now_mono, now_wall) def _fire(self, job: ScheduledJob, now_mono: float, now_wall: float) -> None: + run_id = default_history_store.start_run( + SOURCE_SCHEDULER, job.job_id, job.script_path, + ) + status = STATUS_OK + error_text: Optional[str] = None try: actions = read_action_json(job.script_path) self._execute(actions) except (OSError, ValueError, RuntimeError) as error: + status = STATUS_ERROR + error_text = repr(error) autocontrol_logger.error("scheduler job %s failed: %r", job.job_id, error) + finally: + artifact = (capture_error_snapshot(run_id) + if status == STATUS_ERROR else None) + default_history_store.finish_run( + run_id, status, error_text, artifact_path=artifact, + ) with self._lock: live = self._jobs.get(job.job_id) if live is None: diff --git a/je_auto_control/utils/script_vars/interpolate.py b/je_auto_control/utils/script_vars/interpolate.py index baba464b..cc0103d4 100644 --- a/je_auto_control/utils/script_vars/interpolate.py +++ b/je_auto_control/utils/script_vars/interpolate.py @@ -12,7 +12,7 @@ from pathlib import Path from typing import Any, Mapping, MutableMapping -_PLACEHOLDER = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}") +_PLACEHOLDER = re.compile(r"\$\{([A-Za-z_]\w*)\}") def interpolate_value(value: Any, variables: Mapping[str, Any]) -> Any: diff --git a/je_auto_control/utils/shell_process/shell_exec.py b/je_auto_control/utils/shell_process/shell_exec.py index 67d236dc..fc2e11c2 100644 --- a/je_auto_control/utils/shell_process/shell_exec.py +++ b/je_auto_control/utils/shell_process/shell_exec.py @@ -1,6 +1,6 @@ import queue import shlex -import subprocess +import subprocess # nosec B404 # reason: ShellManager intentionally invokes user-supplied subprocesses without shell import sys from threading import Thread from typing import List, Union diff --git a/je_auto_control/utils/socket_server/auto_control_socket_server.py b/je_auto_control/utils/socket_server/auto_control_socket_server.py index 61106806..93a7be61 100644 --- a/je_auto_control/utils/socket_server/auto_control_socket_server.py +++ b/je_auto_control/utils/socket_server/auto_control_socket_server.py @@ -19,12 +19,12 @@ def handle(self) -> None: else: try: execute_str = json.loads(command_string) - for _, execute_return in execute_action(execute_str).items(): + for execute_return in execute_action(execute_str).values(): socket.sendall(str(execute_return).encode("utf-8")) socket.sendall("\n".encode("utf-8")) socket.sendall("Return_Data_Over_JE".encode("utf-8")) socket.sendall("\n".encode("utf-8")) - except (json.JSONDecodeError, ValueError, RuntimeError) as error: + except (ValueError, RuntimeError) as error: autocontrol_logger.error("socket command failed: %r", error) try: socket.sendall(str(error).encode("utf-8")) diff --git a/je_auto_control/utils/triggers/trigger_engine.py b/je_auto_control/utils/triggers/trigger_engine.py index c9a116c9..073e9bb5 100644 --- a/je_auto_control/utils/triggers/trigger_engine.py +++ b/je_auto_control/utils/triggers/trigger_engine.py @@ -16,6 +16,12 @@ from je_auto_control.utils.json.json_file import read_action_json from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.run_history.artifact_manager import ( + capture_error_snapshot, +) +from je_auto_control.utils.run_history.history_store import ( + SOURCE_TRIGGER, STATUS_ERROR, STATUS_OK, default_history_store, +) @dataclass @@ -62,7 +68,7 @@ def is_fired(self) -> bool: try: return find_window(self.title_substring, case_sensitive=self.case_sensitive) is not None - except (NotImplementedError, OSError, RuntimeError): + except (OSError, RuntimeError): return False @@ -174,12 +180,25 @@ def _poll_once(self) -> None: self._fire(trigger, now) def _fire(self, trigger: _TriggerBase, now: float) -> None: + run_id = default_history_store.start_run( + SOURCE_TRIGGER, trigger.trigger_id, trigger.script_path, + ) + status = STATUS_OK + error_text: Optional[str] = None try: actions = read_action_json(trigger.script_path) self._execute(actions) except (OSError, ValueError, RuntimeError) as error: + status = STATUS_ERROR + error_text = repr(error) autocontrol_logger.error("trigger %s failed: %r", trigger.trigger_id, error) + finally: + artifact = (capture_error_snapshot(run_id) + if status == STATUS_ERROR else None) + default_history_store.finish_run( + run_id, status, error_text, artifact_path=artifact, + ) with self._lock: live = self._triggers.get(trigger.trigger_id) if live is None: diff --git a/je_auto_control/utils/vision/__init__.py b/je_auto_control/utils/vision/__init__.py new file mode 100644 index 00000000..f8a3b505 --- /dev/null +++ b/je_auto_control/utils/vision/__init__.py @@ -0,0 +1,10 @@ +"""AI-vision element locator (VLM-backed).""" +from je_auto_control.utils.vision.vlm_api import ( + VLMNotAvailableError, click_by_description, locate_by_description, +) + +__all__ = [ + "VLMNotAvailableError", + "locate_by_description", + "click_by_description", +] diff --git a/je_auto_control/utils/vision/backends/__init__.py b/je_auto_control/utils/vision/backends/__init__.py new file mode 100644 index 00000000..64e4f7df --- /dev/null +++ b/je_auto_control/utils/vision/backends/__init__.py @@ -0,0 +1,70 @@ +"""VLM backend factory.""" +import os +from typing import Optional + +from je_auto_control.utils.vision.backends.base import ( + VLMBackend, VLMNotAvailableError, +) +from je_auto_control.utils.vision.backends.null_backend import NullVLMBackend + +_cached_backend: Optional[VLMBackend] = None + + +def get_backend() -> VLMBackend: + """Return (and cache) a VLM backend chosen by env vars.""" + global _cached_backend + if _cached_backend is not None: + return _cached_backend + _cached_backend = _build_backend() + return _cached_backend + + +def reset_backend_cache() -> None: + """Force ``get_backend()`` to re-detect on its next call.""" + global _cached_backend + _cached_backend = None + + +def _build_backend() -> VLMBackend: + preferred = os.environ.get("AUTOCONTROL_VLM_BACKEND", "").lower() + order = _preference_order(preferred) + for candidate in order: + backend = _try_build(candidate) + if backend is not None and backend.available: + return backend + return NullVLMBackend( + "no VLM backend ready; set ANTHROPIC_API_KEY or OPENAI_API_KEY " + "and install the matching SDK (anthropic / openai)", + ) + + +def _preference_order(preferred: str): + if preferred == "anthropic": + return ("anthropic", "openai") + if preferred == "openai": + return ("openai", "anthropic") + if os.environ.get("ANTHROPIC_API_KEY"): + return ("anthropic", "openai") + if os.environ.get("OPENAI_API_KEY"): + return ("openai", "anthropic") + return ("anthropic", "openai") + + +def _try_build(name: str): + if name == "anthropic": + from je_auto_control.utils.vision.backends.anthropic_backend import ( + AnthropicVLMBackend, + ) + return AnthropicVLMBackend() + if name == "openai": + from je_auto_control.utils.vision.backends.openai_backend import ( + OpenAIVLMBackend, + ) + return OpenAIVLMBackend() + return None + + +__all__ = [ + "VLMBackend", "VLMNotAvailableError", "NullVLMBackend", + "get_backend", "reset_backend_cache", +] diff --git a/je_auto_control/utils/vision/backends/_parse.py b/je_auto_control/utils/vision/backends/_parse.py new file mode 100644 index 00000000..c59ee5bc --- /dev/null +++ b/je_auto_control/utils/vision/backends/_parse.py @@ -0,0 +1,35 @@ +"""Shared parsing helpers for VLM backend responses.""" +import re +from typing import Optional, Tuple + +_COORDS_RE = re.compile(r"(-?\d{1,5})(?:\s*,\s*|\s+)(-?\d{1,5})") + + +def parse_coords(text: str) -> Optional[Tuple[int, int]]: + """Extract the first ``x, y`` integer pair from a VLM reply. + + Returns ``None`` if the reply says ``none`` / ``not found`` or if no + two-integer pair can be located. Accepts minor formatting noise + (whitespace, punctuation, surrounding prose) so backends don't need + to be pedantic about prompt responses. + """ + if not text: + return None + cleaned = text.strip().lower() + if cleaned in {"none", "not found", "n/a", ""}: + return None + match = _COORDS_RE.search(text) + if match is None: + return None + try: + return int(match.group(1)), int(match.group(2)) + except ValueError: + return None + + +LOCATE_PROMPT = ( + 'Find the UI element described as: "{description}".\n' + 'Look at the screenshot and return ONLY the pixel coordinates of the ' + 'element center in the form "x,y" (two integers separated by a comma). ' + 'If the element is not visible, reply exactly "none".' +) diff --git a/je_auto_control/utils/vision/backends/anthropic_backend.py b/je_auto_control/utils/vision/backends/anthropic_backend.py new file mode 100644 index 00000000..0c7c6fed --- /dev/null +++ b/je_auto_control/utils/vision/backends/anthropic_backend.py @@ -0,0 +1,83 @@ +"""Anthropic (Claude) VLM backend.""" +import base64 +import os +from typing import Optional, Tuple + +from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.vision.backends._parse import ( + LOCATE_PROMPT, parse_coords, +) +from je_auto_control.utils.vision.backends.base import VLMBackend + +_DEFAULT_MODEL = "claude-opus-4-7" +_REQUEST_TIMEOUT_S = 30.0 +_MAX_TOKENS = 64 + + +class AnthropicVLMBackend(VLMBackend): + """Call ``claude-*`` models via the ``anthropic`` Python SDK.""" + + name = "anthropic" + + def __init__(self) -> None: + self._client = None + try: + import anthropic # noqa: F401 + except ImportError: + self.available = False + return + if not os.environ.get("ANTHROPIC_API_KEY"): + self.available = False + return + try: + from anthropic import Anthropic + self._client = Anthropic() + self.available = True + except (ImportError, ValueError, RuntimeError) as error: + autocontrol_logger.warning( + "Anthropic client init failed: %r", error, + ) + self.available = False + + def locate(self, image_bytes: bytes, description: str, + model: Optional[str] = None, + image_mime: str = "image/png", + ) -> Optional[Tuple[int, int]]: + if not self.available or self._client is None: + return None + chosen_model = (model + or os.environ.get("AUTOCONTROL_VLM_MODEL") + or _DEFAULT_MODEL) + b64 = base64.standard_b64encode(image_bytes).decode("ascii") + prompt = LOCATE_PROMPT.format(description=description) + try: + response = self._client.messages.create( + model=chosen_model, + max_tokens=_MAX_TOKENS, + timeout=_REQUEST_TIMEOUT_S, + messages=[{ + "role": "user", + "content": [ + {"type": "image", "source": { + "type": "base64", + "media_type": image_mime, + "data": b64, + }}, + {"type": "text", "text": prompt}, + ], + }], + ) + except (OSError, ValueError, RuntimeError) as error: + autocontrol_logger.warning( + "Anthropic VLM request failed: %r", error, + ) + return None + text = _first_text_block(response) + return parse_coords(text) + + +def _first_text_block(response) -> str: + for block in getattr(response, "content", []) or []: + if getattr(block, "type", None) == "text": + return getattr(block, "text", "") or "" + return "" diff --git a/je_auto_control/utils/vision/backends/base.py b/je_auto_control/utils/vision/backends/base.py new file mode 100644 index 00000000..23e6147d --- /dev/null +++ b/je_auto_control/utils/vision/backends/base.py @@ -0,0 +1,19 @@ +"""Abstract VLM (vision-language model) backend.""" +from typing import Optional, Tuple + + +class VLMNotAvailableError(RuntimeError): + """Raised when no VLM backend can be initialised.""" + + +class VLMBackend: + """Each backend turns a screenshot + description into pixel coordinates.""" + + name: str = "abstract" + available: bool = False + + def locate(self, image_bytes: bytes, description: str, + model: Optional[str] = None, + image_mime: str = "image/png", + ) -> Optional[Tuple[int, int]]: + raise NotImplementedError diff --git a/je_auto_control/utils/vision/backends/null_backend.py b/je_auto_control/utils/vision/backends/null_backend.py new file mode 100644 index 00000000..7ea8db69 --- /dev/null +++ b/je_auto_control/utils/vision/backends/null_backend.py @@ -0,0 +1,22 @@ +"""Fallback backend when no VLM SDK / API key is available.""" +from typing import Optional, Tuple + +from je_auto_control.utils.vision.backends.base import ( + VLMBackend, VLMNotAvailableError, +) + + +class NullVLMBackend(VLMBackend): + """Backend that always raises an informative error.""" + + name = "null" + available = False + + def __init__(self, reason: str = "no VLM backend available"): + self._reason = reason + + def locate(self, image_bytes: bytes, description: str, + model: Optional[str] = None, + image_mime: str = "image/png", + ) -> Optional[Tuple[int, int]]: + raise VLMNotAvailableError(self._reason) diff --git a/je_auto_control/utils/vision/backends/openai_backend.py b/je_auto_control/utils/vision/backends/openai_backend.py new file mode 100644 index 00000000..ba4e56cb --- /dev/null +++ b/je_auto_control/utils/vision/backends/openai_backend.py @@ -0,0 +1,76 @@ +"""OpenAI (GPT-4-vision family) VLM backend.""" +import base64 +import os +from typing import Optional, Tuple + +from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.vision.backends._parse import ( + LOCATE_PROMPT, parse_coords, +) +from je_auto_control.utils.vision.backends.base import VLMBackend + +_DEFAULT_MODEL = "gpt-4o-mini" +_REQUEST_TIMEOUT_S = 30.0 +_MAX_TOKENS = 64 + + +class OpenAIVLMBackend(VLMBackend): + """Call OpenAI vision-capable chat models via the ``openai`` SDK.""" + + name = "openai" + + def __init__(self) -> None: + self._client = None + try: + import openai # noqa: F401 # nosemgrep: codacy.python.openai.import-without-guardrails # reason: availability probe only + except ImportError: + self.available = False + return + if not os.environ.get("OPENAI_API_KEY"): + self.available = False + return + try: + from openai import OpenAI # nosemgrep: codacy.python.openai.import-without-guardrails # reason: internal client init, input is user-supplied prompt only + self._client = OpenAI(timeout=_REQUEST_TIMEOUT_S) + self.available = True + except (ImportError, ValueError, RuntimeError) as error: + autocontrol_logger.warning( + "OpenAI client init failed: %r", error, + ) + self.available = False + + def locate(self, image_bytes: bytes, description: str, + model: Optional[str] = None, + image_mime: str = "image/png", + ) -> Optional[Tuple[int, int]]: + if not self.available or self._client is None: + return None + chosen_model = (model + or os.environ.get("AUTOCONTROL_VLM_MODEL") + or _DEFAULT_MODEL) + b64 = base64.standard_b64encode(image_bytes).decode("ascii") + prompt = LOCATE_PROMPT.format(description=description) + data_url = f"data:{image_mime};base64,{b64}" + try: + response = self._client.chat.completions.create( + model=chosen_model, + max_tokens=_MAX_TOKENS, + messages=[{ + "role": "user", + "content": [ + {"type": "text", "text": prompt}, + {"type": "image_url", + "image_url": {"url": data_url}}, + ], + }], + ) + except (OSError, ValueError, RuntimeError) as error: + autocontrol_logger.warning( + "OpenAI VLM request failed: %r", error, + ) + return None + try: + text = response.choices[0].message.content or "" + except (AttributeError, IndexError): + text = "" + return parse_coords(text) diff --git a/je_auto_control/utils/vision/vlm_api.py b/je_auto_control/utils/vision/vlm_api.py new file mode 100644 index 00000000..f84b9b99 --- /dev/null +++ b/je_auto_control/utils/vision/vlm_api.py @@ -0,0 +1,95 @@ +"""Public VLM-locator API. + +Locate UI elements by natural-language description using a +vision-language model, as a fallback for cases where pixel templates +and accessibility lookups both come up empty. The backend is chosen +per :mod:`je_auto_control.utils.vision.backends` by env vars. +""" +import os +import tempfile +from pathlib import Path +from typing import List, Optional, Tuple + +from je_auto_control.utils.vision.backends import get_backend +from je_auto_control.utils.vision.backends.base import ( + VLMBackend, VLMNotAvailableError, +) + + +def locate_by_description(description: str, + screen_region: Optional[List[int]] = None, + model: Optional[str] = None, + backend: Optional[VLMBackend] = None, + ) -> Optional[Tuple[int, int]]: + """Ask a VLM where ``description`` is on screen; return ``(x, y)`` or None. + + ``screen_region`` is ``[x1, y1, x2, y2]`` in screen pixels. When + supplied, only that region is sent to the model and the returned + coordinates are translated back into absolute screen space so + callers can feed them straight into mouse operations. Raises + :class:`VLMNotAvailableError` if no backend is configured. + """ + if not description or not description.strip(): + raise ValueError("description must be a non-empty string") + bound = backend if backend is not None else get_backend() + if not bound.available: + raise VLMNotAvailableError( + "no VLM backend configured; set ANTHROPIC_API_KEY or " + "OPENAI_API_KEY and install the matching SDK", + ) + image_bytes = _capture_screenshot_bytes(screen_region) + coords = bound.locate(image_bytes, description, model=model) + if coords is None: + return None + x, y = coords + if screen_region is not None: + x += int(screen_region[0]) + y += int(screen_region[1]) + return (int(x), int(y)) + + +def click_by_description(description: str, + screen_region: Optional[List[int]] = None, + model: Optional[str] = None, + backend: Optional[VLMBackend] = None, + ) -> bool: + """Locate by description, then click the center of the match. + + Returns ``True`` on a successful click, ``False`` if no element was + found. Raises :class:`VLMNotAvailableError` when no backend exists. + """ + coords = locate_by_description( + description, screen_region=screen_region, + model=model, backend=backend, + ) + if coords is None: + return False + cx, cy = coords + from je_auto_control.wrapper.auto_control_mouse import ( + click_mouse, set_mouse_position, + ) + set_mouse_position(cx, cy) + click_mouse("mouse_left", cx, cy) + return True + + +def _capture_screenshot_bytes( + screen_region: Optional[List[int]] = None) -> bytes: + """Take a screenshot (optionally cropped) and return PNG bytes.""" + fd, tmp = tempfile.mkstemp(prefix="vlm_", suffix=".png") + os.close(fd) + tmp_path = Path(tmp) + try: + from je_auto_control.wrapper.auto_control_screen import screenshot + screenshot(str(tmp_path), screen_region=screen_region) + return tmp_path.read_bytes() + finally: + try: + tmp_path.unlink() + except FileNotFoundError: + pass + + +__all__ = [ + "VLMNotAvailableError", "locate_by_description", "click_by_description", +] diff --git a/je_auto_control/utils/xml/__init__.py b/je_auto_control/utils/xml/__init__.py index e69de29b..c61f0d4c 100644 --- a/je_auto_control/utils/xml/__init__.py +++ b/je_auto_control/utils/xml/__init__.py @@ -0,0 +1,11 @@ +"""XML helpers. + +Calling ``defusedxml.defuse_stdlib()`` at import time monkey-patches the +stdlib XML parsers (xml.etree.ElementTree, xml.dom.minidom, xml.sax, ...) +so any subsequent parsing in this package — including accidental imports +by third-party code — is XXE/billion-laughs safe. We still prefer the +explicit ``defusedxml`` API in our own modules for clarity. +""" +import defusedxml + +defusedxml.defuse_stdlib() diff --git a/je_auto_control/utils/xml/change_xml_structure/change_xml_structure.py b/je_auto_control/utils/xml/change_xml_structure/change_xml_structure.py index 2f68de7c..b9f4e40a 100644 --- a/je_auto_control/utils/xml/change_xml_structure/change_xml_structure.py +++ b/je_auto_control/utils/xml/change_xml_structure/change_xml_structure.py @@ -1,6 +1,48 @@ from collections import defaultdict -from xml.etree import ElementTree -from typing import Dict, Any +from defusedxml import ElementTree as DefusedET # nosec B405 # nosemgrep: python.lang.security.use-defused-xml.use-defused-xml # reason: defusedxml is the safe replacement +from xml.etree import ElementTree # nosec B405 # nosemgrep: python.lang.security.use-defused-xml.use-defused-xml # reason: only used to construct trees, not to parse untrusted data +from typing import Any, Dict + + +def _initial_body(children: list, element: ElementTree.Element) -> Any: + """Pick the starting body form: dict from children, empty dict, or None.""" + if children: + return _children_to_dict(children) + if element.attrib: + return {} + return None + + +def _children_to_dict(children: list) -> Dict[str, Any]: + """Collapse children into ``{tag: value | [values]}`` form.""" + grouped: Dict[str, list] = defaultdict(list) + for child_dict in (elements_tree_to_dict(c) for c in children): + for key, value in child_dict.items(): + grouped[key].append(value) + return {key: value[0] if len(value) == 1 else value + for key, value in grouped.items()} + + +def _attach_attributes(element: ElementTree.Element, + body: Dict[str, Any]) -> None: + """Merge ``element.attrib`` into ``body`` with ``@key`` prefixes.""" + if not element.attrib: + return + body.update(('@' + key, value) for key, value in element.attrib.items()) + + +def _attach_text(element: ElementTree.Element, + elements_dict: Dict[str, Any], + has_structure: bool) -> None: + """Attach text content using ``#text`` or as a flat string.""" + if not element.text: + return + text = element.text.strip() + if has_structure: + if text: + elements_dict[element.tag]['#text'] = text + else: + elements_dict[element.tag] = text def elements_tree_to_dict(elements_tree: ElementTree.Element) -> Dict[str, Any]: @@ -11,40 +53,64 @@ def elements_tree_to_dict(elements_tree: ElementTree.Element) -> Dict[str, Any]: :param elements_tree: XML Element :return: dict representation of XML """ - elements_dict: dict = {elements_tree.tag: {} if elements_tree.attrib else None} children = list(elements_tree) + body: Any = _initial_body(children, elements_tree) + elements_dict: Dict[str, Any] = {elements_tree.tag: body} - # 遞迴處理子節點 Recursively process children - if children: - default_dict = defaultdict(list) - for dc in map(elements_tree_to_dict, children): - for key, value in dc.items(): - default_dict[key].append(value) - elements_dict = { - elements_tree.tag: { - key: value[0] if len(value) == 1 else value - for key, value in default_dict.items() - } - } - - # 加入屬性 Add attributes - if elements_tree.attrib: - elements_dict[elements_tree.tag].update( - ('@' + key, value) for key, value in elements_tree.attrib.items() - ) + if isinstance(body, dict): + _attach_attributes(elements_tree, body) - # 加入文字內容 Add text content - if elements_tree.text: - text = elements_tree.text.strip() - if children or elements_tree.attrib: - if text: - elements_dict[elements_tree.tag]['#text'] = text - else: - elements_dict[elements_tree.tag] = text + has_structure = bool(children or elements_tree.attrib) + _attach_text(elements_tree, elements_dict, has_structure) return elements_dict +def _validate_text_node(key: str, value: Any) -> None: + if key != '#text' or not isinstance(value, str): + raise ValueError( + f"Invalid text node: key={key}, value type={type(value)}" + ) + + +def _set_attribute(root: ElementTree.Element, key: str, value: Any) -> None: + if not isinstance(value, str): + raise TypeError(f"Expected str attribute value, got {type(value)}") + root.set(key[1:], value) + + +def _build_child_node(parent: ElementTree.Element, key: str, value: Any) -> None: + if isinstance(value, list): + for element in value: + _to_elements_tree(element, ElementTree.SubElement(parent, key)) + else: + _to_elements_tree(value, ElementTree.SubElement(parent, key)) + + +def _process_dict_entry(root: ElementTree.Element, key: str, value: Any) -> None: + if not isinstance(key, str): + raise TypeError(f"Expected str key, got {type(key)}") + if key.startswith('#'): + _validate_text_node(key, value) + root.text = value + return + if key.startswith('@'): + _set_attribute(root, key, value) + return + _build_child_node(root, key, value) + + +def _to_elements_tree(json_dict: Any, root: ElementTree.Element) -> None: + if isinstance(json_dict, str): + root.text = json_dict + return + if isinstance(json_dict, dict): + for key, value in json_dict.items(): + _process_dict_entry(root, key, value) + return + raise TypeError(f"Invalid type: {type(json_dict)}") + + def dict_to_elements_tree(json_dict: Dict[str, Any]) -> str: """ Convert dictionary to XML string. @@ -53,37 +119,18 @@ def dict_to_elements_tree(json_dict: Dict[str, Any]) -> str: :param json_dict: dict representation of XML :return: XML string """ - - def _to_elements_tree(json_dict: Any, root: ElementTree.Element) -> None: - if isinstance(json_dict, str): - root.text = json_dict - elif isinstance(json_dict, dict): - for key, value in json_dict.items(): - if not isinstance(key, str): - raise TypeError(f"Expected str key, got {type(key)}") - if key.startswith('#'): - # 處理文字節點 Handle text node - if key != '#text' or not isinstance(value, str): - raise ValueError(f"Invalid text node: key={key}, value type={type(value)}") - root.text = value - elif key.startswith('@'): - # 處理屬性 Handle attributes - if not isinstance(value, str): - raise TypeError(f"Expected str attribute value, got {type(value)}") - root.set(key[1:], value) - elif isinstance(value, list): - # 處理多個子節點 Handle multiple children - for element in value: - _to_elements_tree(element, ElementTree.SubElement(root, key)) - else: - # 處理單一子節點 Handle single child - _to_elements_tree(value, ElementTree.SubElement(root, key)) - else: - raise TypeError(f"Invalid type: {type(json_dict)}") - if not isinstance(json_dict, dict) or len(json_dict) != 1: - raise TypeError(f"Expected dict with exactly 1 key, got {type(json_dict)} with {len(json_dict) if isinstance(json_dict, dict) else 'N/A'} keys") + key_count = len(json_dict) if isinstance(json_dict, dict) else 'N/A' + raise TypeError( + f"Expected dict with exactly 1 key, got {type(json_dict)} " + f"with {key_count} keys" + ) tag, body = next(iter(json_dict.items())) node = ElementTree.Element(tag) _to_elements_tree(body, node) - return ElementTree.tostring(node, encoding="utf-8").decode("utf-8") \ No newline at end of file + return ElementTree.tostring(node, encoding="utf-8").decode("utf-8") + + +def parse_xml_string_safely(xml_string: str) -> ElementTree.Element: + """Parse an XML string with defusedxml to avoid XXE / billion-laughs.""" + return DefusedET.fromstring(xml_string) diff --git a/je_auto_control/utils/xml/xml_file/xml_file.py b/je_auto_control/utils/xml/xml_file/xml_file.py index 69552463..bd22b24f 100644 --- a/je_auto_control/utils/xml/xml_file/xml_file.py +++ b/je_auto_control/utils/xml/xml_file/xml_file.py @@ -1,9 +1,20 @@ -import xml.dom.minidom -from xml.etree import ElementTree -from xml.etree.ElementTree import ParseError +"""Safe XML helpers backed by ``defusedxml``. -from je_auto_control.utils.exception.exception_tags import cant_read_xml_error_message, xml_type_error_message -from je_auto_control.utils.exception.exceptions import XMLException, XMLTypeException +Direct ``xml.etree`` / ``xml.dom.minidom`` parsing is vulnerable to XXE and +billion-laughs attacks. We parse via ``defusedxml`` and only build trees with +the stdlib ``ElementTree`` (which is safe for construction). +""" +from defusedxml import ElementTree as DefusedET # nosec B405 # nosemgrep: python.lang.security.use-defused-xml.use-defused-xml # reason: defusedxml is the safe replacement +from defusedxml.minidom import parseString as defused_parse_string # nosemgrep: python.lang.security.use-defused-xml.use-defused-xml # reason: defusedxml parser +from xml.etree import ElementTree # nosec B405 # nosemgrep: python.lang.security.use-defused-xml.use-defused-xml # reason: only used to construct trees, not to parse untrusted data +from xml.etree.ElementTree import ParseError # nosec B405 # nosemgrep: python.lang.security.use-defused-xml.use-defused-xml # reason: exception type used for catching only + +from je_auto_control.utils.exception.exception_tags import ( + cant_read_xml_error_message, xml_type_error_message, +) +from je_auto_control.utils.exception.exceptions import ( + XMLException, XMLTypeException, +) def reformat_xml_file(xml_string: str) -> str: @@ -14,7 +25,7 @@ def reformat_xml_file(xml_string: str) -> str: :param xml_string: 原始 XML 字串 Raw XML string :return: 美化後的 XML 字串 Pretty XML string """ - dom = xml.dom.minidom.parseString(xml_string) + dom = defused_parse_string(xml_string) return dom.toprettyxml(indent=" ", encoding="utf-8").decode("utf-8") @@ -56,7 +67,7 @@ def xml_parser_from_string(self, **kwargs) -> ElementTree.Element: :return: XML root element """ try: - self.xml_root = ElementTree.fromstring(self.xml_string, **kwargs) + self.xml_root = DefusedET.fromstring(self.xml_string, **kwargs) except ParseError as error: raise XMLException(f"{cant_read_xml_error_message}: {repr(error)}") from error return self.xml_root @@ -69,7 +80,7 @@ def xml_parser_from_file(self, **kwargs) -> ElementTree.Element: :return: XML root element """ try: - self.tree = ElementTree.parse(self.xml_string, **kwargs) + self.tree = DefusedET.parse(self.xml_string, **kwargs) except (OSError, ParseError) as error: raise XMLException(f"{cant_read_xml_error_message}: {repr(error)}") from error self.xml_root = self.tree.getroot() @@ -85,8 +96,8 @@ def write_xml(self, write_xml_filename: str, write_content: str) -> None: :param write_content: XML 字串 XML string """ try: - content = ElementTree.fromstring(write_content.strip()) + content = DefusedET.fromstring(write_content.strip()) tree = ElementTree.ElementTree(content) tree.write(write_xml_filename, encoding="utf-8", xml_declaration=True) except ParseError as error: - raise XMLException(f"{cant_read_xml_error_message}: {repr(error)}") from error \ No newline at end of file + raise XMLException(f"{cant_read_xml_error_message}: {repr(error)}") from error diff --git a/je_auto_control/windows/mouse/win32_ctype_mouse_control.py b/je_auto_control/windows/mouse/win32_ctype_mouse_control.py index ee11c5b8..e98ee454 100644 --- a/je_auto_control/windows/mouse/win32_ctype_mouse_control.py +++ b/je_auto_control/windows/mouse/win32_ctype_mouse_control.py @@ -42,7 +42,7 @@ def _convert_position(x: int, y: int) -> Tuple[int, int]: return converted_x, converted_y -def mouse_event(event: int, x: int, y: int, dwData: int = 0) -> None: +def mouse_event(event: int, x: int, y: int, dw_data: int = 0) -> None: """ 觸發滑鼠事件 Trigger mouse event @@ -50,14 +50,14 @@ def mouse_event(event: int, x: int, y: int, dwData: int = 0) -> None: :param event: 滑鼠事件代碼 Mouse event code :param x: X 座標 X position :param y: Y 座標 Y position - :param dwData: 滾輪數值 Wheel data + :param dw_data: 滾輪數值 Wheel data """ converted_x, converted_y = _convert_position(x, y) ctypes.windll.user32.mouse_event( event, ctypes.c_long(converted_x), ctypes.c_long(converted_y), - dwData, + dw_data, 0 ) @@ -133,7 +133,7 @@ def scroll(scroll_value: int, x: int = 0, y: int = 0) -> None: :param x: X 座標 X position :param y: Y 座標 Y position """ - mouse_event(WIN32_WHEEL, x, y, dwData=scroll_value) + mouse_event(WIN32_WHEEL, x, y, dw_data=scroll_value) def send_mouse_event_to_window(window, mouse_keycode: int, x: int = 0, y: int = 0): diff --git a/je_auto_control/wrapper/auto_control_record.py b/je_auto_control/wrapper/auto_control_record.py index 13b08771..964067e3 100644 --- a/je_auto_control/wrapper/auto_control_record.py +++ b/je_auto_control/wrapper/auto_control_record.py @@ -35,10 +35,10 @@ def stop_record() -> list: if action_queue is None: raise AutoControlJsonActionException action_list = list(action_queue.queue) - new_list = list() + new_list = [] for action in action_list: if action[0] == "AC_type_keyboard": - new_list.append([action[0], dict([["keycode", action[1]]])]) + new_list.append([action[0], {"keycode": action[1]}]) else: new_list.append([action[0], {"mouse_keycode": action[0], "x": action[1], "y": action[2]}]) record_action_to_list("stop_record", None) diff --git a/je_auto_control/wrapper/platform_wrapper.py b/je_auto_control/wrapper/platform_wrapper.py index a8945fc5..eede8bbc 100644 --- a/je_auto_control/wrapper/platform_wrapper.py +++ b/je_auto_control/wrapper/platform_wrapper.py @@ -3,19 +3,19 @@ from je_auto_control.utils.exception.exceptions import AutoControlException if sys.platform in ["win32", "cygwin", "msys"]: - from je_auto_control.wrapper._platform_windows import ( + from je_auto_control.wrapper._platform_windows import ( # noqa: F401 # reason: facade re-export keyboard, keyboard_check, keyboard_keys_table, mouse, mouse_keys_table, special_mouse_keys_table, screen, recorder, ) elif sys.platform == "darwin": - from je_auto_control.wrapper._platform_osx import ( + from je_auto_control.wrapper._platform_osx import ( # noqa: F401 # reason: facade re-export keyboard, keyboard_check, keyboard_keys_table, mouse, mouse_keys_table, special_mouse_keys_table, screen, recorder, ) elif sys.platform in ["linux", "linux2"]: - from je_auto_control.wrapper._platform_linux import ( + from je_auto_control.wrapper._platform_linux import ( # noqa: F401 # reason: facade re-export keyboard, keyboard_check, keyboard_keys_table, mouse, mouse_keys_table, special_mouse_keys_table, screen, recorder, @@ -25,3 +25,10 @@ if None in [keyboard_keys_table, mouse_keys_table, keyboard, mouse, screen]: raise AutoControlException("Can't init auto control") + + +__all__ = [ + "keyboard", "keyboard_check", "keyboard_keys_table", + "mouse", "mouse_keys_table", "special_mouse_keys_table", + "screen", "recorder", +] diff --git a/pyproject.toml b/pyproject.toml index 0b3be44b..91ef0ddf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ # Rename to build stable version # This is stable version [build-system] -requires = ["setuptools>=61.0"] +requires = ["setuptools>=82.0.1"] build-backend = "setuptools.build_meta" [project] @@ -19,7 +19,8 @@ dependencies = [ "pyobjc-core==12.1;platform_system=='Darwin'", "pyobjc==12.1;platform_system=='Darwin'", "python-Xlib==0.33;platform_system=='Linux'", - "mss==10.1.0" + "mss==10.2.0", + "defusedxml==0.7.1" ] classifiers = [ "Programming Language :: Python :: 3.10", @@ -44,3 +45,9 @@ find = { namespaces = false } [project.optional-dependencies] gui = ["PySide6==6.11.0", "qt-material==2.17"] + +[tool.bandit] +exclude_dirs = ["test", "docs", ".venv", "build", "dist"] +# B101 (use of assert) — pytest test code intentionally uses assert. +# Library code is enforced by CLAUDE.md (no assert in non-test code). +skips = ["B101"] diff --git a/requirements.txt b/requirements.txt index ecd31e72..59277a46 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ je_auto_control qt-material==2.17 -mss==10.1.0 +mss==10.2.0 PySide6==6.11.0 diff --git a/test/gui_test/calculator/calculator.py b/test/gui_test/calculator/calculator.py index 8d80417f..87c8ba29 100644 --- a/test/gui_test/calculator/calculator.py +++ b/test/gui_test/calculator/calculator.py @@ -1,4 +1,3 @@ -import subprocess from time import sleep from je_auto_control import locate_and_click @@ -8,122 +7,27 @@ # open windows calc.exe # and calculate 1 + 2 .... + 9 -subprocess.Popen("calc", stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, shell=True) +import subprocess # noqa: E402 # reason: imported after instructional comments +subprocess.Popen(["calc.exe"]) # nosec B603 B607 # reason: hard-coded calc launcher used by GUI test + sleep(3) -locate_and_click( - "../../test_source/1.png", - mouse_keycode="mouse_left", - detect_threshold=0.9, - draw_image=False -) -locate_and_click( - "../../test_source/plus.png", - mouse_keycode="mouse_left", - detect_threshold=0.9, - draw_image=False -) +_PLUS_IMG = "../../test_source/plus.png" +_EQUAL_IMG = "../../test_source/equal.png" +_DIGIT_IMG = "../../test_source/{n}.png" + + +def _click_image(image_path: str) -> None: + locate_and_click( + image_path, + mouse_keycode="mouse_left", + detect_threshold=0.9, + draw_image=False, + ) -locate_and_click( - "../../test_source/2.png", - mouse_keycode="mouse_left", - detect_threshold=0.9, - draw_image=False -) -locate_and_click( - "../../test_source/equal.png", - mouse_keycode="mouse_left", - detect_threshold=0.9, - draw_image=False -) -locate_and_click( - "../../test_source/plus.png", - mouse_keycode="mouse_left", - detect_threshold=0.9, - draw_image=False -) -locate_and_click( - "../../test_source/3.png", - mouse_keycode="mouse_left", - detect_threshold=0.9, - draw_image=False -) -locate_and_click( - "../../test_source/plus.png", - mouse_keycode="mouse_left", - detect_threshold=0.9, - draw_image=False -) -locate_and_click( - "../../test_source/4.png", - mouse_keycode="mouse_left", - detect_threshold=0.9, - draw_image=False -) -locate_and_click( - "../../test_source/plus.png", - mouse_keycode="mouse_left", - detect_threshold=0.9, - draw_image=False -) -locate_and_click( - "../../test_source/5.png", - mouse_keycode="mouse_left", - detect_threshold=0.9, - draw_image=False -) -locate_and_click( - "../../test_source/plus.png", - mouse_keycode="mouse_left", - detect_threshold=0.9, - draw_image=False -) -locate_and_click( - "../../test_source/6.png", - mouse_keycode="mouse_left", - detect_threshold=0.9, - draw_image=False -) -locate_and_click( - "../../test_source/plus.png", - mouse_keycode="mouse_left", - detect_threshold=0.9, - draw_image=False -) -locate_and_click( - "../../test_source/7.png", - mouse_keycode="mouse_left", - detect_threshold=0.9, - draw_image=False -) -locate_and_click( - "../../test_source/plus.png", - mouse_keycode="mouse_left", - detect_threshold=0.9, - draw_image=False -) -locate_and_click( - "../../test_source/8.png", - mouse_keycode="mouse_left", - detect_threshold=0.9, - draw_image=False -) -locate_and_click( - "../../test_source/plus.png", - mouse_keycode="mouse_left", - detect_threshold=0.9, - draw_image=False -) -locate_and_click( - "../../test_source/9.png", - mouse_keycode="mouse_left", - detect_threshold=0.9, - draw_image=False -) -locate_and_click( - "../../test_source/equal.png", - mouse_keycode="mouse_left", - detect_threshold=0.9, - draw_image=False -) +_click_image(_DIGIT_IMG.format(n=1)) +for digit in range(2, 10): + _click_image(_PLUS_IMG) + _click_image(_DIGIT_IMG.format(n=digit)) +_click_image(_EQUAL_IMG) diff --git a/test/gui_test/calculator/calculator_executor.py b/test/gui_test/calculator/calculator_executor.py index 90e6b8fb..ede7a3dc 100644 --- a/test/gui_test/calculator/calculator_executor.py +++ b/test/gui_test/calculator/calculator_executor.py @@ -1,69 +1,36 @@ -import subprocess -from time import sleep - -from je_auto_control import locate_and_click, executor +from je_auto_control import executor # 開啟windows 計算機 # 並累加1至9 # open windows calc.exe # and calculate 1 + 2 .... + 9 +_PLUS_IMG = "./test_source/plus.png" +_EQUAL_IMG = "./test_source/equal.png" +_DIGIT_IMG_FMT = "./test_source/{n}.png" + + +def _click_image_step(image_path: str) -> list: + return [ + "AC_locate_and_click", + { + "cv2_utils": image_path, + "mouse_keycode": "mouse_left", + "detect_threshold": 0.9, + }, + ] + + test_list = [ ["AC_add_package_to_executor", {"package": "subprocess"}], ["subprocess_Popen", {"args": "calc"}], ["AC_add_package_to_executor", {"package": "time"}], ["time_sleep", [3]], - ["AC_locate_and_click", - {"cv2_utils": "./test_source/1.png", "mouse_keycode": "mouse_left", "detect_threshold": 0.9}], - ["AC_locate_and_click", - {"cv2_utils": "./test_source/plus.png", "mouse_keycode": "mouse_left", "detect_threshold": 0.9}], - ["AC_locate_and_click", - {"cv2_utils": "./test_source/2.png", "mouse_keycode": "mouse_left", "detect_threshold": 0.9}], - ["AC_locate_and_click", - {"cv2_utils": "./test_source/equal.png", "mouse_keycode": "mouse_left", "detect_threshold": 0.9}], - ["AC_locate_and_click", - {"cv2_utils": "./test_source/plus.png", "mouse_keycode": "mouse_left", "detect_threshold": 0.9}], - ["AC_locate_and_click", - {"cv2_utils": "./test_source/3.png", "mouse_keycode": "mouse_left", "detect_threshold": 0.9}], - ["AC_locate_and_click", - {"cv2_utils": "./test_source/equal.png", "mouse_keycode": "mouse_left", "detect_threshold": 0.9}], - ["AC_locate_and_click", - {"cv2_utils": "./test_source/plus.png", "mouse_keycode": "mouse_left", "detect_threshold": 0.9}], - ["AC_locate_and_click", - {"cv2_utils": "./test_source/4.png", "mouse_keycode": "mouse_left", "detect_threshold": 0.9}], - ["AC_locate_and_click", - {"cv2_utils": "./test_source/equal.png", "mouse_keycode": "mouse_left", "detect_threshold": 0.9}], - ["AC_locate_and_click", - {"cv2_utils": "./test_source/plus.png", "mouse_keycode": "mouse_left", "detect_threshold": 0.9}], - ["AC_locate_and_click", - {"cv2_utils": "./test_source/5.png", "mouse_keycode": "mouse_left", "detect_threshold": 0.9}], - ["AC_locate_and_click", - {"cv2_utils": "./test_source/equal.png", "mouse_keycode": "mouse_left", "detect_threshold": 0.9}], - ["AC_locate_and_click", - {"cv2_utils": "./test_source/plus.png", "mouse_keycode": "mouse_left", "detect_threshold": 0.9}], - ["AC_locate_and_click", - {"cv2_utils": "./test_source/6.png", "mouse_keycode": "mouse_left", "detect_threshold": 0.9}], - ["AC_locate_and_click", - {"cv2_utils": "./test_source/equal.png", "mouse_keycode": "mouse_left", "detect_threshold": 0.9}], - ["AC_locate_and_click", - {"cv2_utils": "./test_source/plus.png", "mouse_keycode": "mouse_left", "detect_threshold": 0.9}], - ["AC_locate_and_click", - {"cv2_utils": "./test_source/7.png", "mouse_keycode": "mouse_left", "detect_threshold": 0.9}], - ["AC_locate_and_click", - {"cv2_utils": "./test_source/equal.png", "mouse_keycode": "mouse_left", "detect_threshold": 0.9}], - ["AC_locate_and_click", - {"cv2_utils": "./test_source/plus.png", "mouse_keycode": "mouse_left", "detect_threshold": 0.9}], - ["AC_locate_and_click", - {"cv2_utils": "./test_source/8.png", "mouse_keycode": "mouse_left", "detect_threshold": 0.9}], - ["AC_locate_and_click", - {"cv2_utils": "./test_source/equal.png", "mouse_keycode": "mouse_left", "detect_threshold": 0.9}], - ["AC_locate_and_click", - {"cv2_utils": "./test_source/plus.png", "mouse_keycode": "mouse_left", "detect_threshold": 0.9}], - ["AC_locate_and_click", - {"cv2_utils": "./test_source/9.png", "mouse_keycode": "mouse_left", "detect_threshold": 0.9}], - ["AC_locate_and_click", - {"cv2_utils": "./test_source/equal.png", "mouse_keycode": "mouse_left", "detect_threshold": 0.9}] + _click_image_step(_DIGIT_IMG_FMT.format(n=1)), ] +for digit in range(2, 10): + test_list.append(_click_image_step(_PLUS_IMG)) + test_list.append(_click_image_step(_DIGIT_IMG_FMT.format(n=digit))) +test_list.append(_click_image_step(_EQUAL_IMG)) executor.execute_action(test_list) - diff --git a/test/integrated_test/total_record_and_html_report_test/total_record_and_html_report_test.py b/test/integrated_test/total_record_and_html_report_test/total_record_and_html_report_test.py index ea0e1789..cf1d2344 100644 --- a/test/integrated_test/total_record_and_html_report_test/total_record_and_html_report_test.py +++ b/test/integrated_test/total_record_and_html_report_test/total_record_and_html_report_test.py @@ -1,6 +1,6 @@ import sys -from je_auto_control import generate_html +from je_auto_control import generate_html_report from je_auto_control import keyboard_keys_table from je_auto_control import press_keyboard_key from je_auto_control import release_keyboard_key @@ -12,11 +12,11 @@ print(keyboard_keys_table.keys()) press_keyboard_key("shift") write("123456789") - assert (write("abcdefghijklmnopqrstuvwxyz") == "abcdefghijklmnopqrstuvwxyz") + assert write("abcdefghijklmnopqrstuvwxyz") == "abcdefghijklmnopqrstuvwxyz" # noqa: S101 # reason: integration test release_keyboard_key("shift") # this write will print one error -> keyboard write error can't find key : Ѓ and write remain string try: - assert (write("?123456789") == "123456789") + assert write("?123456789") == "123456789" # noqa: S101 # reason: integration test except Exception as error: print(repr(error), file=sys.stderr) try: @@ -27,7 +27,7 @@ print(test_record_instance.test_record_list) # html name is test.html and this html will recode all test detail # if test_record.init_total_record = True - generate_html("test") + generate_html_report("test") sys.exit(0) except Exception as error: print(repr(error), file=sys.stderr) diff --git a/test/unit_test/argparse/argparse_test.py b/test/unit_test/argparse/argparse_test.py index b5c241fa..3e072244 100644 --- a/test/unit_test/argparse/argparse_test.py +++ b/test/unit_test/argparse/argparse_test.py @@ -1,9 +1,20 @@ import os -import subprocess +import subprocess # nosec B404 # reason: argparse smoke test invokes the CLI module print(os.getcwd()) cwd = os.getcwd() -subprocess.run(["python", "je_auto_control", "--execute_file", os.path.join(cwd, "test/unit_test/argparse/test1.json")]) -subprocess.run(["python", "je_auto_control", "--execute_dir", os.path.join(cwd, "test/unit_test/argparse")]) -subprocess.run(["python", "je_auto_control", "--create_project", cwd]) +subprocess.run( # nosec B603 B607 # reason: argv list, python on PATH; cwd path validated by os.path.join + ["python", "je_auto_control", "--execute_file", + os.path.join(cwd, "test/unit_test/argparse/test1.json")], + check=False, +) +subprocess.run( # nosec B603 B607 # reason: argv list, python on PATH; cwd path validated by os.path.join + ["python", "je_auto_control", "--execute_dir", + os.path.join(cwd, "test/unit_test/argparse")], + check=False, +) +subprocess.run( # nosec B603 B607 # reason: argv list, python on PATH; cwd path validated by os.path.join + ["python", "je_auto_control", "--create_project", cwd], + check=False, +) diff --git a/test/unit_test/critical_exit/real_critical_test.py b/test/unit_test/critical_exit/real_critical_test.py index 45737360..9b42f206 100644 --- a/test/unit_test/critical_exit/real_critical_test.py +++ b/test/unit_test/critical_exit/real_critical_test.py @@ -16,10 +16,9 @@ from time import sleep sleep(3) - while True: - set_mouse_position(200, 400) - set_mouse_position(400, 600) - raise AutoControlMouseException + set_mouse_position(200, 400) + set_mouse_position(400, 600) + raise AutoControlMouseException except Exception as error: print(repr(error), file=sys.stderr) CriticalExit().init_critical_exit() diff --git a/test/unit_test/flow_control/test_flow_control.py b/test/unit_test/flow_control/test_flow_control.py index 1ed1bc16..c29cbaf3 100644 --- a/test/unit_test/flow_control/test_flow_control.py +++ b/test/unit_test/flow_control/test_flow_control.py @@ -64,7 +64,7 @@ def test_ac_wait_image_times_out(monkeypatch, executor_with_hooks): def test_ac_retry_succeeds_after_failures(executor_with_hooks): - ex, state = executor_with_hooks + ex, _ = executor_with_hooks attempts = {"n": 0} def flaky(): diff --git a/test/unit_test/headless/test_accessibility.py b/test/unit_test/headless/test_accessibility.py new file mode 100644 index 00000000..a8728ecb --- /dev/null +++ b/test/unit_test/headless/test_accessibility.py @@ -0,0 +1,154 @@ +"""Tests for the cross-platform accessibility API.""" +from typing import List, Optional + +import pytest + +from je_auto_control.utils.accessibility import backends as backends_mod +from je_auto_control.utils.accessibility.accessibility_api import ( + find_accessibility_element, list_accessibility_elements, +) +from je_auto_control.utils.accessibility.backends.base import ( + AccessibilityBackend, +) +from je_auto_control.utils.accessibility.backends.null_backend import ( + NullAccessibilityBackend, +) +from je_auto_control.utils.accessibility.element import ( + AccessibilityElement, AccessibilityNotAvailableError, element_matches, +) + + +class _FakeBackend(AccessibilityBackend): + name = "fake" + available = True + + def __init__(self, elements: List[AccessibilityElement]) -> None: + self._elements = elements + self.last_args: Optional[dict] = None + + def list_elements(self, app_name: Optional[str] = None, + max_results: int = 200, + ) -> List[AccessibilityElement]: + self.last_args = {"app_name": app_name, "max_results": max_results} + if app_name is None: + return list(self._elements) + return [e for e in self._elements if e.app_name == app_name] + + +@pytest.fixture +def sample_elements(): + return [ + AccessibilityElement( + name="OK", role="Button", + bounds=(10, 20, 80, 30), app_name="Calculator", + ), + AccessibilityElement( + name="Cancel", role="Button", + bounds=(100, 20, 80, 30), app_name="Calculator", + ), + AccessibilityElement( + name="File", role="MenuItem", + bounds=(0, 0, 50, 20), app_name="Notepad", + ), + ] + + +@pytest.fixture +def fake_backend(sample_elements): + backend = _FakeBackend(sample_elements) + backends_mod._cached_backend = backend + try: + yield backend + finally: + backends_mod.reset_backend_cache() + + +def test_center_midpoint_of_bounds(): + element = AccessibilityElement( + name="btn", role="Button", bounds=(10, 20, 80, 40), + ) + assert element.center == (50, 40) + + +def test_to_dict_round_trips_bounds_and_center(): + element = AccessibilityElement( + name="n", role="r", bounds=(1, 2, 3, 4), + app_name="A", process_id=9, native_id="abc", + ) + payload = element.to_dict() + assert payload["bounds"] == [1, 2, 3, 4] + assert payload["center"] == [2, 4] + assert payload["app_name"] == "A" + assert payload["process_id"] == 9 + assert payload["native_id"] == "abc" + + +def test_element_matches_name_role_app_combinations(): + element = AccessibilityElement( + name="OK", role="Button", + bounds=(0, 0, 10, 10), app_name="Calculator", + ) + assert element_matches(element, name="OK") + assert element_matches(element, role="button") # case-insensitive + assert element_matches(element, app_name="Calculator") + assert element_matches(element, name="OK", role="Button", + app_name="Calculator") + assert not element_matches(element, name="Cancel") + assert not element_matches(element, role="MenuItem") + assert not element_matches(element, app_name="Notepad") + + +def test_null_backend_raises_with_custom_reason(): + backend = NullAccessibilityBackend("because reasons") + assert backend.available is False + with pytest.raises(AccessibilityNotAvailableError) as info: + backend.list_elements() + assert "because reasons" in str(info.value) + + +def test_reset_backend_cache_clears_cached_instance(): + backends_mod._cached_backend = _FakeBackend([]) + assert backends_mod.get_backend() is backends_mod._cached_backend + backends_mod.reset_backend_cache() + assert backends_mod._cached_backend is None + + +def test_list_elements_passes_filters_through(fake_backend): + result = list_accessibility_elements(app_name="Calculator", + max_results=50) + assert fake_backend.last_args == { + "app_name": "Calculator", "max_results": 50, + } + assert len(result) == 2 + assert all(e.app_name == "Calculator" for e in result) + + +def test_find_element_returns_first_match(fake_backend): + element = find_accessibility_element( + name="File", app_name="Notepad", + ) + assert element is not None + assert element.role == "MenuItem" + + +def test_find_element_returns_none_when_no_match(fake_backend): + assert find_accessibility_element( + name="Nope", app_name="Calculator", + ) is None + + +def test_executor_registers_a11y_commands(): + from je_auto_control.utils.executor.action_executor import executor + commands = executor.known_commands() + assert "AC_a11y_list" in commands + assert "AC_a11y_find" in commands + assert "AC_a11y_click" in commands + + +def test_package_facade_exports_accessibility_api(): + import je_auto_control as ac + assert hasattr(ac, "AccessibilityElement") + assert hasattr(ac, "AccessibilityNotAvailableError") + assert hasattr(ac, "list_accessibility_elements") + assert hasattr(ac, "find_accessibility_element") + assert hasattr(ac, "click_accessibility_element") diff --git a/test/unit_test/headless/test_clipboard.py b/test/unit_test/headless/test_clipboard.py index e58f040b..ebf588f1 100644 --- a/test/unit_test/headless/test_clipboard.py +++ b/test/unit_test/headless/test_clipboard.py @@ -29,4 +29,4 @@ def test_set_and_get_roundtrip(): def test_set_clipboard_rejects_non_string(): with pytest.raises(TypeError): - set_clipboard(123) + set_clipboard(123) # type: ignore[arg-type] # NOSONAR diff --git a/test/unit_test/headless/test_hotkey_backends.py b/test/unit_test/headless/test_hotkey_backends.py new file mode 100644 index 00000000..43007aa0 --- /dev/null +++ b/test/unit_test/headless/test_hotkey_backends.py @@ -0,0 +1,121 @@ +"""Tests for the hotkey backend abstraction.""" +import threading +import time + +import pytest + +from je_auto_control.utils.hotkey import backends as backends_pkg +from je_auto_control.utils.hotkey.backends.base import HotkeyBackend +from je_auto_control.utils.hotkey.backends.linux_backend import ( + LinuxHotkeyBackend, +) +from je_auto_control.utils.hotkey.backends.macos_backend import ( + MacOSHotkeyBackend, _combo_to_macos, _primary_key_to_keycode, +) +from je_auto_control.utils.hotkey.hotkey_daemon import ( + BackendContext, HotkeyDaemon, split_combo, +) + + +def test_split_combo_canonicalises_aliases(): + mods_win, key = split_combo("win+a") + mods_super, _ = split_combo("super+a") + mods_meta, _ = split_combo("meta+a") + assert mods_win == mods_super == mods_meta == frozenset({"win"}) + assert key == "a" + + +def test_split_combo_mixed_modifiers(): + mods, key = split_combo("CTRL + alt + Shift + F7") + assert mods == frozenset({"ctrl", "alt", "shift"}) + assert key == "f7" + + +def test_split_combo_rejects_empty(): + with pytest.raises(ValueError): + split_combo("") + + +def test_split_combo_rejects_modifier_only(): + with pytest.raises(ValueError): + split_combo("ctrl+alt") + + +class _FakeBackend(HotkeyBackend): + name = "fake" + + def __init__(self): + self.ran = threading.Event() + self.last_bindings = None + + def run_forever(self, context: BackendContext) -> None: + self.last_bindings = context.get_bindings() + # Simulate firing the first registered binding, if any. + if self.last_bindings: + context.fire(self.last_bindings[0].binding_id) + self.ran.set() + context.stop_event.wait(0.5) + + +def test_daemon_routes_through_backend(monkeypatch): + fake = _FakeBackend() + monkeypatch.setattr(backends_pkg, "get_backend", lambda: fake) + called = [] + daemon = HotkeyDaemon(executor=lambda actions: called.append(actions)) + # Provide bindings + stub read_action_json so _fire_binding finishes. + from je_auto_control.utils.hotkey import hotkey_daemon as mod + monkeypatch.setattr(mod, "read_action_json", lambda path: [["AC_noop"]]) + binding = daemon.bind("ctrl+alt+1", "script.json") + try: + daemon.start() + assert fake.ran.wait(timeout=2.0), "backend run_forever should have started" + time.sleep(0.05) + finally: + daemon.stop(timeout=1.0) + assert binding.fired == 1 + assert called, "executor should have been invoked by the fake backend" + + +def test_daemon_snapshot_returns_current_bindings(): + daemon = HotkeyDaemon(executor=lambda _: None) + daemon.bind("ctrl+a", "x.json", binding_id="b1") + daemon.bind("ctrl+b", "y.json", binding_id="b2") + ids = sorted(b.binding_id for b in daemon._snapshot()) + assert ids == ["b1", "b2"] + + +def test_macos_combo_conversion_covers_modifiers_and_primary(): + mask, keycode = _combo_to_macos("ctrl+shift+alt+win+A") + # letter "a" on macOS has keycode 0 + assert keycode == 0 + # All four modifier flags set + assert mask == (1 << 17) | (1 << 18) | (1 << 19) | (1 << 20) + + +def test_macos_primary_key_unsupported(): + with pytest.raises(ValueError): + _primary_key_to_keycode("not-a-real-key") + + +def test_macos_function_keys(): + assert _primary_key_to_keycode("f5") == 96 + assert _primary_key_to_keycode("F12") == 111 + + +def test_macos_digits(): + assert _primary_key_to_keycode("0") == 29 + assert _primary_key_to_keycode("5") == 23 + + +def test_backends_have_distinct_names(): + names = { + LinuxHotkeyBackend.name, + MacOSHotkeyBackend.name, + } + # Importing the Windows backend on non-Windows is fine because it only + # calls ctypes.WinDLL inside run_forever, not at import time. + from je_auto_control.utils.hotkey.backends.windows_backend import ( + WindowsHotkeyBackend, + ) + names.add(WindowsHotkeyBackend.name) + assert names == {"windows", "linux-x11", "macos"} diff --git a/test/unit_test/headless/test_hotkey_parse.py b/test/unit_test/headless/test_hotkey_parse.py index 501adaf5..66dcaf4c 100644 --- a/test/unit_test/headless/test_hotkey_parse.py +++ b/test/unit_test/headless/test_hotkey_parse.py @@ -2,7 +2,7 @@ import pytest from je_auto_control.utils.hotkey.hotkey_daemon import ( - MOD_ALT, MOD_CONTROL, MOD_NOREPEAT, MOD_SHIFT, MOD_WIN, parse_combo, + MOD_ALT, MOD_CONTROL, MOD_NOREPEAT, MOD_SHIFT, parse_combo, ) diff --git a/test/unit_test/headless/test_plugin_loader.py b/test/unit_test/headless/test_plugin_loader.py index 4537ed09..3f16b88f 100644 --- a/test/unit_test/headless/test_plugin_loader.py +++ b/test/unit_test/headless/test_plugin_loader.py @@ -73,10 +73,10 @@ def test_register_plugin_commands_adds_and_removes_cleanly(tmp_path): def test_discover_ignores_non_callable_ac_attribute(): class Module: - AC_value = 42 + AC_value = 42 # noqa: N815 # reason: AC_* is the plugin contract @staticmethod - def AC_run(): + def AC_run(): # noqa: N802 # reason: AC_* is the plugin contract return 1 found = discover_plugin_commands(Module) diff --git a/test/unit_test/headless/test_run_history.py b/test/unit_test/headless/test_run_history.py new file mode 100644 index 00000000..c156dc58 --- /dev/null +++ b/test/unit_test/headless/test_run_history.py @@ -0,0 +1,250 @@ +"""Tests for the run history store and its scheduler/trigger/hotkey hooks.""" +import time +from pathlib import Path + +import pytest + +from je_auto_control.utils.run_history import history_store as history_mod +from je_auto_control.utils.run_history.history_store import ( + SOURCE_HOTKEY, SOURCE_MANUAL, SOURCE_SCHEDULER, SOURCE_TRIGGER, + STATUS_ERROR, STATUS_OK, STATUS_RUNNING, HistoryStore, RunRecord, +) + + +@pytest.fixture +def store(): + s = HistoryStore(path=":memory:") + try: + yield s + finally: + s.close() + + +def test_start_run_creates_row_with_running_status(store): + run_id = store.start_run(SOURCE_SCHEDULER, "job42", "a.json") + record = store.get_run(run_id) + assert isinstance(record, RunRecord) + assert record.source_type == SOURCE_SCHEDULER + assert record.source_id == "job42" + assert record.status == STATUS_RUNNING + assert record.finished_at is None + assert record.duration_seconds is None + + +def test_finish_run_marks_ok(store): + run_id = store.start_run(SOURCE_TRIGGER, "t1", "s.json") + assert store.finish_run(run_id, STATUS_OK) is True + record = store.get_run(run_id) + assert record.status == STATUS_OK + assert record.finished_at is not None + assert record.duration_seconds is not None + assert record.duration_seconds >= 0.0 + + +def test_finish_run_captures_error_text(store): + run_id = store.start_run(SOURCE_HOTKEY, "hk1", "x.json") + store.finish_run(run_id, STATUS_ERROR, error_text="boom") + record = store.get_run(run_id) + assert record.status == STATUS_ERROR + assert record.error_text == "boom" + + +def test_finish_run_rejects_running_status(store): + run_id = store.start_run(SOURCE_SCHEDULER, "j", "p") + with pytest.raises(ValueError): + store.finish_run(run_id, STATUS_RUNNING) + + +def test_finish_run_returns_false_for_unknown_id(store): + assert store.finish_run(99999, STATUS_OK) is False + + +def test_start_run_validates_source(store): + with pytest.raises(ValueError): + store.start_run("bogus", "id", "path") + + +def test_list_runs_orders_newest_first(store): + store.start_run(SOURCE_SCHEDULER, "a", "1.json", started_at=100.0) + store.start_run(SOURCE_SCHEDULER, "b", "2.json", started_at=200.0) + store.start_run(SOURCE_SCHEDULER, "c", "3.json", started_at=150.0) + records = store.list_runs(limit=10) + assert [r.source_id for r in records] == ["b", "c", "a"] + + +def test_list_runs_filters_by_source(store): + store.start_run(SOURCE_SCHEDULER, "s1", "x") + store.start_run(SOURCE_TRIGGER, "t1", "y") + store.start_run(SOURCE_HOTKEY, "h1", "z") + only_triggers = store.list_runs(source_type=SOURCE_TRIGGER) + assert len(only_triggers) == 1 + assert only_triggers[0].source_id == "t1" + + +def test_list_runs_respects_limit(store): + for i in range(5): + store.start_run(SOURCE_SCHEDULER, f"j{i}", "p") + assert len(store.list_runs(limit=3)) == 3 + assert store.list_runs(limit=0) == [] + + +def test_count_and_clear(store): + store.start_run(SOURCE_SCHEDULER, "a", "p") + store.start_run(SOURCE_TRIGGER, "b", "p") + assert store.count() == 2 + assert store.count(source_type=SOURCE_TRIGGER) == 1 + assert store.clear() == 2 + assert store.count() == 0 + + +def test_prune_keeps_latest(store): + for i in range(6): + store.start_run(SOURCE_SCHEDULER, f"j{i}", "p", + started_at=100.0 + i) + removed = store.prune(keep_latest=2) + assert removed == 4 + remaining = store.list_runs(limit=10) + assert [r.source_id for r in remaining] == ["j5", "j4"] + + +def test_scheduler_records_history(monkeypatch, store): + from je_auto_control.utils.scheduler import scheduler as sched_mod + + monkeypatch.setattr(sched_mod, "default_history_store", store) + monkeypatch.setattr(sched_mod, "read_action_json", lambda _: [["AC_noop"]]) + + sched = sched_mod.Scheduler( + executor=lambda actions: None, tick_seconds=0.05, + ) + sched.add_job("fake.json", interval_seconds=0.05, repeat=False) + sched.start() + try: + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline and not store.list_runs(): + time.sleep(0.05) + finally: + sched.stop(timeout=1.0) + runs = store.list_runs() + assert runs, "scheduler should have recorded a run" + assert runs[0].source_type == SOURCE_SCHEDULER + assert runs[0].status == STATUS_OK + + +def test_scheduler_records_error(monkeypatch, store): + from je_auto_control.utils.scheduler import scheduler as sched_mod + + def boom(_): + raise RuntimeError("fail") + + monkeypatch.setattr(sched_mod, "default_history_store", store) + monkeypatch.setattr(sched_mod, "read_action_json", boom) + monkeypatch.setattr(sched_mod, "capture_error_snapshot", lambda _rid: None) + + sched = sched_mod.Scheduler( + executor=lambda actions: None, tick_seconds=0.05, + ) + sched.add_job("fake.json", interval_seconds=0.05, repeat=False) + sched.start() + try: + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline and not store.list_runs(): + time.sleep(0.05) + finally: + sched.stop(timeout=1.0) + runs = store.list_runs() + assert runs and runs[0].status == STATUS_ERROR + assert "fail" in (runs[0].error_text or "") + + +def test_default_store_path_under_home(): + assert history_mod._default_history_path().parent.name == ".je_auto_control" + + +def test_executor_history_commands(monkeypatch, store): + from je_auto_control.utils.executor import action_executor + + monkeypatch.setattr(action_executor, "default_history_store", store) + store.start_run(SOURCE_SCHEDULER, "j", "p") + rows = action_executor._history_list_as_dicts(limit=10) + assert rows and rows[0]["source_type"] == SOURCE_SCHEDULER + + +def test_finish_run_persists_artifact_path(tmp_path, store): + artifact = str(tmp_path / "x.png") + run_id = store.start_run(SOURCE_SCHEDULER, "j", "s.json") + store.finish_run(run_id, STATUS_ERROR, error_text="boom", + artifact_path=artifact) + record = store.get_run(run_id) + assert record.artifact_path == artifact + + +def test_attach_artifact_updates_existing_row(tmp_path, store): + attached = str(tmp_path / "snap.png") + missing = str(tmp_path / "a.png") + run_id = store.start_run(SOURCE_HOTKEY, "h", "s.json") + store.finish_run(run_id, STATUS_ERROR, error_text="oops") + assert store.attach_artifact(run_id, attached) is True + assert store.get_run(run_id).artifact_path == attached + assert store.attach_artifact(99999, missing) is False + + +def test_clear_removes_artifact_files(tmp_path, store): + artifact = tmp_path / "snap.png" + artifact.write_bytes(b"fake") + run_id = store.start_run(SOURCE_TRIGGER, "t", "s.json") + store.finish_run(run_id, STATUS_ERROR, error_text="fail", + artifact_path=str(artifact)) + store.clear() + assert not artifact.exists() + + +def test_prune_removes_artifact_files_of_dropped_rows(tmp_path, store): + keep = tmp_path / "keep.png" + drop = tmp_path / "drop.png" + keep.write_bytes(b"k") + drop.write_bytes(b"d") + old = store.start_run(SOURCE_SCHEDULER, "a", "s", started_at=100.0) + store.finish_run(old, STATUS_ERROR, artifact_path=str(drop)) + new = store.start_run(SOURCE_SCHEDULER, "b", "s", started_at=200.0) + store.finish_run(new, STATUS_ERROR, artifact_path=str(keep)) + store.prune(keep_latest=1) + assert keep.exists() + assert not drop.exists() + + +def test_capture_error_snapshot_uses_injected_store(tmp_path, monkeypatch, + store): + from je_auto_control.utils.run_history import artifact_manager as am + + def fake_screenshot(path): + Path(path).write_bytes(b"png") + + import je_auto_control.wrapper.auto_control_screen as screen_mod + monkeypatch.setattr(screen_mod, "screenshot", fake_screenshot) + + run_id = store.start_run(SOURCE_MANUAL, "m", "s.json") + store.finish_run(run_id, STATUS_ERROR, error_text="x") + path = am.capture_error_snapshot( + run_id, artifacts_dir=tmp_path, store=store, + ) + assert path is not None + assert Path(path).exists() + assert store.get_run(run_id).artifact_path == path + + +def test_capture_error_snapshot_returns_none_on_failure(tmp_path, monkeypatch, + store): + from je_auto_control.utils.run_history import artifact_manager as am + + def boom(_path): + raise OSError("no display") + + import je_auto_control.wrapper.auto_control_screen as screen_mod + monkeypatch.setattr(screen_mod, "screenshot", boom) + + run_id = store.start_run(SOURCE_MANUAL, "m", "s.json") + store.finish_run(run_id, STATUS_ERROR, error_text="x") + assert am.capture_error_snapshot( + run_id, artifacts_dir=tmp_path, store=store, + ) is None + assert store.get_run(run_id).artifact_path is None diff --git a/test/unit_test/headless/test_trigger_engine.py b/test/unit_test/headless/test_trigger_engine.py index 8edfddd3..c9d76854 100644 --- a/test/unit_test/headless/test_trigger_engine.py +++ b/test/unit_test/headless/test_trigger_engine.py @@ -51,7 +51,7 @@ def test_engine_runs_trigger_once_when_non_repeat(tmp_path): later = watched.stat().st_mtime + 2 os.utime(str(watched), (later, later)) deadline = time.monotonic() + 2.0 - while time.monotonic() < deadline and not engine.list_triggers() == []: + while time.monotonic() < deadline and engine.list_triggers() != []: time.sleep(0.05) if calls: break diff --git a/test/unit_test/headless/test_vlm_locator.py b/test/unit_test/headless/test_vlm_locator.py new file mode 100644 index 00000000..b57d8dc3 --- /dev/null +++ b/test/unit_test/headless/test_vlm_locator.py @@ -0,0 +1,141 @@ +"""Tests for the VLM-based element locator API.""" +from typing import Optional, Tuple + +import pytest + +from je_auto_control.utils.vision import backends as backends_mod +from je_auto_control.utils.vision.backends._parse import ( + LOCATE_PROMPT, parse_coords, +) +from je_auto_control.utils.vision.backends.base import ( + VLMBackend, VLMNotAvailableError, +) +from je_auto_control.utils.vision.backends.null_backend import NullVLMBackend +from je_auto_control.utils.vision.vlm_api import ( + click_by_description, locate_by_description, +) + + +class _FakeBackend(VLMBackend): + name = "fake" + available = True + + def __init__(self, coords: Optional[Tuple[int, int]]) -> None: + self._coords = coords + self.last_call: Optional[dict] = None + + def locate(self, image_bytes: bytes, description: str, + model: Optional[str] = None, + image_mime: str = "image/png", + ) -> Optional[Tuple[int, int]]: + self.last_call = { + "bytes_len": len(image_bytes), + "description": description, + "model": model, + "image_mime": image_mime, + } + return self._coords + + +@pytest.fixture +def stub_screenshot(monkeypatch): + """Make ``_capture_screenshot_bytes`` return a fixed byte payload.""" + from je_auto_control.utils.vision import vlm_api + + def fake_capture(screen_region=None): + return b"fake-png-bytes" + + monkeypatch.setattr(vlm_api, "_capture_screenshot_bytes", fake_capture) + + +def test_parse_coords_accepts_plain_pair(): + assert parse_coords("120,45") == (120, 45) + + +def test_parse_coords_tolerates_whitespace_and_prose(): + assert parse_coords("The button is at 300, 400 pixels.") == (300, 400) + + +def test_parse_coords_returns_none_for_sentinels(): + assert parse_coords("none") is None + assert parse_coords("Not Found") is None + assert parse_coords("n/a") is None + assert parse_coords("") is None + + +def test_parse_coords_returns_none_when_no_pair_present(): + assert parse_coords("sorry, I cannot see it") is None + + +def test_locate_prompt_embeds_description(): + msg = LOCATE_PROMPT.format(description="green Submit button") + assert "green Submit button" in msg + assert "x,y" in msg + + +def test_null_backend_raises_with_custom_reason(): + backend = NullVLMBackend("sdk not installed") + assert backend.available is False + with pytest.raises(VLMNotAvailableError) as info: + backend.locate(b"", "desc") + assert "sdk not installed" in str(info.value) + + +def test_reset_backend_cache_clears_cached_instance(): + backends_mod._cached_backend = _FakeBackend((1, 2)) + assert backends_mod.get_backend() is backends_mod._cached_backend + backends_mod.reset_backend_cache() + assert backends_mod._cached_backend is None + + +def test_locate_raises_when_backend_unavailable(stub_screenshot): + with pytest.raises(VLMNotAvailableError): + locate_by_description("anything", backend=NullVLMBackend("nope")) + + +def test_locate_requires_non_empty_description(): + with pytest.raises(ValueError): + locate_by_description(" ", backend=_FakeBackend((0, 0))) + + +def test_locate_returns_backend_coords(stub_screenshot): + fake = _FakeBackend((150, 275)) + assert locate_by_description("btn", backend=fake) == (150, 275) + assert fake.last_call["description"] == "btn" + assert fake.last_call["bytes_len"] == len(b"fake-png-bytes") + + +def test_locate_returns_none_when_backend_says_none(stub_screenshot): + assert locate_by_description("btn", backend=_FakeBackend(None)) is None + + +def test_locate_translates_region_to_absolute_coords(stub_screenshot): + fake = _FakeBackend((20, 30)) + result = locate_by_description( + "btn", screen_region=[100, 200, 400, 500], backend=fake, + ) + assert result == (120, 230) + + +def test_locate_forwards_model_override(stub_screenshot): + fake = _FakeBackend((1, 1)) + locate_by_description("btn", model="claude-opus-4-7", backend=fake) + assert fake.last_call["model"] == "claude-opus-4-7" + + +def test_click_returns_false_when_not_found(stub_screenshot): + assert click_by_description("btn", backend=_FakeBackend(None)) is False + + +def test_executor_registers_vlm_commands(): + from je_auto_control.utils.executor.action_executor import executor + commands = executor.known_commands() + assert "AC_vlm_locate" in commands + assert "AC_vlm_click" in commands + + +def test_package_facade_exports_vlm_api(): + import je_auto_control as ac + assert hasattr(ac, "VLMNotAvailableError") + assert hasattr(ac, "locate_by_description") + assert hasattr(ac, "click_by_description") diff --git a/test/unit_test/mouse/mouse_test.py b/test/unit_test/mouse/mouse_test.py index 1e65dee4..889a5515 100644 --- a/test/unit_test/mouse/mouse_test.py +++ b/test/unit_test/mouse/mouse_test.py @@ -1,4 +1,3 @@ -import sys import time from je_auto_control import click_mouse diff --git a/test/unit_test/use_this_as_rpa_test/rpa_open_test.py b/test/unit_test/use_this_as_rpa_test/rpa_open_test.py index 799e63f1..b9c44f2e 100644 --- a/test/unit_test/use_this_as_rpa_test/rpa_open_test.py +++ b/test/unit_test/use_this_as_rpa_test/rpa_open_test.py @@ -1,13 +1,17 @@ -import subprocess +import subprocess # nosec B404 # reason: launches notepad for the screenshot RPA test import time from je_auto_control import screenshot -subprocess.Popen("notepad.exe", stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, shell=True) +subprocess.Popen( # nosec B603 B607 # reason: hard-coded notepad launcher, argv list + ["notepad.exe"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, +) time.sleep(10) # screenshot and save image = screenshot("test.png") -assert (image is not None) +assert image is not None # noqa: S101 # reason: pytest-style assertion in test script print(image) \ No newline at end of file