From 5993412d52d7cba63cac0475564e231d6d8e3e6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=A8=8A=E9=94=B6=E5=AE=B8?= Date: Tue, 28 Apr 2026 23:47:08 +0800 Subject: [PATCH 1/2] =?UTF-8?q?HDR=E5=8A=9F=E8=83=BD=E5=88=9D=E6=AD=A5?= =?UTF-8?q?=E5=AE=9E=E7=8E=B0=EF=BC=8C=E6=98=AFAdd=EF=BC=8C=E4=B8=8D?= =?UTF-8?q?=E5=BD=B1=E5=93=8D=E7=8E=B0=E5=9C=A8=E9=9D=9EHDR=E7=9A=84?= =?UTF-8?q?=E5=8A=9F=E8=83=BD=E5=AE=9E=E7=8E=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Screencap/FramePoolScreencap.cpp | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/source/MaaWin32ControlUnit/Screencap/FramePoolScreencap.cpp b/source/MaaWin32ControlUnit/Screencap/FramePoolScreencap.cpp index ec4c2f0c2d..2cd3fcc145 100644 --- a/source/MaaWin32ControlUnit/Screencap/FramePoolScreencap.cpp +++ b/source/MaaWin32ControlUnit/Screencap/FramePoolScreencap.cpp @@ -153,19 +153,30 @@ std::optional FramePoolScreencap::screencap() if (border_left + client_width > raw.cols) { border_left = raw.cols - client_width; } - if (client_height > raw.rows) { - client_height = raw.rows; + if (client_height > raw_height) { + client_height = raw_height; } if (border_top + client_height > raw.rows) { border_top = raw.rows - client_height; } cv::Rect client_roi(border_left, border_top, client_width, client_height); - cv::Mat image = raw(client_roi); - cv::Mat result = bgra_to_bgr(image); - cached_image_ = result.clone(); - return result; + switch (texture_desc_.Format) { + case DXGI_FORMAT_B8G8R8A8_UNORM: { + cv::Mat raw(raw_height, raw_width, CV_8UC4, mapped.pData, mapped.RowPitch); + cached_image_ = bgra_to_bgr(raw(client_roi)); + return cached_image_; + } + case DXGI_FORMAT_R16G16B16A16_FLOAT: { + cv::Mat raw(raw_height, raw_width, CV_16FC4, mapped.pData, mapped.RowPitch); + cached_image_ = raw(client_roi).clone(); + return cached_image_; + } + default: + LogError << "Unsupported frame format" << VAR(texture_desc_.Format); + return std::nullopt; + } } bool FramePoolScreencap::init() From fc201d65b475dfeb1421f3b65432aadb09fdfe96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=A8=8A=E9=94=B6=E5=AE=B8?= Date: Tue, 28 Apr 2026 23:49:04 +0800 Subject: [PATCH 2/2] =?UTF-8?q?HDR=E5=8A=9F=E8=83=BD=E5=88=9D=E6=AD=A5?= =?UTF-8?q?=E5=AE=9E=E7=8E=B0=EF=BC=8C=E6=98=AFAdd=EF=BC=8C=E4=B8=8D?= =?UTF-8?q?=E5=BD=B1=E5=93=8D=E7=8E=B0=E5=9C=A8=E9=9D=9EHDR=E7=9A=84?= =?UTF-8?q?=E5=8A=9F=E8=83=BD=E5=AE=9E=E7=8E=B0=E3=80=82=E9=87=87=E7=94=A8?= =?UTF-8?q?=E4=BA=86=E8=BD=AC=E6=8D=A2=E7=AE=97=E6=B3=95=EF=BC=8C=E5=BB=B6?= =?UTF-8?q?=E8=BF=9F=E5=92=8CCPU=E5=8D=A0=E7=94=A8=E6=8E=A7=E5=88=B6?= =?UTF-8?q?=E5=9C=A8=E5=90=88=E7=90=86=E8=8C=83=E5=9B=B4=E5=95=A6=EF=BC=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Controller/ControllerAgent.cpp | 697 +++++++++-- .../MaaFramework/Controller/ControllerAgent.h | 12 +- source/MaaWin32ControlUnit/Base/UnitBase.h | 11 + .../Manager/Win32ControlUnitMgr.cpp | 51 +- .../Manager/Win32ControlUnitMgr.h | 7 +- .../Screencap/DesktopDupWindowScreencap.cpp | 65 +- .../Screencap/DesktopDupWindowScreencap.h | 8 +- .../Screencap/FramePoolScreencap.cpp | 1075 ++++++++++++++++- .../Screencap/FramePoolScreencap.h | 36 + .../FramePoolWithPseudoMinimizeScreencap.h | 2 + .../Screencap/GdiScreencap.cpp | 34 + .../Screencap/GdiScreencap.h | 2 + .../Screencap/HdrDisplayUtils.cpp | 371 ++++++ .../Screencap/HdrDisplayUtils.hpp | 24 + .../Screencap/HwndUtils.hpp | 4 +- .../Screencap/PrintWindowScreencap.cpp | 36 +- .../Screencap/PrintWindowScreencap.h | 3 +- .../PrintWindowWithPseudoMinimizeScreencap.h | 2 + .../Screencap/ScreenDCScreencap.cpp | 38 +- .../Screencap/ScreenDCScreencap.h | 3 +- 20 files changed, 2268 insertions(+), 213 deletions(-) create mode 100644 source/MaaWin32ControlUnit/Screencap/HdrDisplayUtils.cpp create mode 100644 source/MaaWin32ControlUnit/Screencap/HdrDisplayUtils.hpp diff --git a/source/MaaFramework/Controller/ControllerAgent.cpp b/source/MaaFramework/Controller/ControllerAgent.cpp index 7268d858b1..60fa1db411 100644 --- a/source/MaaFramework/Controller/ControllerAgent.cpp +++ b/source/MaaFramework/Controller/ControllerAgent.cpp @@ -1,12 +1,205 @@ #include "ControllerAgent.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + #include "Global/OptionMgr.h" #include "Global/PluginMgr.h" #include "MaaFramework/MaaMsg.h" #include "MaaUtils/ImageIo.h" #include "MaaUtils/NoWarningCV.hpp" +#include "MaaUtils/Time.hpp" #include "Resource/ResourceMgr.h" +namespace +{ +constexpr float kLinearToSrgbThreshold = 0.0031308f; +constexpr float kInvSrgbGamma = 1.0f / 2.4f; +constexpr float kHdrDetectEpsilon = 1e-3f; +constexpr float kMinLuminance = 1e-6f; +constexpr float kHdrMiddleGray = 0.18f; +constexpr float kHdrTargetHighlight = 0.85f; +constexpr float kHdrMinExposure = 1.0f / 32.0f; +constexpr float kHdrMaxExposure = 4.0f; +constexpr size_t kHdrTargetSamples = 16384; +constexpr int kHdrProbeImageCount = 5; +constexpr std::string_view kHdrScreenshotEnabledMsg = "Controller.Screenshot.HdrCapture.Enabled"; +constexpr std::string_view kHdrDisplayCompensationEnabledMsg = "Controller.Screenshot.HdrDisplayCompensation.Enabled"; +constexpr std::string_view kHdrScreenshotDisabledMsg = "Controller.Screenshot.HdrCapture.Disabled"; + +struct HdrSceneStats +{ + float min_linear = 0.0f; + float max_linear = 0.0f; + float avg_log_luminance = 0.0f; + float highlight_luminance = 0.0f; + bool valid = false; +}; + +bool is_hdr_rgba_image(const cv::Mat& image) +{ + return image.type() == CV_16FC4 || image.type() == CV_32FC4; +} + +bool get_json_bool(const json::object& obj, const std::string& key) +{ + auto opt = obj.find(key); + return opt.has_value() && opt->is_boolean() && opt->as_boolean(); +} + +float tone_map_filmic(float linear) +{ + linear = std::max(linear, 0.0f); + const float numerator = linear * (2.51f * linear + 0.03f); + const float denominator = linear * (2.43f * linear + 0.59f) + 0.14f; + if (denominator <= 0.0f) { + return 0.0f; + } + return std::clamp(numerator / denominator, 0.0f, 1.0f); +} + +uint8_t linear_to_srgb_u8(float linear) +{ + linear = std::clamp(linear, 0.0f, 1.0f); + + const float srgb = linear <= kLinearToSrgbThreshold ? linear * 12.92f : 1.055f * std::pow(linear, kInvSrgbGamma) - 0.055f; + return static_cast(std::lround(std::clamp(srgb, 0.0f, 1.0f) * 255.0f)); +} + +cv::Mat hdr_rgba_to_bgr8( + const cv::Mat& hdr_rgba, + bool* tone_mapping_applied = nullptr) // 转换HDR RGBA图像到8位BGR图像，并自动应用电影式色调映射 +{ + if (hdr_rgba.empty()) { + if (tone_mapping_applied) { + *tone_mapping_applied = false; + } + return { }; + } + + cv::Mat rgba32f; + if (hdr_rgba.type() == CV_32FC4) { + rgba32f = hdr_rgba; + } + else { + hdr_rgba.convertTo(rgba32f, CV_32FC4); + } + + const auto analyze_scene = [&]() -> HdrSceneStats { + HdrSceneStats stats; + + const double total_pixels = static_cast(rgba32f.rows) * rgba32f.cols; + const int stride = std::max(1, static_cast(std::sqrt(total_pixels / static_cast(kHdrTargetSamples)))); + + stats.min_linear = std::numeric_limits::max(); + stats.max_linear = std::numeric_limits::lowest(); + + std::vector luminances; + luminances.reserve(static_cast((rgba32f.rows / stride + 1) * (rgba32f.cols / stride + 1))); + + double log_sum = 0.0; + size_t sample_count = 0; + + for (int y = 0; y < rgba32f.rows; y += stride) { + const auto* row = rgba32f.ptr(y); + for (int x = 0; x < rgba32f.cols; x += stride) { + const cv::Vec4f& pixel = row[x]; + for (int c = 0; c < 3; ++c) { + stats.min_linear = std::min(stats.min_linear, pixel[c]); + stats.max_linear = std::max(stats.max_linear, pixel[c]); + } + + const float r = std::max(pixel[0], 0.0f); + const float g = std::max(pixel[1], 0.0f); + const float b = std::max(pixel[2], 0.0f); + const float luminance = std::max(kMinLuminance, 0.2126f * r + 0.7152f * g + 0.0722f * b); + + luminances.emplace_back(luminance); + log_sum += std::log(luminance); + ++sample_count; + } + } + + if (sample_count == 0 || luminances.empty()) { + return stats; + } + + const size_t highlight_index = static_cast(std::floor((luminances.size() - 1) * 0.95)); + std::nth_element(luminances.begin(), luminances.begin() + highlight_index, luminances.end()); + + stats.avg_log_luminance = static_cast(std::exp(log_sum / static_cast(sample_count))); + stats.highlight_luminance = luminances[highlight_index]; + stats.valid = true; + return stats; + }; + + const HdrSceneStats stats = analyze_scene(); + const bool needs_tone_mapping = stats.valid && (stats.max_linear > 1.0f + kHdrDetectEpsilon || stats.min_linear < -kHdrDetectEpsilon); + if (tone_mapping_applied) { + *tone_mapping_applied = needs_tone_mapping; + } + float exposure = 1.0f; + if (needs_tone_mapping) { + if (stats.avg_log_luminance > kMinLuminance) { + exposure = kHdrMiddleGray / stats.avg_log_luminance; + } + if (stats.highlight_luminance > kMinLuminance) { + exposure = std::min(exposure, kHdrTargetHighlight / stats.highlight_luminance); + } + exposure = std::clamp(exposure, kHdrMinExposure, kHdrMaxExposure); + + static auto last_hdr_log_time = std::chrono::steady_clock::time_point { }; + const auto now = std::chrono::steady_clock::now(); + if (last_hdr_log_time == std::chrono::steady_clock::time_point { } || now - last_hdr_log_time > std::chrono::seconds(5)) { + last_hdr_log_time = now; + LogInfo << "HDR screenshot tone mapping" << VAR(stats.min_linear) << VAR(stats.max_linear) << VAR(stats.avg_log_luminance) + << VAR(stats.highlight_luminance) << VAR(exposure); + } + } + + cv::Mat bgr8(rgba32f.rows, rgba32f.cols, CV_8UC3); + + cv::parallel_for_(cv::Range(0, rgba32f.rows), [&](const cv::Range& range) { + for (int y = range.start; y < range.end; ++y) { + const auto* src_row = rgba32f.ptr(y); + auto* dst_row = bgr8.ptr(y); + + for (int x = 0; x < rgba32f.cols; ++x) { + float r = std::max(src_row[x][0], 0.0f); + float g = std::max(src_row[x][1], 0.0f); + float b = std::max(src_row[x][2], 0.0f); + + if (needs_tone_mapping) { + r *= exposure; + g *= exposure; + b *= exposure; + + const float luminance = 0.2126f * r + 0.7152f * g + 0.0722f * b; + const float mapped_luminance = tone_map_filmic(luminance); + const float scale = luminance > kMinLuminance ? (mapped_luminance / luminance) : 0.0f; + + r *= scale; + g *= scale; + b *= scale; + } + + dst_row[x] = cv::Vec3b(linear_to_srgb_u8(b), linear_to_srgb_u8(g), linear_to_srgb_u8(r)); + } + } + }); + + return bgr8; +} +} // namespace + MAA_CTRL_NS_BEGIN ControllerAgent::ControllerAgent(std::shared_ptr control_unit) @@ -56,122 +249,106 @@ bool ControllerAgent::set_option(MaaCtrlOption key, MaaOptionValue value, MaaOpt MaaCtrlId ControllerAgent::post_connection() { - auto id = post({ .type = Action::Type::connect }); - return focus_id(id); + return post({ .type = Action::Type::connect }, true); } MaaCtrlId ControllerAgent::post_click(int x, int y, int contact, int pressure) { ClickParam p { .point = { x, y }, .contact = contact, .pressure = pressure }; - auto id = post({ .type = Action::Type::click, .param = std::move(p) }); - return focus_id(id); + return post({ .type = Action::Type::click, .param = std::move(p) }, true); } MaaCtrlId ControllerAgent::post_swipe(int x1, int y1, int x2, int y2, int duration, int contact, int pressure) { + const uint safe_duration = static_cast(std::max(duration, 0)); SwipeParam p { .begin = { x1, y1 }, .end = { { x2, y2 } }, - .duration = { static_cast(duration) }, + .duration = { safe_duration }, .contact = contact, .pressure = pressure }; - auto id = post({ .type = Action::Type::swipe, .param = std::move(p) }); - return focus_id(id); + return post({ .type = Action::Type::swipe, .param = std::move(p) }, true); } MaaCtrlId ControllerAgent::post_click_key(int keycode) { ClickKeyParam p { .keycode = { keycode } }; - auto id = post({ .type = Action::Type::click_key, .param = std::move(p) }); - return focus_id(id); + return post({ .type = Action::Type::click_key, .param = std::move(p) }, true); } MaaCtrlId ControllerAgent::post_input_text(const std::string& text) { InputTextParam p { .text = text }; - auto id = post({ .type = Action::Type::input_text, .param = std::move(p) }); - return focus_id(id); + return post({ .type = Action::Type::input_text, .param = std::move(p) }, true); } MaaCtrlId ControllerAgent::post_start_app(const std::string& intent) { AppParam p { .package = intent }; - auto id = post({ .type = Action::Type::start_app, .param = std::move(p) }); - return focus_id(id); + return post({ .type = Action::Type::start_app, .param = std::move(p) }, true); } MaaCtrlId ControllerAgent::post_stop_app(const std::string& intent) { AppParam p { .package = intent }; - auto id = post({ .type = Action::Type::stop_app, .param = std::move(p) }); - return focus_id(id); + return post({ .type = Action::Type::stop_app, .param = std::move(p) }, true); } MaaCtrlId ControllerAgent::post_screencap() { - auto id = post({ .type = Action::Type::screencap }); - return focus_id(id); + return post({ .type = Action::Type::screencap }, true); } MaaCtrlId ControllerAgent::post_touch_down(int contact, int x, int y, int pressure) { TouchParam p { .contact = contact, .point = { x, y }, .pressure = pressure }; - auto id = post({ .type = Action::Type::touch_down, .param = std::move(p) }); - return focus_id(id); + return post({ .type = Action::Type::touch_down, .param = std::move(p) }, true); } MaaCtrlId ControllerAgent::post_touch_move(int contact, int x, int y, int pressure) { TouchParam p { .contact = contact, .point = { x, y }, .pressure = pressure }; - auto id = post({ .type = Action::Type::touch_move, .param = std::move(p) }); - return focus_id(id); + return post({ .type = Action::Type::touch_move, .param = std::move(p) }, true); } MaaCtrlId ControllerAgent::post_touch_up(int contact) { TouchParam p { .contact = contact }; - auto id = post({ .type = Action::Type::touch_up, .param = std::move(p) }); - return focus_id(id); + return post({ .type = Action::Type::touch_up, .param = std::move(p) }, true); } MaaCtrlId ControllerAgent::post_relative_move(int dx, int dy) { RelativeMoveParam p { .dx = dx, .dy = dy }; - auto id = post({ .type = Action::Type::relative_move, .param = std::move(p) }); - return focus_id(id); + return post({ .type = Action::Type::relative_move, .param = std::move(p) }, true); } MaaCtrlId ControllerAgent::post_key_down(int keycode) { ClickKeyParam p { .keycode = { keycode } }; - auto id = post({ .type = Action::Type::key_down, .param = std::move(p) }); - return focus_id(id); + return post({ .type = Action::Type::key_down, .param = std::move(p) }, true); } MaaCtrlId ControllerAgent::post_key_up(int keycode) { ClickKeyParam p { .keycode = { keycode } }; - auto id = post({ .type = Action::Type::key_up, .param = std::move(p) }); - return focus_id(id); + return post({ .type = Action::Type::key_up, .param = std::move(p) }, true); } MaaCtrlId ControllerAgent::post_scroll(int dx, int dy) { ScrollParam p { .dx = dx, .dy = dy }; - auto id = post({ .type = Action::Type::scroll, .param = std::move(p) }); - return focus_id(id); + return post({ .type = Action::Type::scroll, .param = std::move(p) }, true); } MaaCtrlId ControllerAgent::post_shell(const std::string& cmd, int64_t timeout) { ShellParam p { .cmd = cmd, .shell_timeout = timeout }; - auto id = post({ .type = Action::Type::shell, .param = std::move(p) }); - return focus_id(id); + return post({ .type = Action::Type::shell, .param = std::move(p) }, true); } MaaCtrlId ControllerAgent::post_inactive() { - auto id = post({ .type = Action::Type::inactive }); - return focus_id(id); + return post({ .type = Action::Type::inactive }, true); } MaaStatus ControllerAgent::status(MaaCtrlId ctrl_id) const @@ -219,14 +396,23 @@ std::string ControllerAgent::cached_shell_output() const std::string ControllerAgent::get_uuid() { - if (uuid_cache_.empty()) { - request_uuid(); + { + std::unique_lock lock(uuid_mutex_); + if (!uuid_cache_.empty()) { + return uuid_cache_; + } } + + request_uuid(); + + std::unique_lock lock(uuid_mutex_); return uuid_cache_; } bool ControllerAgent::get_resolution(int32_t& width, int32_t& height) const { + std::unique_lock lock(image_mutex_); + if (image_raw_width_ == 0 || image_raw_height_ == 0) { return false; } @@ -238,6 +424,10 @@ bool ControllerAgent::get_resolution(int32_t& width, int32_t& height) const json::object ControllerAgent::get_info() const { + if (!control_unit_) { + LogError << "control_unit_ is nullptr"; + return {}; + } return control_unit_->get_info(); } @@ -260,7 +450,10 @@ void ControllerAgent::post_stop() { LogFunc; - need_to_stop_ = true; + { + std::unique_lock lock(stop_mutex_); + need_to_stop_ = true; + } if (action_runner_ && action_runner_->running()) { action_runner_->clear(); @@ -400,7 +593,7 @@ bool ControllerAgent::handle_inactive() return ret; } -MaaCtrlId ControllerAgent::post(Action action) +MaaCtrlId ControllerAgent::post(Action action, bool notify) { // LogInfo << VAR(action.type) << VAR(action.param); @@ -411,17 +604,12 @@ MaaCtrlId ControllerAgent::post(Action action) if (!action_runner_) { return MaaInvalidId; } - return action_runner_->post(std::move(action)); -} - -MaaCtrlId ControllerAgent::focus_id(MaaCtrlId id) -{ - if (id == MaaInvalidId) { - return id; - } - std::unique_lock lock { focus_ids_mutex_ }; - return *focus_ids_.emplace(id).first; + MaaCtrlId id = action_runner_->post(std::move(action)); + if (notify && id != MaaInvalidId) { + focus_ids_.emplace(id); + } + return id; } bool ControllerAgent::handle_connect() @@ -434,6 +622,14 @@ bool ControllerAgent::handle_connect() bool ret = control_unit_->connect(); request_uuid(); + { + std::unique_lock lock(image_mutex_); + hdr_screenshot_mode_notified_ = false; + last_hdr_capture_active_ = false; + last_hdr_gpu_processed_ = false; + last_hdr_display_compensated_ = false; + hdr_probe_images_remaining_ = 0; + } return ret; } @@ -508,31 +704,27 @@ bool ControllerAgent::handle_swipe(const SwipeParam& param) const uint end_hold = param.end_hold.empty() ? 0 : (i < param.end_hold.size()) ? param.end_hold.at(i) : param.end_hold.back(); if (use_touch_down_up) { - constexpr double kInterval = 10; // ms - const std::chrono::milliseconds delay(static_cast(kInterval)); - - const double total_step = duration / kInterval; - const double x_step_len = (end.x - begin.x) / total_step; - const double y_step_len = (end.y - begin.y) / total_step; + constexpr int kIntervalMs = 10; + const std::chrono::milliseconds delay(kIntervalMs); + const size_t total_steps = std::max(1, static_cast(std::ceil(static_cast(duration) / kIntervalMs))); + const double x_step_len = static_cast(end.x - begin.x) / static_cast(total_steps); + const double y_step_len = static_cast(end.y - begin.y) / static_cast(total_steps); auto now = std::chrono::steady_clock::now(); - for (int step = 1; step < total_step; ++step) { - int mx = static_cast(begin.x + step * x_step_len); - int my = static_cast(begin.y + step * y_step_len); + for (size_t step = 1; step <= total_steps; ++step) { + int mx = end.x; + int my = end.y; + if (step < total_steps) { + mx = static_cast(std::round(begin.x + step * x_step_len)); + my = static_cast(std::round(begin.y + step * y_step_len)); + } std::this_thread::sleep_until(now + delay); now = std::chrono::steady_clock::now(); ret &= control_unit_->touch_move(param.contact, mx, my, param.pressure); } - - std::this_thread::sleep_until(now + delay); - - now = std::chrono::steady_clock::now(); - ret &= control_unit_->touch_move(param.contact, end.x, end.y, param.pressure); - - std::this_thread::sleep_until(now + delay); } else { ret &= control_unit_->swipe(begin.x, begin.y, end.x, end.y, duration); @@ -562,29 +754,35 @@ bool ControllerAgent::handle_multi_swipe(const MultiSwipeParam& param) return false; } - constexpr double kInterval = 10; // ms - const std::chrono::milliseconds delay(static_cast(kInterval)); + constexpr int kIntervalMs = 10; + const std::chrono::milliseconds delay(kIntervalMs); struct SegmentOperating { cv::Point begin { }; cv::Point end { }; - double total_step = 0; - double x_step_len = 0; - double y_step_len = 0; + size_t total_steps = 1; + double x_step_len = 0.0; + double y_step_len = 0.0; size_t step_index = 0; - uint duration = 0; uint end_hold = 0; }; - struct CotactOperating + struct ContactOperating { std::vector seg; size_t seg_index = 0; + uint hold_until = 0; + bool started = false; + bool finished = false; + }; + + const auto calc_total_steps = [](uint duration_ms) -> size_t { + return std::max(1, static_cast(std::ceil(static_cast(duration_ms) / kIntervalMs))); }; // contact index < end index < op >> - std::vector operating(param.swipes.size()); + std::vector operating(param.swipes.size()); for (size_t s_i = 0; s_i < param.swipes.size(); ++s_i) { const SwipeParam& s = param.swipes.at(s_i); @@ -598,15 +796,12 @@ bool ControllerAgent::handle_multi_swipe(const MultiSwipeParam& param) SegmentOperating o; o.begin = begin; o.end = end; - o.total_step = duration / kInterval; - o.x_step_len = (end.x - begin.x) / o.total_step; - o.y_step_len = (end.y - begin.y) / o.total_step; - o.step_index = 0; - o.duration = duration; + o.total_steps = calc_total_steps(duration); + o.x_step_len = static_cast(end.x - begin.x) / static_cast(o.total_steps); + o.y_step_len = static_cast(end.y - begin.y) / static_cast(o.total_steps); o.end_hold = end_hold; operating.at(s_i).seg.emplace_back(std::move(o)); - operating.at(s_i).seg_index = 0; begin = end; } @@ -616,8 +811,8 @@ bool ControllerAgent::handle_multi_swipe(const MultiSwipeParam& param) auto now = starting; bool ret = !param.swipes.empty(); - size_t over_count = 0; - while (over_count < param.swipes.size()) { + size_t finished_count = 0; + while (finished_count < param.swipes.size()) { uint now_point = static_cast(std::chrono::duration_cast(now - starting).count()); for (size_t i = 0; i < param.swipes.size(); ++i) { @@ -634,33 +829,62 @@ bool ControllerAgent::handle_multi_swipe(const MultiSwipeParam& param) } auto& contact_op = operating.at(i); - if (contact_op.seg_index >= contact_op.seg.size()) { - continue; // all segments done + if (contact_op.finished) { + continue; } - auto& seg_op = contact_op.seg.at(contact_op.seg_index); - - if (seg_op.step_index == 0) { + if (contact_op.seg.empty()) { + LogError << "Invalid multi swipe: empty path" << VAR(i); + contact_op.finished = true; + ++finished_count; + ret = false; + continue; + } + if (now_point < contact_op.hold_until) { + continue; + } + if (!contact_op.started) { if (!s.only_hover) { - ret &= control_unit_->touch_down(contact, seg_op.begin.x, seg_op.begin.y, s.pressure); + const auto& first_seg = contact_op.seg.front(); + ret &= control_unit_->touch_down(contact, first_seg.begin.x, first_seg.begin.y, s.pressure); } - ++seg_op.step_index; + contact_op.started = true; } - else if (seg_op.step_index < seg_op.total_step) { - int mx = static_cast(seg_op.begin.x + seg_op.step_index * seg_op.x_step_len); - int my = static_cast(seg_op.begin.y + seg_op.step_index * seg_op.y_step_len); - ret &= control_unit_->touch_move(contact, mx, my, s.pressure); - ++seg_op.step_index; - } - else if (seg_op.step_index == seg_op.total_step) { + + if (contact_op.seg_index >= contact_op.seg.size()) { if (!s.only_hover) { ret &= control_unit_->touch_up(contact); } + contact_op.finished = true; + ++finished_count; + continue; + } + + auto& seg_op = contact_op.seg.at(contact_op.seg_index); + if (seg_op.step_index < seg_op.total_steps) { ++seg_op.step_index; - ++over_count; + + int mx = seg_op.end.x; + int my = seg_op.end.y; + if (seg_op.step_index < seg_op.total_steps) { + mx = static_cast(std::round(seg_op.begin.x + seg_op.step_index * seg_op.x_step_len)); + my = static_cast(std::round(seg_op.begin.y + seg_op.step_index * seg_op.y_step_len)); + } + + ret &= control_unit_->touch_move(contact, mx, my, s.pressure); + if (seg_op.step_index == seg_op.total_steps) { + contact_op.hold_until = now_point + seg_op.end_hold; + } } - else { // step_index > total + else { ++contact_op.seg_index; - continue; + contact_op.hold_until = 0; + if (contact_op.seg_index >= contact_op.seg.size()) { + if (!s.only_hover) { + ret &= control_unit_->touch_up(contact); + } + contact_op.finished = true; + ++finished_count; + } } } @@ -910,6 +1134,8 @@ bool ControllerAgent::handle_shell(const ShellParam& param) bool ControllerAgent::check_stop() { + std::unique_lock lock(stop_mutex_); + if (!need_to_stop_) { return true; } @@ -933,18 +1159,23 @@ bool ControllerAgent::run_action(typename AsyncRunner::Id id, Action act notify = focus_ids_.erase(id) > 0; } - const json::value cb_detail = { - { "ctrl_id", id }, - { "uuid", get_uuid() }, - { "action", action.type }, - { "param", action.param }, - { "info", control_unit_->get_info() }, - }; + const auto make_cb_detail = [&]() -> json::value { + json::object info; + if (control_unit_) { + info = control_unit_->get_info(); + } - // LogInfo << cb_detail.to_string(); + return { + { "ctrl_id", id }, + { "uuid", get_uuid() }, + { "action", action.type }, + { "param", action.param }, + { "info", std::move(info) }, + }; + }; if (notify) { - notifier_.notify(this, MaaMsg_Controller_Action_Starting, cb_detail); + notifier_.notify(this, MaaMsg_Controller_Action_Starting, make_cb_detail()); } switch (action.type) { @@ -1025,11 +1256,8 @@ bool ControllerAgent::run_action(typename AsyncRunner::Id id, Action act ret = false; } - if (ret && notify) { - notifier_.notify(this, MaaMsg_Controller_Action_Succeeded, cb_detail); - } - else if (!ret) { - notifier_.notify(this, MaaMsg_Controller_Action_Failed, cb_detail); + if (notify) { + notifier_.notify(this, ret ? MaaMsg_Controller_Action_Succeeded : MaaMsg_Controller_Action_Failed, make_cb_detail()); } return ret; @@ -1041,16 +1269,33 @@ cv::Point ControllerAgent::preproc_touch_point(const cv::Point& p) return p; } - if (image_target_width_ == 0 || image_target_height_ == 0) { - LogWarn << "Invalid image target size" << VAR(image_target_width_) << VAR(image_target_height_); + auto load_scale_info = [this]() { + std::unique_lock lock(image_mutex_); + return std::array { image_raw_width_, image_raw_height_, image_target_width_, image_target_height_ }; + }; + + auto scale_info = load_scale_info(); + if (scale_info[2] == 0 || scale_info[3] == 0) { + LogWarn << "Invalid image target size" << VAR(scale_info[2]) << VAR(scale_info[3]); if (!init_scale_info()) { return { }; } + + scale_info = load_scale_info(); } - double scale_width = static_cast(image_raw_width_) / image_target_width_; - double scale_height = static_cast(image_raw_height_) / image_target_height_; + const int raw_width = scale_info[0]; + const int raw_height = scale_info[1]; + const int target_width = scale_info[2]; + const int target_height = scale_info[3]; + if (raw_width == 0 || raw_height == 0 || target_width == 0 || target_height == 0) { + LogError << "Invalid scale info" << VAR(raw_width) << VAR(raw_height) << VAR(target_width) << VAR(target_height); + return { }; + } + + double scale_width = static_cast(raw_width) / target_width; + double scale_height = static_cast(raw_height) / target_height; int proced_x = static_cast(std::round(p.x * scale_width)); int proced_y = static_cast(std::round(p.y * scale_height)); @@ -1061,26 +1306,172 @@ cv::Point ControllerAgent::preproc_touch_point(const cv::Point& p) bool ControllerAgent::postproc_screenshot(const cv::Mat& raw) { if (raw.empty()) { + std::unique_lock lock(image_mutex_); image_ = cv::Mat(); LogError << "Empty screenshot"; return false; } - if (raw.cols != image_raw_width_ || raw.rows != image_raw_height_ || image_target_width_ == 0 || image_target_height_ == 0) { - LogInfo << "Resolution changed" << VAR(raw.cols) << VAR(raw.rows) << VAR(image_raw_width_) << VAR(image_raw_height_); + const json::object control_info = control_unit_ ? control_unit_->get_info() : json::object {}; + const bool hdr_raw_image = is_hdr_rgba_image(raw); + const bool hdr_capture_active = hdr_raw_image || get_json_bool(control_info, "hdr_capture_active"); + const bool hdr_preprocessed_upstream = !hdr_raw_image && get_json_bool(control_info, "hdr_preprocessed"); + const bool hdr_gpu_processed = get_json_bool(control_info, "hdr_gpu_processed"); + const bool hdr_display_compensated = get_json_bool(control_info, "display_hdr_compensated"); + int target_width = 0; + int target_height = 0; + int resize_method = cv::INTER_AREA; + bool notify_hdr_mode = false; + + { + std::unique_lock lock(image_mutex_); + if (raw.cols != image_raw_width_ || raw.rows != image_raw_height_ || image_target_width_ == 0 || image_target_height_ == 0) { + LogInfo << "Resolution changed" << VAR(raw.cols) << VAR(raw.rows) << VAR(image_raw_width_) << VAR(image_raw_height_); + + image_raw_width_ = raw.cols; + image_raw_height_ = raw.rows; + + if (!calc_target_image_size()) { + image_ = cv::Mat(); + LogError << "Invalid target image size"; + return false; + } + } - image_raw_width_ = raw.cols; - image_raw_height_ = raw.rows; + target_width = image_target_width_; + target_height = image_target_height_; + resize_method = image_resize_method_; + + const bool hdr_mode_changed = !hdr_screenshot_mode_notified_ || last_hdr_capture_active_ != hdr_capture_active + || last_hdr_gpu_processed_ != hdr_gpu_processed + || last_hdr_display_compensated_ != hdr_display_compensated; + if (hdr_mode_changed) { + hdr_screenshot_mode_notified_ = true; + last_hdr_capture_active_ = hdr_capture_active; + last_hdr_gpu_processed_ = hdr_gpu_processed; + last_hdr_display_compensated_ = hdr_display_compensated; + hdr_probe_images_remaining_ = (hdr_capture_active || hdr_display_compensated) ? kHdrProbeImageCount : 0; + notify_hdr_mode = true; + } + } - if (!calc_target_image_size()) { + if (notify_hdr_mode) { + notify_hdr_screenshot_mode(hdr_capture_active, hdr_gpu_processed, hdr_display_compensated); + } + + cv::Mat processed; + if (hdr_capture_active && !hdr_preprocessed_upstream) { + // Some OpenCV builds are unstable when resizing CV_16FC4 frames directly. + // Tone-map to an 8-bit preview first, then resize in SDR space. + cv::Mat hdr_bgr8 = hdr_rgba_to_bgr8(raw); + if (hdr_bgr8.empty()) { + std::unique_lock lock(image_mutex_); + image_ = cv::Mat(); + return false; + } + + if (hdr_bgr8.cols == target_width && hdr_bgr8.rows == target_height) { + processed = std::move(hdr_bgr8); + } + else { + cv::resize(hdr_bgr8, processed, { target_width, target_height }, 0, 0, resize_method); + } + } + else { + cv::Mat resized; + if (raw.cols == target_width && raw.rows == target_height) { + resized = raw; + } + else { + cv::resize(raw, resized, { target_width, target_height }, 0, 0, resize_method); + } + + if (resized.type() == CV_8UC3) { + processed = resized.clone(); + } + else if (resized.type() == CV_8UC4) { + cv::cvtColor(resized, processed, cv::COLOR_BGRA2BGR); + } + else { + LogError << "Unsupported screenshot type" << VAR(resized.type()); + std::unique_lock lock(image_mutex_); image_ = cv::Mat(); - LogError << "Invalid target image size"; return false; } } - cv::resize(raw, image_, { image_target_width_, image_target_height_ }, 0, 0, image_resize_method_); - return !image_.empty(); + int probe_index = 0; + { + std::unique_lock lock(image_mutex_); + image_ = processed; + + if (!processed.empty() && hdr_probe_images_remaining_ > 0) { + probe_index = kHdrProbeImageCount - hdr_probe_images_remaining_ + 1; + --hdr_probe_images_remaining_; + } + } + + if (probe_index > 0) { + save_hdr_probe_image(processed, probe_index); + } + + return !processed.empty(); +} + +void ControllerAgent::notify_hdr_screenshot_mode(bool hdr_capture_active, bool gpu_processed, bool hdr_display_compensated) +{ + const std::string_view message = hdr_capture_active ? kHdrScreenshotEnabledMsg + : (hdr_display_compensated ? kHdrDisplayCompensationEnabledMsg + : kHdrScreenshotDisabledMsg); + const std::string_view content = + hdr_capture_active + ? (gpu_processed + ? "HDR capture path detected. GPU preprocessing is active before readback, and 5 processed screenshots are being saved under `debug/hdr_probe/`. 检测到 HDR 捕获路径，回传内存前已启用 GPU 预处理，并正在额外保存 5 张处理后的截图到 `debug/hdr_probe/`。" + : "HDR capture path detected. Processed preview is enabled, and 5 processed screenshots are being saved under `debug/hdr_probe/`. 检测到 HDR 捕获路径，已启用处理后的预览，并正在额外保存 5 张处理后的截图到 `debug/hdr_probe/`。") + : (hdr_display_compensated + ? "HDR display compensation is active for the current screenshot method, and 5 processed screenshots are being saved under `debug/hdr_probe/`. 当前截图方式已启用 HDR 显示补偿，并正在额外保存 5 张处理后的截图到 `debug/hdr_probe/`。" + : "HDR capture path not detected. Using the legacy SDR screenshot path. 未检测到 HDR 捕获路径，正在使用原有的 SDR 截图路径。"); + + const json::value cb_detail = json::object { + { "uuid", get_uuid() }, + { "focus", + json::object { + { std::string(message), + json::object { + { "content", std::string(content) }, + { "display", "log" }, + } }, + } }, + }; + + notifier_.notify(this, message, cb_detail); +} + +void ControllerAgent::save_hdr_probe_image(const cv::Mat& image, int probe_index) +{ + if (image.empty()) { + LogWarn << "HDR probe image is empty"; + return; + } + + const auto& option = MAA_GLOBAL_NS::OptionMgr::get_instance(); + const auto probe_dir = std::filesystem::absolute(option.log_dir() / "hdr_probe"); + std::error_code ec; + std::filesystem::create_directories(probe_dir, ec); + if (ec) { + LogError << "Failed to create HDR probe directory" << VAR(probe_dir) << VAR(ec.message()); + return; + } + + const auto filename = std::format("{}_hdr_probe_{:0>2}.png", format_now_for_filename(), probe_index); + const auto filepath = probe_dir / path(filename); + + if (imwrite(filepath, image)) { + LogInfo << "Saved HDR probe screenshot" << VAR(probe_index) << VAR(filepath); + } + else { + LogError << "Failed to save HDR probe screenshot" << VAR(probe_index) << VAR(filepath); + } } bool ControllerAgent::calc_target_image_size() @@ -1147,9 +1538,17 @@ bool ControllerAgent::request_uuid() return false; } - uuid_cache_.clear(); + std::string uuid; + bool ret = control_unit_->request_uuid(uuid); - return control_unit_->request_uuid(uuid_cache_); + std::unique_lock lock(uuid_mutex_); + if (!ret) { + uuid_cache_.clear(); + return false; + } + + uuid_cache_ = std::move(uuid); + return true; } bool ControllerAgent::init_scale_info() @@ -1162,13 +1561,23 @@ bool ControllerAgent::set_image_target_long_side(MaaOptionValue value, MaaOption { LogDebug; + if (!value) { + LogError << "option value is nullptr"; + return false; + } if (val_size != sizeof(int32_t)) { LogError << "invalid value size: " << val_size; return false; } - image_target_long_side_ = *reinterpret_cast(value); - image_target_short_side_ = 0; + const auto target_long_side = *reinterpret_cast(value); + if (target_long_side < 0) { + LogError << "invalid image target long side: " << target_long_side; + return false; + } + std::unique_lock lock(image_mutex_); + image_target_long_side_ = target_long_side; + image_target_short_side_ = 0; clear_target_image_size(); LogInfo << "image_target_width_ = " << image_target_long_side_; @@ -1179,13 +1588,23 @@ bool ControllerAgent::set_image_target_short_side(MaaOptionValue value, MaaOptio { LogDebug; + if (!value) { + LogError << "option value is nullptr"; + return false; + } if (val_size != sizeof(int32_t)) { LogError << "invalid value size: " << val_size; return false; } - image_target_long_side_ = 0; - image_target_short_side_ = *reinterpret_cast(value); + const auto target_short_side = *reinterpret_cast(value); + if (target_short_side < 0) { + LogError << "invalid image target short side: " << target_short_side; + return false; + } + std::unique_lock lock(image_mutex_); + image_target_long_side_ = 0; + image_target_short_side_ = target_short_side; clear_target_image_size(); LogInfo << "image_target_height_ = " << image_target_short_side_; @@ -1196,10 +1615,15 @@ bool ControllerAgent::set_image_use_raw_size(MaaOptionValue value, MaaOptionValu { LogDebug; + if (!value) { + LogError << "option value is nullptr"; + return false; + } if (val_size != sizeof(bool)) { LogError << "invalid value size: " << val_size; return false; } + std::unique_lock lock(image_mutex_); image_use_raw_size_ = *reinterpret_cast(value); clear_target_image_size(); @@ -1211,6 +1635,10 @@ bool ControllerAgent::set_mouse_lock_follow_option(MaaOptionValue value, MaaOpti { LogDebug; + if (!value) { + LogError << "option value is nullptr"; + return false; + } if (val_size != sizeof(bool)) { LogError << "invalid value size: " << val_size; return false; @@ -1235,6 +1663,10 @@ bool ControllerAgent::set_screenshot_resize_method(MaaOptionValue value, MaaOpti { LogDebug; + if (!value) { + LogError << "option value is nullptr"; + return false; + } if (val_size != sizeof(int32_t)) { LogError << "invalid value size: " << val_size; return false; @@ -1247,6 +1679,7 @@ bool ControllerAgent::set_screenshot_resize_method(MaaOptionValue value, MaaOpti return false; } + std::unique_lock lock(image_mutex_); image_resize_method_ = raw; LogInfo << "image_resize_method_ = " << image_resize_method_; return true; @@ -1256,6 +1689,10 @@ bool ControllerAgent::set_background_managed_keys_option(MaaOptionValue value, M { LogDebug; + if (val_size > 0 && !value) { + LogError << "option value is nullptr"; + return false; + } if (val_size != 0 && val_size % sizeof(int32_t) != 0) { LogError << "invalid value size: " << val_size; return false; diff --git a/source/MaaFramework/Controller/ControllerAgent.h b/source/MaaFramework/Controller/ControllerAgent.h index b69d68eb8c..76d96a1078 100644 --- a/source/MaaFramework/Controller/ControllerAgent.h +++ b/source/MaaFramework/Controller/ControllerAgent.h @@ -271,14 +271,15 @@ class ControllerAgent : public MaaController bool handle_shell(const ShellParam& param); bool handle_inactive(); - MaaCtrlId post(Action action); - MaaCtrlId focus_id(MaaCtrlId id); + MaaCtrlId post(Action action, bool notify = false); bool check_stop(); private: bool run_action(typename AsyncRunner::Id id, Action action); cv::Point preproc_touch_point(const cv::Point& p); bool postproc_screenshot(const cv::Mat& raw); + void notify_hdr_screenshot_mode(bool hdr_capture_active, bool gpu_processed, bool hdr_display_compensated); + void save_hdr_probe_image(const cv::Mat& image, int probe_index); bool calc_target_image_size(); void clear_target_image_size(); bool request_uuid(); @@ -293,6 +294,7 @@ class ControllerAgent : public MaaController bool set_background_managed_keys_option(MaaOptionValue value, MaaOptionValueSize val_size); private: + mutable std::mutex stop_mutex_; bool need_to_stop_ = false; private: @@ -303,6 +305,7 @@ class ControllerAgent : public MaaController cv::Mat image_; mutable std::mutex shell_output_mutex_; std::string shell_output_; + mutable std::mutex uuid_mutex_; bool image_use_raw_size_ = false; int image_target_long_side_ = 0; @@ -314,6 +317,11 @@ class ControllerAgent : public MaaController int image_resize_method_ = 3; // cv::INTER_AREA std::string uuid_cache_; + bool hdr_screenshot_mode_notified_ = false; + bool last_hdr_capture_active_ = false; + bool last_hdr_gpu_processed_ = false; + bool last_hdr_display_compensated_ = false; + int hdr_probe_images_remaining_ = 0; std::set::Id> focus_ids_; std::mutex focus_ids_mutex_; diff --git a/source/MaaWin32ControlUnit/Base/UnitBase.h b/source/MaaWin32ControlUnit/Base/UnitBase.h index dcc4aa3f8f..f4df0d1154 100644 --- a/source/MaaWin32ControlUnit/Base/UnitBase.h +++ b/source/MaaWin32ControlUnit/Base/UnitBase.h @@ -2,6 +2,7 @@ #include +#include "MaaControlUnit/ControlUnitAPI.h" #include "MaaFramework/MaaDef.h" #include "MaaUtils/NoWarningCVMat.hpp" @@ -9,6 +10,15 @@ MAA_CTRL_UNIT_NS_BEGIN +struct ScreencapInfo +{ + bool hdr_capture_active = false; + bool hdr_preprocessed = false; + bool gpu_processed = false; + bool display_hdr_active = false; + bool display_hdr_compensated = false; +}; + class ScreencapBase { public: @@ -16,6 +26,7 @@ class ScreencapBase public: virtual std::optional screencap() = 0; + virtual ScreencapInfo last_screencap_info() const { return {}; } virtual void inactive() { } diff --git a/source/MaaWin32ControlUnit/Manager/Win32ControlUnitMgr.cpp b/source/MaaWin32ControlUnit/Manager/Win32ControlUnitMgr.cpp index 0ffd53a429..a77b3ded5c 100644 --- a/source/MaaWin32ControlUnit/Manager/Win32ControlUnitMgr.cpp +++ b/source/MaaWin32ControlUnit/Manager/Win32ControlUnitMgr.cpp @@ -39,6 +39,8 @@ bool Win32ControlUnitMgr::connect() { connected_ = false; screencap_.reset(); + active_screencap_method_ = ScreencapMethod::UnknownYet; + last_screencap_info_ = {}; #ifndef MAA_WIN32_COMPATIBLE // 设置 Per-Monitor DPI Aware V2，确保 GetClientRect/GetWindowRect 等 API 返回物理像素。 @@ -107,6 +109,27 @@ std::unordered_map Win32ControlUnitMgr::make_input(MaaWin32InputMethod m } std::shared_ptr - Win32ControlUnitMgr::speed_test(const std::unordered_map>& units) const + Win32ControlUnitMgr::speed_test(const std::unordered_map>& units) { LogFunc; @@ -202,6 +225,7 @@ std::shared_ptr return nullptr; } + active_screencap_method_ = fastest; LogInfo << "The fastest method is" << fastest << VAR(cost); return units.at(fastest); } @@ -270,11 +294,25 @@ bool Win32ControlUnitMgr::screencap(cv::Mat& image) auto opt = screencap_->screencap(); if (!opt) { - LogError << "failed to screencap"; - return false; + LogWarn << "failed to screencap, reinitializing screencap method"; + if (!init_screencap()) { + LogError << "failed to reinitialize screencap method"; + return false; + } + if (!screencap_) { + LogError << "screencap_ is null after reinitialization"; + return false; + } + + opt = screencap_->screencap(); + if (!opt) { + LogError << "failed to screencap after reinitialization"; + return false; + } } image = std::move(opt).value(); + last_screencap_info_ = screencap_->last_screencap_info(); return true; } @@ -485,8 +523,15 @@ json::object Win32ControlUnitMgr::get_info() const info["type"] = "win32"; info["hwnd"] = reinterpret_cast(hwnd_); info["screencap_method"] = static_cast(screencap_method_); + info["active_screencap_method"] = static_cast(active_screencap_method_); + info["active_screencap_method_name"] = screencap_method_name(active_screencap_method_); info["mouse_method"] = static_cast(mouse_method_); info["keyboard_method"] = static_cast(keyboard_method_); + info["hdr_capture_active"] = last_screencap_info_.hdr_capture_active; + info["hdr_preprocessed"] = last_screencap_info_.hdr_preprocessed; + info["hdr_gpu_processed"] = last_screencap_info_.gpu_processed; + info["display_hdr_active"] = last_screencap_info_.display_hdr_active; + info["display_hdr_compensated"] = last_screencap_info_.display_hdr_compensated; return info; } diff --git a/source/MaaWin32ControlUnit/Manager/Win32ControlUnitMgr.h b/source/MaaWin32ControlUnit/Manager/Win32ControlUnitMgr.h index 036c365c87..c0651d4daa 100644 --- a/source/MaaWin32ControlUnit/Manager/Win32ControlUnitMgr.h +++ b/source/MaaWin32ControlUnit/Manager/Win32ControlUnitMgr.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -76,8 +77,8 @@ class Win32ControlUnitMgr : public Win32ControlUnitAPI std::unordered_map> build_screencap_units() const; bool init_screencap(); std::shared_ptr make_input(MaaWin32InputMethod method) const; - - std::shared_ptr speed_test(const std::unordered_map>& units) const; + static std::string_view screencap_method_name(ScreencapMethod method); + std::shared_ptr speed_test(const std::unordered_map>& units); private: HWND hwnd_ = nullptr; @@ -89,6 +90,8 @@ class Win32ControlUnitMgr : public Win32ControlUnitAPI std::shared_ptr mouse_ = nullptr; std::shared_ptr keyboard_ = nullptr; std::shared_ptr screencap_ = nullptr; + ScreencapMethod active_screencap_method_ = ScreencapMethod::UnknownYet; + ScreencapInfo last_screencap_info_ {}; std::shared_ptr background_keyboard_ = nullptr; std::unordered_set managed_keys_; diff --git a/source/MaaWin32ControlUnit/Screencap/DesktopDupWindowScreencap.cpp b/source/MaaWin32ControlUnit/Screencap/DesktopDupWindowScreencap.cpp index 31d2b08914..0b97de638d 100644 --- a/source/MaaWin32ControlUnit/Screencap/DesktopDupWindowScreencap.cpp +++ b/source/MaaWin32ControlUnit/Screencap/DesktopDupWindowScreencap.cpp @@ -5,15 +5,59 @@ #include "HwndUtils.hpp" #include "MaaUtils/Logger.h" +namespace +{ +struct MonitorCountEnumContext +{ + int count = 0; +}; + +BOOL CALLBACK count_intersecting_monitors_proc(HMONITOR, HDC, LPRECT, LPARAM lparam) +{ + auto& context = *reinterpret_cast(lparam); + ++context.count; + return TRUE; +} + +bool rect_spans_multiple_monitors(const RECT& rect) +{ + if (rect.right <= rect.left || rect.bottom <= rect.top) { + return false; + } + + MonitorCountEnumContext context { }; + if (!EnumDisplayMonitors(nullptr, &rect, count_intersecting_monitors_proc, reinterpret_cast(&context))) { + LogWarn << "EnumDisplayMonitors failed" << GetLastError(); + return false; + } + + return context.count > 1; +} +} // namespace + MAA_CTRL_UNIT_NS_BEGIN std::optional DesktopDupWindowScreencap::screencap() { if (!hwnd_) { LogError << "hwnd_ is nullptr"; + last_screencap_info_ = {}; + return std::nullopt; + } + + RECT initial_client_rect_screen = get_window_client_rect_screen(); + if (initial_client_rect_screen.right <= initial_client_rect_screen.left || initial_client_rect_screen.bottom <= initial_client_rect_screen.top) { + LogError << "Invalid initial client rect" << VAR(initial_client_rect_screen.left) << VAR(initial_client_rect_screen.top) + << VAR(initial_client_rect_screen.right) << VAR(initial_client_rect_screen.bottom); + last_screencap_info_ = {}; return std::nullopt; } + if (rect_spans_multiple_monitors(initial_client_rect_screen)) { + LogInfo << "Client rect spans multiple monitors, falling back to screen DC capture"; + return screencap_from_screen_dc(); + } + // Ensure the window is fully visible on the monitor before screencap if (!ensure_window_on_screen(hwnd_)) { LogWarn << "Failed to ensure window on screen"; @@ -22,6 +66,7 @@ std::optional DesktopDupWindowScreencap::screencap() // 调用基类方法获取全屏截图（BGR格式） auto opt_img = DesktopDupScreencap::screencap(); if (!opt_img) { + last_screencap_info_ = {}; return std::nullopt; } const cv::Mat& img = *opt_img; @@ -31,14 +76,15 @@ std::optional DesktopDupWindowScreencap::screencap() if (client_rect_screen.right <= client_rect_screen.left || client_rect_screen.bottom <= client_rect_screen.top) { LogError << "Invalid client rect" << VAR(client_rect_screen.left) << VAR(client_rect_screen.top) << VAR(client_rect_screen.right) << VAR(client_rect_screen.bottom); + last_screencap_info_ = {}; return std::nullopt; } // 获取当前输出（显示器）的桌面坐标 RECT output_desktop = get_output_desktop_coordinates(); if (output_desktop.right <= output_desktop.left || output_desktop.bottom <= output_desktop.top) { - LogError << "Failed to get output desktop coordinates"; - return std::nullopt; + LogWarn << "Failed to get output desktop coordinates, falling back to screen DC capture"; + return screencap_from_screen_dc(); } // 将窗口坐标转换为相对于该显示器的坐标 @@ -49,18 +95,26 @@ std::optional DesktopDupWindowScreencap::screencap() // 检查裁剪区域是否在图像范围内 if (crop_x < 0 || crop_y < 0 || crop_x + client_width > img.cols || crop_y + client_height > img.rows) { - LogError << "Client rect out of bounds" << VAR(crop_x) << VAR(crop_y) << VAR(client_width) << VAR(client_height) << VAR(img.cols) - << VAR(img.rows) << VAR(output_desktop.left) << VAR(output_desktop.top); - return std::nullopt; + LogWarn << "Client rect out of bounds, falling back to screen DC capture" << VAR(crop_x) << VAR(crop_y) << VAR(client_width) + << VAR(client_height) << VAR(img.cols) << VAR(img.rows) << VAR(output_desktop.left) << VAR(output_desktop.top); + return screencap_from_screen_dc(); } // 裁剪出窗口客户区 cv::Rect roi(crop_x, crop_y, client_width, client_height); cv::Mat cropped = img(roi); + last_screencap_info_ = {}; return cropped; } +std::optional DesktopDupWindowScreencap::screencap_from_screen_dc() +{ + auto fallback = screen_dc_fallback_.screencap(); + last_screencap_info_ = screen_dc_fallback_.last_screencap_info(); + return fallback; +} + RECT DesktopDupWindowScreencap::get_window_client_rect_screen() const { RECT client_rect = { 0 }; @@ -110,4 +164,3 @@ RECT DesktopDupWindowScreencap::get_output_desktop_coordinates() const } MAA_CTRL_UNIT_NS_END - diff --git a/source/MaaWin32ControlUnit/Screencap/DesktopDupWindowScreencap.h b/source/MaaWin32ControlUnit/Screencap/DesktopDupWindowScreencap.h index e183473883..9ab4065b27 100644 --- a/source/MaaWin32ControlUnit/Screencap/DesktopDupWindowScreencap.h +++ b/source/MaaWin32ControlUnit/Screencap/DesktopDupWindowScreencap.h @@ -5,6 +5,7 @@ #include "Base/UnitBase.h" #include "DesktopDupScreencap.h" +#include "ScreenDCScreencap.h" MAA_CTRL_UNIT_NS_BEGIN @@ -13,6 +14,7 @@ class DesktopDupWindowScreencap : public DesktopDupScreencap public: explicit DesktopDupWindowScreencap(HWND hwnd) : DesktopDupScreencap(hwnd) + , screen_dc_fallback_(hwnd) { } @@ -20,11 +22,15 @@ class DesktopDupWindowScreencap : public DesktopDupScreencap public: // from ScreencapBase virtual std::optional screencap() override; + virtual ScreencapInfo last_screencap_info() const override { return last_screencap_info_; } private: + std::optional screencap_from_screen_dc(); RECT get_window_client_rect_screen() const; RECT get_output_desktop_coordinates() const; + + ScreenDCScreencap screen_dc_fallback_; + ScreencapInfo last_screencap_info_ {}; }; MAA_CTRL_UNIT_NS_END - diff --git a/source/MaaWin32ControlUnit/Screencap/FramePoolScreencap.cpp b/source/MaaWin32ControlUnit/Screencap/FramePoolScreencap.cpp index 2cd3fcc145..04c13876dc 100644 --- a/source/MaaWin32ControlUnit/Screencap/FramePoolScreencap.cpp +++ b/source/MaaWin32ControlUnit/Screencap/FramePoolScreencap.cpp @@ -2,6 +2,7 @@ #if MAA_FRAMEPOOL_SCREENCAP_AVAILABLE +#include #include #include #include @@ -9,13 +10,474 @@ #include #include +#include +#include +#include +#include +#include +#include #include +#include +#include +#include "HdrDisplayUtils.hpp" #include "HwndUtils.hpp" #include "MaaUtils/Logger.h" #include "MaaUtils/NoWarningCV.hpp" #include "MaaUtils/Time.hpp" +namespace +{ +constexpr LONG kDisplayConfigErrorInsufficientBuffer = 122; +constexpr int kMaxDisplayConfigRetries = 3; + +using CaptureFrame = winrt::Windows::Graphics::Capture::Direct3D11CaptureFrame; + +constexpr auto kFramePollInterval = std::chrono::milliseconds(2); +constexpr auto kColdFrameTimeout = std::chrono::milliseconds(500); +constexpr auto kSteadyFrameTimeout = std::chrono::milliseconds(50); +constexpr float kLinearToSrgbThreshold = 0.0031308f; +constexpr float kInvSrgbGamma = 1.0f / 2.4f; +constexpr float kHdrDetectEpsilon = 1e-3f; +constexpr float kMinLuminance = 1e-6f; +constexpr float kHdrMiddleGray = 0.18f; +constexpr float kHdrTargetHighlight = 0.85f; +constexpr float kHdrMinExposure = 1.0f / 32.0f; +constexpr float kHdrMaxExposure = 4.0f; +constexpr int kHdrStatsLongSide = 160; + +struct HdrSceneStats +{ + float min_linear = 0.0f; + float max_linear = 0.0f; + float avg_log_luminance = 0.0f; + float highlight_luminance = 0.0f; + bool valid = false; +}; + +struct alignas(16) HdrGpuShaderConstants +{ + std::array source_offset {}; + std::array source_scale {}; + float exposure = 1.0f; + float apply_tone_mapping = 0.0f; + std::array padding {}; +}; + +cv::Size calc_hdr_stats_size(int width, int height) +{ + if (width <= 0 || height <= 0) { + return {}; + } + + if (width >= height) { + const int stats_width = kHdrStatsLongSide; + const int stats_height = std::max(1, static_cast(std::lround(static_cast(kHdrStatsLongSide) * height / width))); + return { stats_width, stats_height }; + } + + const int stats_height = kHdrStatsLongSide; + const int stats_width = std::max(1, static_cast(std::lround(static_cast(kHdrStatsLongSide) * width / height))); + return { stats_width, stats_height }; +} + +HdrSceneStats analyze_hdr_scene(const cv::Mat& hdr_rgba) +{ + HdrSceneStats stats; + if (hdr_rgba.empty()) { + return stats; + } + + cv::Mat rgba32f; + if (hdr_rgba.type() == CV_32FC4) { + rgba32f = hdr_rgba; + } + else { + hdr_rgba.convertTo(rgba32f, CV_32FC4); + } + + stats.min_linear = std::numeric_limits::max(); + stats.max_linear = std::numeric_limits::lowest(); + + std::vector luminances; + luminances.reserve(static_cast(rgba32f.rows * rgba32f.cols)); + + double log_sum = 0.0; + size_t sample_count = 0; + + for (int y = 0; y < rgba32f.rows; ++y) { + const auto* row = rgba32f.ptr(y); + for (int x = 0; x < rgba32f.cols; ++x) { + const cv::Vec4f& pixel = row[x]; + for (int c = 0; c < 3; ++c) { + stats.min_linear = std::min(stats.min_linear, pixel[c]); + stats.max_linear = std::max(stats.max_linear, pixel[c]); + } + + const float r = std::max(pixel[0], 0.0f); + const float g = std::max(pixel[1], 0.0f); + const float b = std::max(pixel[2], 0.0f); + const float luminance = std::max(kMinLuminance, 0.2126f * r + 0.7152f * g + 0.0722f * b); + luminances.emplace_back(luminance); + log_sum += std::log(luminance); + ++sample_count; + } + } + + if (sample_count == 0 || luminances.empty()) { + return stats; + } + + const size_t highlight_index = static_cast(std::floor((luminances.size() - 1) * 0.95)); + std::nth_element(luminances.begin(), luminances.begin() + highlight_index, luminances.end()); + + stats.avg_log_luminance = static_cast(std::exp(log_sum / static_cast(sample_count))); + stats.highlight_luminance = luminances[highlight_index]; + stats.valid = true; + return stats; +} + +float calc_hdr_exposure(const HdrSceneStats& stats, bool& apply_tone_mapping) +{ + apply_tone_mapping = stats.valid && (stats.max_linear > 1.0f + kHdrDetectEpsilon || stats.min_linear < -kHdrDetectEpsilon); + if (!apply_tone_mapping) { + return 1.0f; + } + + float exposure = 1.0f; + if (stats.avg_log_luminance > kMinLuminance) { + exposure = kHdrMiddleGray / stats.avg_log_luminance; + } + if (stats.highlight_luminance > kMinLuminance) { + exposure = std::min(exposure, kHdrTargetHighlight / stats.highlight_luminance); + } + return std::clamp(exposure, kHdrMinExposure, kHdrMaxExposure); +} + +constexpr std::string_view kHdrFullscreenVertexShader = R"( +struct VSOut +{ + float4 position : SV_POSITION; + float2 uv : TEXCOORD0; +}; + +VSOut main(uint vertex_id : SV_VertexID) +{ + VSOut output; + + float2 positions[3] = { + float2(-1.0, -1.0), + float2(-1.0, 3.0), + float2( 3.0, -1.0) + }; + float2 uvs[3] = { + float2(0.0, 1.0), + float2(0.0, -1.0), + float2(2.0, 1.0) + }; + + output.position = float4(positions[vertex_id], 0.0, 1.0); + output.uv = uvs[vertex_id]; + return output; +} +)"; + +constexpr std::string_view kHdrCopyPixelShader = R"( +cbuffer Params : register(b0) +{ + float2 SourceOffset; + float2 SourceScale; + float Exposure; + float ApplyToneMapping; + float2 Padding; +}; + +Texture2D SourceTexture : register(t0); +SamplerState LinearClampSampler : register(s0); + +struct VSOut +{ + float4 position : SV_POSITION; + float2 uv : TEXCOORD0; +}; + +float4 main(VSOut input) : SV_TARGET +{ + float2 sample_uv = input.uv * SourceScale + SourceOffset; + return SourceTexture.Sample(LinearClampSampler, sample_uv); +} +)"; + +constexpr std::string_view kHdrTonemapPixelShader = R"( +cbuffer Params : register(b0) +{ + float2 SourceOffset; + float2 SourceScale; + float Exposure; + float ApplyToneMapping; + float2 Padding; +}; + +Texture2D SourceTexture : register(t0); +SamplerState LinearClampSampler : register(s0); + +struct VSOut +{ + float4 position : SV_POSITION; + float2 uv : TEXCOORD0; +}; + +float tone_map_filmic(float value) +{ + value = max(value, 0.0); + float numerator = value * (2.51 * value + 0.03); + float denominator = value * (2.43 * value + 0.59) + 0.14; + return denominator > 0.0 ? saturate(numerator / denominator) : 0.0; +} + +float linear_to_srgb(float value) +{ + value = saturate(value); + return value <= 0.0031308 ? value * 12.92 : 1.055 * pow(value, 1.0 / 2.4) - 0.055; +} + +float4 main(VSOut input) : SV_TARGET +{ + float2 sample_uv = input.uv * SourceScale + SourceOffset; + float3 rgb = max(SourceTexture.Sample(LinearClampSampler, sample_uv).rgb, 0.0.xxx); + + if (ApplyToneMapping > 0.5) + { + rgb *= Exposure; + + float luminance = dot(rgb, float3(0.2126, 0.7152, 0.0722)); + float mapped_luminance = tone_map_filmic(luminance); + float scale = luminance > 1e-6 ? (mapped_luminance / luminance) : 0.0; + rgb *= scale; + } + + return float4(linear_to_srgb(rgb.r), linear_to_srgb(rgb.g), linear_to_srgb(rgb.b), 1.0); +} +)"; + +using D3DCompileProc = decltype(&D3DCompile); + +D3DCompileProc get_d3d_compile_proc() +{ + static const auto loader = []() -> std::pair { + constexpr std::array shader_compiler_dlls { + L"d3dcompiler_47.dll", + L"d3dcompiler_46.dll", + L"d3dcompiler_43.dll", + }; + + for (const wchar_t* dll_name : shader_compiler_dlls) { + HMODULE module = LoadLibraryW(dll_name); + if (!module) { + continue; + } + + auto proc = reinterpret_cast(GetProcAddress(module, "D3DCompile")); + if (!proc) { + FreeLibrary(module); + continue; + } + + LogInfo << "Loaded D3D shader compiler dynamically" << VAR(dll_name); + return { module, proc }; + } + + LogWarn << "No D3D shader compiler DLL found; HDR GPU preprocess will fall back to CPU"; + return { nullptr, nullptr }; + }(); + + return loader.second; +} + +bool get_window_frame_bounds(HWND hwnd, RECT& frame_rect) +{ + if (!hwnd || !IsWindow(hwnd)) { + return false; + } + + HRESULT hr = DwmGetWindowAttribute(hwnd, DWMWA_EXTENDED_FRAME_BOUNDS, &frame_rect, sizeof(frame_rect)); + if (SUCCEEDED(hr)) { + return true; + } + + LogWarn << "DwmGetWindowAttribute failed, falling back to GetWindowRect" << VAR(hwnd) << VAR(hr); + return GetWindowRect(hwnd, &frame_rect) != 0; +} + +bool rect_has_area(const RECT& rect) +{ + return rect.right > rect.left && rect.bottom > rect.top; +} + +bool is_texture_desc_compatible(const D3D11_TEXTURE2D_DESC& lhs, const D3D11_TEXTURE2D_DESC& rhs) +{ + return lhs.Width == rhs.Width && lhs.Height == rhs.Height && lhs.MipLevels == rhs.MipLevels && lhs.ArraySize == rhs.ArraySize + && lhs.Format == rhs.Format && lhs.SampleDesc.Count == rhs.SampleDesc.Count && lhs.SampleDesc.Quality == rhs.SampleDesc.Quality; +} + +struct OwnedWindowEnumContext +{ + HWND root = nullptr; + DWORD process_id = 0; + RECT bounds = { 0 }; + bool has_bounds = false; +}; + +BOOL CALLBACK enum_owned_windows_proc(HWND hwnd, LPARAM lparam) +{ + auto& context = *reinterpret_cast(lparam); + if (!hwnd || !IsWindow(hwnd) || !IsWindowVisible(hwnd) || IsIconic(hwnd)) { + return TRUE; + } + + DWORD process_id = 0; + GetWindowThreadProcessId(hwnd, &process_id); + if (process_id != context.process_id) { + return TRUE; + } + + HWND root = GetAncestor(hwnd, GA_ROOTOWNER); + if (root != context.root) { + return TRUE; + } + + RECT frame_rect { 0 }; + if (!get_window_frame_bounds(hwnd, frame_rect)) { + return TRUE; + } + + if (!context.has_bounds) { + context.bounds = frame_rect; + context.has_bounds = true; + } + else { + context.bounds.left = std::min(context.bounds.left, frame_rect.left); + context.bounds.top = std::min(context.bounds.top, frame_rect.top); + context.bounds.right = std::max(context.bounds.right, frame_rect.right); + context.bounds.bottom = std::max(context.bounds.bottom, frame_rect.bottom); + } + + return TRUE; +} + +bool supports_include_secondary_windows() +{ + using namespace winrt::Windows::Foundation::Metadata; + return ApiInformation::IsPropertyPresent(L"Windows.Graphics.Capture.GraphicsCaptureSession", L"IncludeSecondaryWindows"); +} + +std::optional get_capture_bounds(HWND hwnd, bool include_secondary_windows) +{ + RECT frame_rect { 0 }; + if (!get_window_frame_bounds(hwnd, frame_rect) || !rect_has_area(frame_rect)) { + return std::nullopt; + } + + RECT capture_bounds = frame_rect; + if (!include_secondary_windows) { + return capture_bounds; + } + + HWND root = GetAncestor(hwnd, GA_ROOTOWNER); + if (!root) { + root = hwnd; + } + + DWORD process_id = 0; + GetWindowThreadProcessId(hwnd, &process_id); + OwnedWindowEnumContext enum_context { + .root = root, + .process_id = process_id, + .bounds = capture_bounds, + .has_bounds = true, + }; + + EnumWindows(enum_owned_windows_proc, reinterpret_cast(&enum_context)); + if (!enum_context.has_bounds || !rect_has_area(enum_context.bounds)) { + return std::nullopt; + } + + return enum_context.bounds; +} + +struct CaptureMonitorEnumContext +{ + std::vector* monitors = nullptr; +}; + +BOOL CALLBACK enum_capture_monitors_proc(HMONITOR monitor, HDC, LPRECT, LPARAM lparam) +{ + auto& context = *reinterpret_cast(lparam); + if (context.monitors && monitor) { + context.monitors->emplace_back(monitor); + } + return TRUE; +} + +std::vector get_capture_monitors(const RECT& capture_bounds) +{ + std::vector monitors; + if (!rect_has_area(capture_bounds)) { + return monitors; + } + + CaptureMonitorEnumContext context { .monitors = &monitors }; + if (!EnumDisplayMonitors(nullptr, &capture_bounds, enum_capture_monitors_proc, reinterpret_cast(&context))) { + LogWarn << "EnumDisplayMonitors failed" << GetLastError(); + } + + if (monitors.empty()) { + POINT center { + capture_bounds.left + (capture_bounds.right - capture_bounds.left) / 2, + capture_bounds.top + (capture_bounds.bottom - capture_bounds.top) / 2, + }; + HMONITOR fallback_monitor = MonitorFromPoint(center, MONITOR_DEFAULTTONEAREST); + if (fallback_monitor) { + monitors.emplace_back(fallback_monitor); + } + } + + std::sort(monitors.begin(), monitors.end(), [](HMONITOR lhs, HMONITOR rhs) { + return reinterpret_cast(lhs) < reinterpret_cast(rhs); + }); + monitors.erase(std::unique(monitors.begin(), monitors.end()), monitors.end()); + return monitors; +} + +std::optional is_windows_hdr_enabled(const std::vector& target_monitors) +{ + if (target_monitors.empty()) { + LogDebug << "No capture monitors supplied; using global HDR query"; + return MAA_CTRL_UNIT_NS::query_windows_hdr_enabled_go_compatible(); + } + + bool has_valid_result = false; + for (HMONITOR monitor : target_monitors) { + const auto hdr_state = MAA_CTRL_UNIT_NS::query_hdr_display_state(monitor); + if (!hdr_state.has_value() || !hdr_state->valid) { + continue; + } + + has_valid_result = true; + if (hdr_state->hdr_enabled) { + return true; + } + } + + if (has_valid_result) { + return false; + } + + LogWarn << "Per-monitor HDR query failed, falling back to global HDR query"; + return MAA_CTRL_UNIT_NS::query_windows_hdr_enabled_go_compatible(); +} +} // namespace + MAA_CTRL_UNIT_NS_BEGIN FramePoolScreencap::~FramePoolScreencap() @@ -43,23 +505,20 @@ std::optional FramePoolScreencap::screencap() return std::nullopt; } - winrt::Windows::Graphics::Capture::Direct3D11CaptureFrame frame = nullptr; + CaptureFrame frame = nullptr; try { - // 先清空 FramePool 中可能残留的旧帧 - while (auto old_frame = cap_frame_pool_.TryGetNextFrame()) { - old_frame.Close(); - } - - // 等待新帧到来 - using namespace std::chrono_literals; + // 优先消费 frame pool 里现成的最新帧；静态画面拿不到新帧时尽快回退到缓存，避免频繁卡满长超时。 + const auto timeout = cached_image_.empty() ? kColdFrameTimeout : kSteadyFrameTimeout; auto start_time = std::chrono::steady_clock::now(); - while (duration_since(start_time) < 2000ms) { - std::this_thread::sleep_for(2ms); - frame = cap_frame_pool_.TryGetNextFrame(); + while (duration_since(start_time) < timeout) { + while (auto next_frame = cap_frame_pool_.TryGetNextFrame()) { + frame = std::move(next_frame); + } if (frame) { break; } + std::this_thread::sleep_for(kFramePollInterval); } } catch (const winrt::hresult_error& e) { @@ -73,6 +532,7 @@ std::optional FramePoolScreencap::screencap() LogError << "Failed to get frame and no cached image available"; return std::nullopt; } + LogDebug << "No new frame available, reusing cached image"; return cached_image_.clone(); } @@ -95,21 +555,11 @@ std::optional FramePoolScreencap::screencap() return std::nullopt; } - if (!readable_texture_ && !init_texture(texture)) { - LogError << "falied to init_texture"; - return std::nullopt; - } - d3d_context_->CopyResource(readable_texture_.get(), texture.get()); - - D3D11_MAPPED_SUBRESOURCE mapped { 0 }; - ret = d3d_context_->Map(readable_texture_.get(), 0, D3D11_MAP_READ, 0, &mapped); - if (FAILED(ret)) { - LogError << "Map failed" << VAR(ret); - return std::nullopt; - } - OnScopeLeave([&]() { d3d_context_->Unmap(readable_texture_.get(), 0); }); + D3D11_TEXTURE2D_DESC current_desc { }; + texture->GetDesc(¤t_desc); - cv::Mat raw(texture_desc_.Height, texture_desc_.Width, CV_8UC4, mapped.pData, mapped.RowPitch); + const int raw_width = static_cast(current_desc.Width); + const int raw_height = static_cast(current_desc.Height); RECT client_rect = { 0 }; if (!GetClientRect(hwnd_, &client_rect)) { @@ -123,20 +573,15 @@ std::optional FramePoolScreencap::screencap() return std::nullopt; } - // 使用 DWM 实际可视帧边界，排除 GetWindowRect 中包含的不可见 DWM 阴影区域， - // 以正确计算 WGC 捕获画面中的边框偏移 - RECT frame_rect = { 0 }; - HRESULT hr = DwmGetWindowAttribute(hwnd_, DWMWA_EXTENDED_FRAME_BOUNDS, &frame_rect, sizeof(frame_rect)); - if (FAILED(hr)) { - LogWarn << "DwmGetWindowAttribute failed, falling back to GetWindowRect" << VAR(hr); - if (!GetWindowRect(hwnd_, &frame_rect)) { - LogError << "GetWindowRect failed"; - return std::nullopt; - } + const auto capture_bounds_opt = get_capture_bounds(hwnd_, include_secondary_windows_enabled()); + if (!capture_bounds_opt.has_value()) { + LogError << "Failed to get target capture bounds"; + return std::nullopt; } + const RECT capture_bounds = *capture_bounds_opt; - int border_left = client_top_left.x - frame_rect.left; - int border_top = client_top_left.y - frame_rect.top; + int border_left = client_top_left.x - capture_bounds.left; + int border_top = client_top_left.y - capture_bounds.top; int client_width = client_rect.right - client_rect.left; int client_height = client_rect.bottom - client_rect.top; @@ -147,29 +592,75 @@ std::optional FramePoolScreencap::screencap() if (border_top < 0) { border_top = 0; } - if (client_width > raw.cols) { - client_width = raw.cols; + if (client_width > raw_width) { + client_width = raw_width; } - if (border_left + client_width > raw.cols) { - border_left = raw.cols - client_width; + if (border_left + client_width > raw_width) { + border_left = raw_width - client_width; } if (client_height > raw_height) { client_height = raw_height; } - if (border_top + client_height > raw.rows) { - border_top = raw.rows - client_height; + if (border_top + client_height > raw_height) { + border_top = raw_height - client_height; } cv::Rect client_roi(border_left, border_top, client_width, client_height); - switch (texture_desc_.Format) { + if (current_desc.Format == DXGI_FORMAT_R16G16B16A16_FLOAT) { + if (cv::Mat gpu_processed; try_process_hdr_on_gpu(texture, client_roi, gpu_processed)) { + last_screencap_info_ = { + .hdr_capture_active = true, + .hdr_preprocessed = true, + .gpu_processed = true, + .display_hdr_active = true, + .display_hdr_compensated = false, + }; + cached_image_ = std::move(gpu_processed); + return cached_image_; + } + } + + if (!readable_texture_ || !is_texture_desc_compatible(current_desc, texture_desc_)) { + if (readable_texture_) { + LogInfo << "Capture frame texture changed, recreating staging texture" + << VAR(texture_desc_.Width) << VAR(texture_desc_.Height) << VAR(texture_desc_.Format) + << VAR(current_desc.Width) << VAR(current_desc.Height) << VAR(current_desc.Format); + } + readable_texture_ = nullptr; + texture_desc_ = { 0 }; + if (!init_texture(texture)) { + LogError << "falied to init_texture"; + return std::nullopt; + } + } + + d3d_context_->CopyResource(readable_texture_.get(), texture.get()); + + D3D11_MAPPED_SUBRESOURCE mapped { 0 }; + ret = d3d_context_->Map(readable_texture_.get(), 0, D3D11_MAP_READ, 0, &mapped); + if (FAILED(ret)) { + LogError << "Map failed" << VAR(ret); + return std::nullopt; + } + OnScopeLeave([&]() { d3d_context_->Unmap(readable_texture_.get(), 0); }); + + switch (current_desc.Format) { case DXGI_FORMAT_B8G8R8A8_UNORM: { cv::Mat raw(raw_height, raw_width, CV_8UC4, mapped.pData, mapped.RowPitch); + last_screencap_info_ = {}; cached_image_ = bgra_to_bgr(raw(client_roi)); return cached_image_; } case DXGI_FORMAT_R16G16B16A16_FLOAT: { cv::Mat raw(raw_height, raw_width, CV_16FC4, mapped.pData, mapped.RowPitch); + last_screencap_info_ = { + .hdr_capture_active = true, + .hdr_preprocessed = false, + .gpu_processed = false, + .display_hdr_active = true, + .display_hdr_compensated = false, + }; cached_image_ = raw(client_roi).clone(); return cached_image_; } @@ -202,7 +693,7 @@ bool FramePoolScreencap::init() nullptr, D3D_DRIVER_TYPE_HARDWARE, nullptr, - 0, + D3D11_CREATE_DEVICE_BGRA_SUPPORT, nullptr, 0, D3D11_SDK_VERSION, @@ -273,21 +764,48 @@ bool FramePoolScreencap::init() return false; } - try { - cap_frame_pool_ = winrt::Windows::Graphics::Capture::Direct3D11CaptureFramePool::Create( - d3d_device_interop, - winrt::Windows::Graphics::DirectX::DirectXPixelFormat::B8G8R8A8UIntNormalized, - 1, - cap_item_.Size()); + const HMONITOR current_monitor = MonitorFromWindow(hwnd_, MONITOR_DEFAULTTONEAREST); + if (!current_monitor) { + LogWarn << "MonitorFromWindow failed, falling back to HDR-first probing"; } - catch (const winrt::hresult_error& e) { - LogError << "Direct3D11CaptureFramePool::Create failed" << VAR(e.code()) << VAR(winrt::to_string(e.message())); - return false; + + const bool try_secondary_windows = supports_include_secondary_windows(); + std::vector capture_monitors; + if (const auto capture_bounds = get_capture_bounds(hwnd_, try_secondary_windows)) { + capture_monitors = get_capture_monitors(*capture_bounds); + } + if (capture_monitors.empty() && current_monitor) { + capture_monitors.emplace_back(current_monitor); } - if (!cap_frame_pool_) { - LogError << "Direct3D11CaptureFramePool::Create returned null"; - return false; + if (!capture_monitors.empty()) { + LogInfo << "Capture intersects monitors" << VAR(capture_monitors.size()); + } + + const std::optional hdr_enabled = is_windows_hdr_enabled(capture_monitors); + const bool prefer_hdr = hdr_enabled.value_or(true); + + if (!hdr_enabled.has_value()) { + LogWarn << "Advanced-color display detection failed, keeping HDR-first probing"; + } + else { + LogInfo << "Advanced-color display detection result" << VAR(hdr_enabled.value()); + } + + if (prefer_hdr) { + LogInfo << "Attempting float frame pool for advanced-color display"; + if (!try_init_frame_pool(d3d_device_interop, winrt::Windows::Graphics::DirectX::DirectXPixelFormat::R16G16B16A16Float)) { + LogWarn << "Float frame pool unavailable, falling back to SDR"; + if (!try_init_frame_pool(d3d_device_interop, winrt::Windows::Graphics::DirectX::DirectXPixelFormat::B8G8R8A8UIntNormalized)) { + return false; + } + } + } + else { + LogInfo << "Advanced-color display is off, using SDR frame pool"; + if (!try_init_frame_pool(d3d_device_interop, winrt::Windows::Graphics::DirectX::DirectXPixelFormat::B8G8R8A8UIntNormalized)) { + return false; + } } try { @@ -302,13 +820,13 @@ bool FramePoolScreencap::init() return false; } - // 尝试关闭截图时的黄色边框（Windows 11 及部分 Win10 版本支持） + // Try to disable the yellow capture border when the OS supports it. try_disable_border(); // 尝试关闭截图时的鼠标指针（Windows 10 2004 及以上支持） try_disable_cursor(); - // 尝试包含从属窗口（弹窗、工具提示等） + // Try to include owned popups such as dialogs or tooltips in the capture. try_include_secondary_windows(); try { @@ -325,6 +843,9 @@ bool FramePoolScreencap::init() last_capture_size_.first = size.Width; last_capture_size_.second = size.Height; } + last_monitor_ = current_monitor; + last_capture_monitors_ = std::move(capture_monitors); + last_capture_prefers_hdr_ = prefer_hdr; return true; } @@ -352,9 +873,24 @@ void FramePoolScreencap::uninit() } readable_texture_ = nullptr; + reset_hdr_gpu_resources(); + hdr_vertex_shader_ = nullptr; + hdr_copy_pixel_shader_ = nullptr; + hdr_tonemap_pixel_shader_ = nullptr; + hdr_sampler_state_ = nullptr; + hdr_constant_buffer_ = nullptr; + hdr_gpu_pipeline_attempted_ = false; + hdr_gpu_pipeline_available_ = false; cap_item_ = nullptr; + cap_pixel_format_ = winrt::Windows::Graphics::DirectX::DirectXPixelFormat::B8G8R8A8UIntNormalized; texture_desc_ = { 0 }; + last_monitor_ = nullptr; + last_capture_monitors_.clear(); + last_capture_prefers_hdr_ = true; last_capture_size_ = { }; + cached_image_ = cv::Mat(); + include_secondary_windows_enabled_ = false; + last_screencap_info_ = {}; } bool FramePoolScreencap::check_and_handle_size_changed() @@ -368,6 +904,22 @@ bool FramePoolScreencap::check_and_handle_size_changed() return false; } + const HMONITOR current_monitor = MonitorFromWindow(hwnd_, MONITOR_DEFAULTTONEAREST); + const bool primary_monitor_changed = current_monitor && last_monitor_ && current_monitor != last_monitor_; + + std::vector current_capture_monitors; + if (const auto capture_bounds = get_capture_bounds(hwnd_, include_secondary_windows_enabled())) { + current_capture_monitors = get_capture_monitors(*capture_bounds); + } + if (current_capture_monitors.empty() && current_monitor) { + current_capture_monitors.emplace_back(current_monitor); + } + + const bool capture_monitors_changed = current_capture_monitors != last_capture_monitors_; + const std::optional hdr_enabled = is_windows_hdr_enabled(current_capture_monitors); + const bool prefer_hdr = hdr_enabled.value_or(true); + const bool hdr_preference_changed = prefer_hdr != last_capture_prefers_hdr_; + winrt::Windows::Graphics::SizeInt32 current_size { }; try { current_size = cap_item_.Size(); @@ -377,13 +929,27 @@ bool FramePoolScreencap::check_and_handle_size_changed() uninit(); return false; } - // 如果窗口大小没有变化，直接返回 - if (current_size.Width == last_capture_size_.first && current_size.Height == last_capture_size_.second) { + if (!primary_monitor_changed && !capture_monitors_changed && !hdr_preference_changed && current_size.Width == last_capture_size_.first + && current_size.Height == last_capture_size_.second) { return true; } - LogInfo << "Window size changed, recreating frame pool" << VAR(current_size.Width) << VAR(current_size.Height) - << VAR(last_capture_size_.first) << VAR(last_capture_size_.second); + if (primary_monitor_changed) { + LogInfo << "Window monitor changed, recreating frame pool" << VAR_VOIDP(last_monitor_) << VAR_VOIDP(current_monitor) + << VAR(current_size.Width) << VAR(current_size.Height); + } + else if (capture_monitors_changed) { + LogInfo << "Capture monitor set changed, recreating frame pool" << VAR(last_capture_monitors_.size()) + << VAR(current_capture_monitors.size()) << VAR(current_size.Width) << VAR(current_size.Height); + } + else if (hdr_preference_changed) { + LogInfo << "Capture HDR preference changed, recreating frame pool" << VAR(last_capture_prefers_hdr_) << VAR(prefer_hdr) + << VAR(current_size.Width) << VAR(current_size.Height); + } + else { + LogInfo << "Window size changed, recreating frame pool" << VAR(current_size.Width) << VAR(current_size.Height) + << VAR(last_capture_size_.first) << VAR(last_capture_size_.second); + } // 完全重新初始化以适应新的窗口大小 uninit(); @@ -420,6 +986,385 @@ bool FramePoolScreencap::init_texture(winrt::com_ptr raw_textur return true; } +bool FramePoolScreencap::init_hdr_gpu_pipeline() +{ + if (hdr_gpu_pipeline_attempted_) { + return hdr_gpu_pipeline_available_; + } + hdr_gpu_pipeline_attempted_ = true; + + if (!d3d_device_) { + LogError << "d3d_device_ is null"; + return false; + } + + const auto compile_shader = [](std::string_view source, const char* entry, const char* target, winrt::com_ptr& bytecode) { + const auto d3d_compile = get_d3d_compile_proc(); + if (!d3d_compile) { + LogWarn << "D3DCompile is unavailable, skip HDR GPU preprocess shader compilation"; + return false; + } + + winrt::com_ptr errors; + winrt::com_ptr compiled; + + const HRESULT ret = d3d_compile( + source.data(), + source.size(), + nullptr, + nullptr, + nullptr, + entry, + target, + D3DCOMPILE_ENABLE_STRICTNESS, + 0, + compiled.put(), + errors.put()); + if (FAILED(ret)) { + const std::string error_text = errors ? std::string(static_cast(errors->GetBufferPointer()), errors->GetBufferSize()) : ""; + LogWarn << "D3DCompile failed" << VAR(entry) << VAR(target) << VAR(ret) << VAR(error_text); + return false; + } + + bytecode = std::move(compiled); + return true; + }; + + winrt::com_ptr vertex_shader_blob; + if (!compile_shader(kHdrFullscreenVertexShader, "main", "vs_4_0", vertex_shader_blob)) { + return false; + } + + HRESULT ret = d3d_device_->CreateVertexShader( + vertex_shader_blob->GetBufferPointer(), + vertex_shader_blob->GetBufferSize(), + nullptr, + hdr_vertex_shader_.put()); + if (FAILED(ret)) { + LogWarn << "CreateVertexShader failed" << VAR(ret); + return false; + } + + winrt::com_ptr copy_pixel_shader_blob; + if (!compile_shader(kHdrCopyPixelShader, "main", "ps_4_0", copy_pixel_shader_blob)) { + return false; + } + ret = d3d_device_->CreatePixelShader( + copy_pixel_shader_blob->GetBufferPointer(), + copy_pixel_shader_blob->GetBufferSize(), + nullptr, + hdr_copy_pixel_shader_.put()); + if (FAILED(ret)) { + LogWarn << "Create copy pixel shader failed" << VAR(ret); + return false; + } + + winrt::com_ptr tonemap_pixel_shader_blob; + if (!compile_shader(kHdrTonemapPixelShader, "main", "ps_4_0", tonemap_pixel_shader_blob)) { + return false; + } + ret = d3d_device_->CreatePixelShader( + tonemap_pixel_shader_blob->GetBufferPointer(), + tonemap_pixel_shader_blob->GetBufferSize(), + nullptr, + hdr_tonemap_pixel_shader_.put()); + if (FAILED(ret)) { + LogWarn << "Create tonemap pixel shader failed" << VAR(ret); + return false; + } + + D3D11_SAMPLER_DESC sampler_desc {}; + sampler_desc.Filter = D3D11_FILTER_MIN_MAG_MIP_LINEAR; + sampler_desc.AddressU = D3D11_TEXTURE_ADDRESS_CLAMP; + sampler_desc.AddressV = D3D11_TEXTURE_ADDRESS_CLAMP; + sampler_desc.AddressW = D3D11_TEXTURE_ADDRESS_CLAMP; + sampler_desc.ComparisonFunc = D3D11_COMPARISON_NEVER; + sampler_desc.MinLOD = 0.0f; + sampler_desc.MaxLOD = D3D11_FLOAT32_MAX; + + ret = d3d_device_->CreateSamplerState(&sampler_desc, hdr_sampler_state_.put()); + if (FAILED(ret)) { + LogWarn << "CreateSamplerState failed" << VAR(ret); + return false; + } + + D3D11_BUFFER_DESC constant_buffer_desc {}; + constant_buffer_desc.ByteWidth = sizeof(HdrGpuShaderConstants); + constant_buffer_desc.Usage = D3D11_USAGE_DEFAULT; + constant_buffer_desc.BindFlags = D3D11_BIND_CONSTANT_BUFFER; + + ret = d3d_device_->CreateBuffer(&constant_buffer_desc, nullptr, hdr_constant_buffer_.put()); + if (FAILED(ret)) { + LogWarn << "CreateBuffer failed" << VAR(ret); + return false; + } + + hdr_gpu_pipeline_available_ = true; + LogInfo << "HDR GPU preprocess pipeline initialized"; + return true; +} + +void FramePoolScreencap::reset_hdr_gpu_resources() +{ + hdr_gpu_source_desc_ = {}; + hdr_gpu_output_size_ = {}; + hdr_gpu_stats_size_ = {}; + hdr_shader_input_texture_ = nullptr; + hdr_shader_input_srv_ = nullptr; + hdr_stats_texture_ = nullptr; + hdr_stats_rtv_ = nullptr; + hdr_stats_staging_ = nullptr; + hdr_output_texture_ = nullptr; + hdr_output_rtv_ = nullptr; + hdr_output_staging_ = nullptr; +} + +bool FramePoolScreencap::ensure_hdr_gpu_resources(const D3D11_TEXTURE2D_DESC& source_desc, const cv::Size& output_size) +{ + if (!hdr_gpu_pipeline_available_ && !init_hdr_gpu_pipeline()) { + return false; + } + + if (output_size.width <= 0 || output_size.height <= 0) { + LogWarn << "Invalid HDR GPU output size" << VAR(output_size.width) << VAR(output_size.height); + return false; + } + + const cv::Size stats_size = calc_hdr_stats_size(output_size.width, output_size.height); + if (stats_size.width <= 0 || stats_size.height <= 0) { + LogWarn << "Invalid HDR stats texture size" << VAR(stats_size.width) << VAR(stats_size.height); + return false; + } + + if (hdr_shader_input_texture_ && is_texture_desc_compatible(hdr_gpu_source_desc_, source_desc) && hdr_gpu_output_size_ == output_size + && hdr_gpu_stats_size_ == stats_size) { + return true; + } + + reset_hdr_gpu_resources(); + + D3D11_TEXTURE2D_DESC shader_input_desc = source_desc; + shader_input_desc.BindFlags = D3D11_BIND_SHADER_RESOURCE; + shader_input_desc.MiscFlags = 0; + shader_input_desc.CPUAccessFlags = 0; + shader_input_desc.Usage = D3D11_USAGE_DEFAULT; + + HRESULT ret = d3d_device_->CreateTexture2D(&shader_input_desc, nullptr, hdr_shader_input_texture_.put()); + if (FAILED(ret)) { + LogWarn << "Create shader input texture failed" << VAR(ret); + return false; + } + + D3D11_SHADER_RESOURCE_VIEW_DESC srv_desc {}; + srv_desc.Format = shader_input_desc.Format; + srv_desc.ViewDimension = D3D11_SRV_DIMENSION_TEXTURE2D; + srv_desc.Texture2D.MipLevels = 1; + + ret = d3d_device_->CreateShaderResourceView(hdr_shader_input_texture_.get(), &srv_desc, hdr_shader_input_srv_.put()); + if (FAILED(ret)) { + LogWarn << "Create shader resource view failed" << VAR(ret); + return false; + } + + D3D11_TEXTURE2D_DESC stats_texture_desc {}; + stats_texture_desc.Width = static_cast(stats_size.width); + stats_texture_desc.Height = static_cast(stats_size.height); + stats_texture_desc.MipLevels = 1; + stats_texture_desc.ArraySize = 1; + stats_texture_desc.Format = DXGI_FORMAT_R16G16B16A16_FLOAT; + stats_texture_desc.SampleDesc.Count = 1; + stats_texture_desc.Usage = D3D11_USAGE_DEFAULT; + stats_texture_desc.BindFlags = D3D11_BIND_RENDER_TARGET; + + ret = d3d_device_->CreateTexture2D(&stats_texture_desc, nullptr, hdr_stats_texture_.put()); + if (FAILED(ret)) { + LogWarn << "Create HDR stats texture failed" << VAR(ret); + return false; + } + + ret = d3d_device_->CreateRenderTargetView(hdr_stats_texture_.get(), nullptr, hdr_stats_rtv_.put()); + if (FAILED(ret)) { + LogWarn << "Create HDR stats RTV failed" << VAR(ret); + return false; + } + + D3D11_TEXTURE2D_DESC stats_staging_desc = stats_texture_desc; + stats_staging_desc.BindFlags = 0; + stats_staging_desc.CPUAccessFlags = D3D11_CPU_ACCESS_READ; + stats_staging_desc.Usage = D3D11_USAGE_STAGING; + + ret = d3d_device_->CreateTexture2D(&stats_staging_desc, nullptr, hdr_stats_staging_.put()); + if (FAILED(ret)) { + LogWarn << "Create HDR stats staging texture failed" << VAR(ret); + return false; + } + + D3D11_TEXTURE2D_DESC output_texture_desc {}; + output_texture_desc.Width = static_cast(output_size.width); + output_texture_desc.Height = static_cast(output_size.height); + output_texture_desc.MipLevels = 1; + output_texture_desc.ArraySize = 1; + output_texture_desc.Format = DXGI_FORMAT_B8G8R8A8_UNORM; + output_texture_desc.SampleDesc.Count = 1; + output_texture_desc.Usage = D3D11_USAGE_DEFAULT; + output_texture_desc.BindFlags = D3D11_BIND_RENDER_TARGET; + + ret = d3d_device_->CreateTexture2D(&output_texture_desc, nullptr, hdr_output_texture_.put()); + if (FAILED(ret)) { + LogWarn << "Create HDR output texture failed" << VAR(ret); + return false; + } + + ret = d3d_device_->CreateRenderTargetView(hdr_output_texture_.get(), nullptr, hdr_output_rtv_.put()); + if (FAILED(ret)) { + LogWarn << "Create HDR output RTV failed" << VAR(ret); + return false; + } + + D3D11_TEXTURE2D_DESC output_staging_desc = output_texture_desc; + output_staging_desc.BindFlags = 0; + output_staging_desc.CPUAccessFlags = D3D11_CPU_ACCESS_READ; + output_staging_desc.Usage = D3D11_USAGE_STAGING; + + ret = d3d_device_->CreateTexture2D(&output_staging_desc, nullptr, hdr_output_staging_.put()); + if (FAILED(ret)) { + LogWarn << "Create HDR output staging texture failed" << VAR(ret); + return false; + } + + hdr_gpu_source_desc_ = source_desc; + hdr_gpu_output_size_ = output_size; + hdr_gpu_stats_size_ = stats_size; + return true; +} + +bool FramePoolScreencap::try_process_hdr_on_gpu(const winrt::com_ptr& texture, const cv::Rect& client_roi, cv::Mat& output) +{ + if (!texture || !d3d_context_) { + return false; + } + + D3D11_TEXTURE2D_DESC source_desc {}; + texture->GetDesc(&source_desc); + + if (!ensure_hdr_gpu_resources(source_desc, client_roi.size())) { + return false; + } + + d3d_context_->CopyResource(hdr_shader_input_texture_.get(), texture.get()); + + const HdrGpuShaderConstants constants { + .source_offset = { static_cast(client_roi.x) / source_desc.Width, static_cast(client_roi.y) / source_desc.Height }, + .source_scale = { static_cast(client_roi.width) / source_desc.Width, static_cast(client_roi.height) / source_desc.Height }, + .exposure = 1.0f, + .apply_tone_mapping = 0.0f, + }; + d3d_context_->UpdateSubresource(hdr_constant_buffer_.get(), 0, nullptr, &constants, 0, 0); + + ID3D11Buffer* constant_buffers[] = { hdr_constant_buffer_.get() }; + ID3D11ShaderResourceView* shader_resources[] = { hdr_shader_input_srv_.get() }; + ID3D11SamplerState* samplers[] = { hdr_sampler_state_.get() }; + + d3d_context_->IASetPrimitiveTopology(D3D11_PRIMITIVE_TOPOLOGY_TRIANGLELIST); + d3d_context_->IASetInputLayout(nullptr); + d3d_context_->VSSetShader(hdr_vertex_shader_.get(), nullptr, 0); + d3d_context_->VSSetConstantBuffers(0, 1, constant_buffers); + d3d_context_->PSSetShaderResources(0, 1, shader_resources); + d3d_context_->PSSetSamplers(0, 1, samplers); + d3d_context_->PSSetConstantBuffers(0, 1, constant_buffers); + + D3D11_VIEWPORT stats_viewport {}; + stats_viewport.Width = static_cast(hdr_gpu_stats_size_.width); + stats_viewport.Height = static_cast(hdr_gpu_stats_size_.height); + stats_viewport.MinDepth = 0.0f; + stats_viewport.MaxDepth = 1.0f; + + ID3D11RenderTargetView* stats_rtv[] = { hdr_stats_rtv_.get() }; + d3d_context_->RSSetViewports(1, &stats_viewport); + d3d_context_->OMSetRenderTargets(1, stats_rtv, nullptr); + d3d_context_->PSSetShader(hdr_copy_pixel_shader_.get(), nullptr, 0); + d3d_context_->Draw(3, 0); + d3d_context_->CopyResource(hdr_stats_staging_.get(), hdr_stats_texture_.get()); + + D3D11_MAPPED_SUBRESOURCE mapped_stats {}; + HRESULT ret = d3d_context_->Map(hdr_stats_staging_.get(), 0, D3D11_MAP_READ, 0, &mapped_stats); + if (FAILED(ret)) { + LogWarn << "Map HDR stats staging texture failed" << VAR(ret); + return false; + } + OnScopeLeave([&]() { d3d_context_->Unmap(hdr_stats_staging_.get(), 0); }); + + cv::Mat hdr_stats_mat(hdr_gpu_stats_size_.height, hdr_gpu_stats_size_.width, CV_16FC4, mapped_stats.pData, mapped_stats.RowPitch); + const HdrSceneStats hdr_scene_stats = analyze_hdr_scene(hdr_stats_mat); + bool apply_tone_mapping = false; + const float exposure = calc_hdr_exposure(hdr_scene_stats, apply_tone_mapping); + + HdrGpuShaderConstants tonemap_constants { + .source_offset = { static_cast(client_roi.x) / source_desc.Width, static_cast(client_roi.y) / source_desc.Height }, + .source_scale = { static_cast(client_roi.width) / source_desc.Width, static_cast(client_roi.height) / source_desc.Height }, + .exposure = exposure, + .apply_tone_mapping = apply_tone_mapping ? 1.0f : 0.0f, + }; + d3d_context_->UpdateSubresource(hdr_constant_buffer_.get(), 0, nullptr, &tonemap_constants, 0, 0); + + D3D11_VIEWPORT output_viewport {}; + output_viewport.Width = static_cast(hdr_gpu_output_size_.width); + output_viewport.Height = static_cast(hdr_gpu_output_size_.height); + output_viewport.MinDepth = 0.0f; + output_viewport.MaxDepth = 1.0f; + + ID3D11RenderTargetView* output_rtv[] = { hdr_output_rtv_.get() }; + d3d_context_->RSSetViewports(1, &output_viewport); + d3d_context_->OMSetRenderTargets(1, output_rtv, nullptr); + d3d_context_->PSSetShader(hdr_tonemap_pixel_shader_.get(), nullptr, 0); + d3d_context_->Draw(3, 0); + + ID3D11ShaderResourceView* null_shader_resources[] = { nullptr }; + d3d_context_->PSSetShaderResources(0, 1, null_shader_resources); + + d3d_context_->CopyResource(hdr_output_staging_.get(), hdr_output_texture_.get()); + + D3D11_MAPPED_SUBRESOURCE mapped_output {}; + ret = d3d_context_->Map(hdr_output_staging_.get(), 0, D3D11_MAP_READ, 0, &mapped_output); + if (FAILED(ret)) { + LogWarn << "Map HDR output staging texture failed" << VAR(ret); + return false; + } + OnScopeLeave([&]() { d3d_context_->Unmap(hdr_output_staging_.get(), 0); }); + + cv::Mat gpu_bgra(hdr_gpu_output_size_.height, hdr_gpu_output_size_.width, CV_8UC4, mapped_output.pData, mapped_output.RowPitch); + output = bgra_to_bgr(gpu_bgra); + return !output.empty(); +} + +bool FramePoolScreencap::try_init_frame_pool( + const winrt::Windows::Graphics::DirectX::Direct3D11::IDirect3DDevice& d3d_device_interop, + winrt::Windows::Graphics::DirectX::DirectXPixelFormat pixel_format) +{ + try { + cap_frame_pool_ = winrt::Windows::Graphics::Capture::Direct3D11CaptureFramePool::Create( + d3d_device_interop, + pixel_format, + 1, + cap_item_.Size()); + } + catch (const winrt::hresult_error& e) { + LogWarn << "Direct3D11CaptureFramePool::Create failed" << VAR(pixel_format) << VAR(e.code()) + << VAR(winrt::to_string(e.message())); + cap_frame_pool_ = nullptr; + return false; + } + + if (!cap_frame_pool_) { + LogWarn << "Direct3D11CaptureFramePool::Create returned null" << VAR(pixel_format); + return false; + } + + cap_pixel_format_ = pixel_format; + LogInfo << "Frame pool initialized" << VAR(cap_pixel_format_); + return true; +} + void FramePoolScreencap::try_disable_border() { LogFunc; @@ -427,7 +1372,8 @@ void FramePoolScreencap::try_disable_border() using namespace winrt::Windows::Foundation::Metadata; using namespace winrt::Windows::Graphics::Capture; - // GraphicsCaptureAccess 和 IsBorderRequired 在 UniversalApiContract v10.0 (Windows 10 2004) 引入 + // GraphicsCaptureAccess and IsBorderRequired were introduced in + // UniversalApiContract v10.0 (Windows 10 2004). if (!ApiInformation::IsApiContractPresent(L"Windows.Foundation.UniversalApiContract", 10)) { LogInfo << "UniversalApiContract v10 not present, border toggle not supported"; return; @@ -498,6 +1444,7 @@ void FramePoolScreencap::try_include_secondary_windows() try { cap_session_.IncludeSecondaryWindows(true); + include_secondary_windows_enabled_ = true; LogInfo << "Secondary windows capture enabled successfully"; } catch (const winrt::hresult_error& e) { diff --git a/source/MaaWin32ControlUnit/Screencap/FramePoolScreencap.h b/source/MaaWin32ControlUnit/Screencap/FramePoolScreencap.h index 9c9ec979f3..73f3e83d82 100644 --- a/source/MaaWin32ControlUnit/Screencap/FramePoolScreencap.h +++ b/source/MaaWin32ControlUnit/Screencap/FramePoolScreencap.h @@ -18,6 +18,7 @@ #include "SafeDXGI.hpp" +#include #include MAA_CTRL_UNIT_NS_BEGIN @@ -34,15 +35,24 @@ class FramePoolScreencap : public ScreencapBase public: // from ScreencapBase virtual std::optional screencap() override; + virtual ScreencapInfo last_screencap_info() const override { return last_screencap_info_; } private: bool init(); + bool try_init_frame_pool( + const winrt::Windows::Graphics::DirectX::Direct3D11::IDirect3DDevice& d3d_device_interop, + winrt::Windows::Graphics::DirectX::DirectXPixelFormat pixel_format); bool init_texture(winrt::com_ptr raw_texture); + bool init_hdr_gpu_pipeline(); + bool ensure_hdr_gpu_resources(const D3D11_TEXTURE2D_DESC& source_desc, const cv::Size& output_size); + bool try_process_hdr_on_gpu(const winrt::com_ptr& texture, const cv::Rect& client_roi, cv::Mat& output); + void reset_hdr_gpu_resources(); void uninit(); bool check_and_handle_size_changed(); void try_disable_border(); void try_disable_cursor(); void try_include_secondary_windows(); + bool include_secondary_windows_enabled() const { return include_secondary_windows_enabled_; } private: HWND hwnd_ = nullptr; @@ -56,10 +66,36 @@ class FramePoolScreencap : public ScreencapBase winrt::Windows::Graphics::Capture::GraphicsCaptureItem cap_item_ = nullptr; winrt::Windows::Graphics::Capture::Direct3D11CaptureFramePool cap_frame_pool_ = nullptr; winrt::Windows::Graphics::Capture::GraphicsCaptureSession cap_session_ = nullptr; + winrt::Windows::Graphics::DirectX::DirectXPixelFormat cap_pixel_format_ = + winrt::Windows::Graphics::DirectX::DirectXPixelFormat::B8G8R8A8UIntNormalized; + HMONITOR last_monitor_ = nullptr; + std::vector last_capture_monitors_; + bool last_capture_prefers_hdr_ = true; // 存储上次的窗口大小，用于检测窗口大小变化 std::pair last_capture_size_ = { 0, 0 }; cv::Mat cached_image_; + bool include_secondary_windows_enabled_ = false; + ScreencapInfo last_screencap_info_ {}; + + bool hdr_gpu_pipeline_attempted_ = false; + bool hdr_gpu_pipeline_available_ = false; + D3D11_TEXTURE2D_DESC hdr_gpu_source_desc_ = { }; + cv::Size hdr_gpu_output_size_ {}; + cv::Size hdr_gpu_stats_size_ {}; + winrt::com_ptr hdr_vertex_shader_ = nullptr; + winrt::com_ptr hdr_copy_pixel_shader_ = nullptr; + winrt::com_ptr hdr_tonemap_pixel_shader_ = nullptr; + winrt::com_ptr hdr_sampler_state_ = nullptr; + winrt::com_ptr hdr_constant_buffer_ = nullptr; + winrt::com_ptr hdr_shader_input_texture_ = nullptr; + winrt::com_ptr hdr_shader_input_srv_ = nullptr; + winrt::com_ptr hdr_stats_texture_ = nullptr; + winrt::com_ptr hdr_stats_rtv_ = nullptr; + winrt::com_ptr hdr_stats_staging_ = nullptr; + winrt::com_ptr hdr_output_texture_ = nullptr; + winrt::com_ptr hdr_output_rtv_ = nullptr; + winrt::com_ptr hdr_output_staging_ = nullptr; }; MAA_CTRL_UNIT_NS_END diff --git a/source/MaaWin32ControlUnit/Screencap/FramePoolWithPseudoMinimizeScreencap.h b/source/MaaWin32ControlUnit/Screencap/FramePoolWithPseudoMinimizeScreencap.h index 1a93ca0dd5..6338a523a0 100644 --- a/source/MaaWin32ControlUnit/Screencap/FramePoolWithPseudoMinimizeScreencap.h +++ b/source/MaaWin32ControlUnit/Screencap/FramePoolWithPseudoMinimizeScreencap.h @@ -26,6 +26,8 @@ class FramePoolWithPseudoMinimizeScreencap : public ScreencapBase return inner_.screencap(); } + virtual ScreencapInfo last_screencap_info() const override { return inner_.last_screencap_info(); } + virtual void inactive() override { if (helper_.is_pseudo_minimized()) { diff --git a/source/MaaWin32ControlUnit/Screencap/GdiScreencap.cpp b/source/MaaWin32ControlUnit/Screencap/GdiScreencap.cpp index efafefbe0f..7292c61838 100644 --- a/source/MaaWin32ControlUnit/Screencap/GdiScreencap.cpp +++ b/source/MaaWin32ControlUnit/Screencap/GdiScreencap.cpp @@ -1,5 +1,8 @@ #include "GdiScreencap.h" +#include + +#include "HdrDisplayUtils.hpp" #include "HwndUtils.hpp" #include "MaaUtils/Logger.h" @@ -9,6 +12,7 @@ std::optional GdiScreencap::screencap() { if (!hwnd_) { LogError << "hwnd_ is nullptr"; + last_screencap_info_ = {}; return std::nullopt; } @@ -35,44 +39,74 @@ std::optional GdiScreencap::screencap() hdc = GetDC(hwnd_); if (!hdc) { LogError << "GetDC failed, error code: " << GetLastError(); + last_screencap_info_ = {}; return std::nullopt; } mem_dc = CreateCompatibleDC(hdc); if (!mem_dc) { LogError << "CreateCompatibleDC failed, error code: " << GetLastError(); + last_screencap_info_ = {}; return std::nullopt; } auto [width, height] = window_size(hwnd_); if (width <= 0 || height <= 0) { LogError << "Invalid window size" << VAR(width) << VAR(height); + last_screencap_info_ = {}; return std::nullopt; } bitmap = CreateCompatibleBitmap(hdc, width, height); if (!bitmap) { LogError << "CreateCompatibleBitmap failed, error code: " << GetLastError(); + last_screencap_info_ = {}; return std::nullopt; } old_obj = SelectObject(mem_dc, bitmap); if (!old_obj) { LogError << "SelectObject failed, error code: " << GetLastError(); + last_screencap_info_ = {}; return std::nullopt; } if (!BitBlt(mem_dc, 0, 0, width, height, hdc, 0, 0, SRCCOPY)) { LogError << "BitBlt failed, error code: " << GetLastError(); + last_screencap_info_ = {}; return std::nullopt; } cv::Mat mat(height, width, CV_8UC4); if (!GetBitmapBits(bitmap, width * height * 4, mat.data)) { LogError << "GetBitmapBits failed, error code: " << GetLastError(); + last_screencap_info_ = {}; return std::nullopt; } + const HMONITOR target_monitor = MonitorFromWindow(hwnd_, MONITOR_DEFAULTTONEAREST); + const std::optional hdr_state = query_hdr_display_state(target_monitor); + if (hdr_state.has_value() && hdr_state->valid && hdr_state->hdr_enabled) { + cv::Mat compensated = compensate_hdr_sdr_capture(mat, hdr_state->sdr_white_nits); + if (!compensated.empty()) { + last_screencap_info_ = { + .hdr_capture_active = false, + .hdr_preprocessed = true, + .gpu_processed = false, + .display_hdr_active = true, + .display_hdr_compensated = true, + }; + static auto last_log_time = std::chrono::steady_clock::time_point {}; + const auto now = std::chrono::steady_clock::now(); + if (last_log_time == std::chrono::steady_clock::time_point {} || now - last_log_time > std::chrono::seconds(5)) { + last_log_time = now; + LogInfo << "GDI HDR display compensation applied" << VAR(hdr_state->sdr_white_nits); + } + return compensated; + } + } + + last_screencap_info_ = {}; return bgra_to_bgr(mat); } diff --git a/source/MaaWin32ControlUnit/Screencap/GdiScreencap.h b/source/MaaWin32ControlUnit/Screencap/GdiScreencap.h index 9dc9bbcd75..5e12acf254 100644 --- a/source/MaaWin32ControlUnit/Screencap/GdiScreencap.h +++ b/source/MaaWin32ControlUnit/Screencap/GdiScreencap.h @@ -19,9 +19,11 @@ class GdiScreencap : public ScreencapBase public: // from ScreencapBase virtual std::optional screencap() override; + virtual ScreencapInfo last_screencap_info() const override { return last_screencap_info_; } private: HWND hwnd_ = nullptr; + ScreencapInfo last_screencap_info_ {}; }; MAA_CTRL_UNIT_NS_END diff --git a/source/MaaWin32ControlUnit/Screencap/HdrDisplayUtils.cpp b/source/MaaWin32ControlUnit/Screencap/HdrDisplayUtils.cpp new file mode 100644 index 0000000000..0e98c8de22 --- /dev/null +++ b/source/MaaWin32ControlUnit/Screencap/HdrDisplayUtils.cpp @@ -0,0 +1,371 @@ +#include "HdrDisplayUtils.hpp" + +#include +#include +#include +#include +#include + +#include "MaaUtils/Logger.h" + +namespace +{ +constexpr LONG kDisplayConfigErrorInsufficientBuffer = 122; +constexpr int kMaxDisplayConfigRetries = 3; +constexpr float kDefaultSdrWhiteNits = 80.0f; +constexpr float kSrgbToLinearThreshold = 0.04045f; +constexpr float kLinearToSrgbThreshold = 0.0031308f; +constexpr float kInvSrgbGamma = 1.0f / 2.4f; +constexpr DISPLAYCONFIG_DEVICE_INFO_TYPE kDisplayConfigDeviceInfoGetAdvancedColorInfo2 = + static_cast(15); +constexpr UINT32 kAdvancedColorActiveMask = 0x2u; +constexpr UINT32 kHdrUserEnabledMask = 0x20u; +constexpr int kAdvancedColorModeHdr = 2; + +float srgb_to_linear(float srgb) +{ + srgb = std::clamp(srgb, 0.0f, 1.0f); + if (srgb <= kSrgbToLinearThreshold) { + return srgb / 12.92f; + } + return std::pow((srgb + 0.055f) / 1.055f, 2.4f); +} + +uint8_t linear_to_srgb_u8(float linear) +{ + linear = std::clamp(linear, 0.0f, 1.0f); + const float srgb = linear <= kLinearToSrgbThreshold ? linear * 12.92f : 1.055f * std::pow(linear, kInvSrgbGamma) - 0.055f; + return static_cast(std::lround(std::clamp(srgb, 0.0f, 1.0f) * 255.0f)); +} + +using DeviceNameBuffer = std::array; + +std::optional get_monitor_device_name(HMONITOR target_monitor) +{ + if (!target_monitor) { + return std::nullopt; + } + + MONITORINFOEXW monitor_info {}; + monitor_info.cbSize = sizeof(monitor_info); + if (!GetMonitorInfoW(target_monitor, &monitor_info)) { + LogWarn << "GetMonitorInfoW failed" << GetLastError(); + return std::nullopt; + } + + DeviceNameBuffer device_name {}; + std::copy_n(monitor_info.szDevice, CCHDEVICENAME, device_name.begin()); + return device_name; +} + +bool device_name_equals(const wchar_t (&lhs)[CCHDEVICENAME], const DeviceNameBuffer& rhs) +{ + for (size_t i = 0; i < CCHDEVICENAME; ++i) { + if (lhs[i] != rhs[i]) { + return false; + } + if (lhs[i] == L'\0') { + return true; + } + } + return true; +} + +bool device_name_equals(const DeviceNameBuffer& lhs, const DeviceNameBuffer& rhs) +{ + for (size_t i = 0; i < CCHDEVICENAME; ++i) { + if (lhs[i] != rhs[i]) { + return false; + } + if (lhs[i] == L'\0') { + return true; + } + } + return true; +} + +struct DisplayConfigGetAdvancedColorInfo2 +{ + DISPLAYCONFIG_DEVICE_INFO_HEADER header {}; + UINT32 value = 0; + DISPLAYCONFIG_COLOR_ENCODING colorEncoding = DISPLAYCONFIG_COLOR_ENCODING_RGB; + UINT32 bitsPerColorChannel = 0; + INT activeColorMode = 0; +}; + +struct PathHdrState +{ + bool valid = false; + bool hdr_user_enabled = false; + bool hdr_enabled = false; + int active_color_mode = 0; +}; + +struct ActiveDisplayPathInfo +{ + DISPLAYCONFIG_PATH_INFO path {}; + std::optional source_device_name; + PathHdrState hdr_state; +}; + +std::optional query_source_device_name(const DISPLAYCONFIG_PATH_INFO& path) +{ + DISPLAYCONFIG_SOURCE_DEVICE_NAME source_name {}; + source_name.header.type = DISPLAYCONFIG_DEVICE_INFO_GET_SOURCE_NAME; + source_name.header.size = sizeof(DISPLAYCONFIG_SOURCE_DEVICE_NAME); + source_name.header.adapterId = path.sourceInfo.adapterId; + source_name.header.id = path.sourceInfo.id; + + const LONG ret = DisplayConfigGetDeviceInfo(&source_name.header); + if (ret != ERROR_SUCCESS) { + return std::nullopt; + } + + DeviceNameBuffer device_name {}; + std::copy_n(source_name.viewGdiDeviceName, CCHDEVICENAME, device_name.begin()); + return device_name; +} + +PathHdrState query_path_hdr_state(const DISPLAYCONFIG_PATH_INFO& path) +{ + DisplayConfigGetAdvancedColorInfo2 color_info2 {}; + color_info2.header.type = kDisplayConfigDeviceInfoGetAdvancedColorInfo2; + color_info2.header.size = sizeof(DisplayConfigGetAdvancedColorInfo2); + color_info2.header.adapterId = path.targetInfo.adapterId; + color_info2.header.id = path.targetInfo.id; + + LONG ret = DisplayConfigGetDeviceInfo(&color_info2.header); + if (ret == ERROR_SUCCESS) { + const bool hdr_user_enabled = (color_info2.value & kHdrUserEnabledMask) != 0; + const bool hdr_active = (color_info2.value & kAdvancedColorActiveMask) != 0 + && color_info2.activeColorMode == kAdvancedColorModeHdr; + return { + .valid = true, + .hdr_user_enabled = hdr_user_enabled, + .hdr_enabled = hdr_active, + .active_color_mode = color_info2.activeColorMode, + }; + } + + DISPLAYCONFIG_GET_ADVANCED_COLOR_INFO color_info {}; + color_info.header.type = DISPLAYCONFIG_DEVICE_INFO_GET_ADVANCED_COLOR_INFO; + color_info.header.size = sizeof(DISPLAYCONFIG_GET_ADVANCED_COLOR_INFO); + color_info.header.adapterId = path.targetInfo.adapterId; + color_info.header.id = path.targetInfo.id; + + ret = DisplayConfigGetDeviceInfo(&color_info.header); + if (ret != ERROR_SUCCESS) { + return {}; + } + + const bool hdr_enabled = (color_info.value & kAdvancedColorActiveMask) != 0; + return { + .valid = true, + .hdr_user_enabled = hdr_enabled, + .hdr_enabled = hdr_enabled, + .active_color_mode = hdr_enabled ? kAdvancedColorModeHdr : 0, + }; +} + +std::optional> query_active_display_paths() +{ + for (int attempt = 0; attempt < kMaxDisplayConfigRetries; ++attempt) { + UINT32 path_count = 0; + UINT32 mode_count = 0; + + LONG ret = GetDisplayConfigBufferSizes(QDC_ONLY_ACTIVE_PATHS, &path_count, &mode_count); + if (ret != ERROR_SUCCESS) { + LogWarn << "GetDisplayConfigBufferSizes failed" << VAR(ret); + return std::nullopt; + } + + if (path_count == 0) { + return std::vector {}; + } + + std::vector path_array(path_count); + std::vector mode_array(mode_count); + + ret = QueryDisplayConfig( + QDC_ONLY_ACTIVE_PATHS, + &path_count, + path_array.data(), + &mode_count, + mode_array.empty() ? nullptr : mode_array.data(), + nullptr); + if (ret == kDisplayConfigErrorInsufficientBuffer) { + continue; + } + if (ret != ERROR_SUCCESS) { + LogWarn << "QueryDisplayConfig failed" << VAR(ret); + return std::nullopt; + } + + std::vector result; + result.reserve(path_count); + int success_count = 0; + + for (UINT32 i = 0; i < path_count; ++i) { + ActiveDisplayPathInfo info {}; + info.path = path_array[i]; + info.source_device_name = query_source_device_name(path_array[i]); + info.hdr_state = query_path_hdr_state(path_array[i]); + if (info.hdr_state.valid) { + ++success_count; + } + result.emplace_back(std::move(info)); + } + + if (success_count == 0) { + LogWarn << "DisplayConfigGetDeviceInfo failed for all active display paths"; + return std::nullopt; + } + + return result; + } + + LogWarn << "QueryDisplayConfig kept returning ERROR_INSUFFICIENT_BUFFER for HDR query"; + return std::nullopt; +} + +void try_query_sdr_white_level(MAA_CTRL_UNIT_NS::HdrDisplayState& state, const DISPLAYCONFIG_PATH_INFO& path) +{ + DISPLAYCONFIG_SDR_WHITE_LEVEL white_level {}; + white_level.header.type = DISPLAYCONFIG_DEVICE_INFO_GET_SDR_WHITE_LEVEL; + white_level.header.size = sizeof(DISPLAYCONFIG_SDR_WHITE_LEVEL); + white_level.header.adapterId = path.targetInfo.adapterId; + white_level.header.id = path.targetInfo.id; + + const LONG ret = DisplayConfigGetDeviceInfo(&white_level.header); + if (ret == ERROR_SUCCESS) { + state.sdr_white_nits = std::max( + kDefaultSdrWhiteNits, + static_cast((static_cast(white_level.SDRWhiteLevel) / 1000.0) * kDefaultSdrWhiteNits)); + } + else { + LogWarn << "DisplayConfigGetDeviceInfo(GET_SDR_WHITE_LEVEL) failed" << VAR(ret); + } +} + +} // namespace + +MAA_CTRL_UNIT_NS_BEGIN + +std::optional query_windows_hdr_enabled_go_compatible() +{ + const auto active_paths = query_active_display_paths(); + if (!active_paths.has_value()) { + return std::nullopt; + } + + for (const auto& info : *active_paths) { + if (info.hdr_state.valid && info.hdr_state.hdr_enabled) { + return true; + } + } + + return false; +} + +std::optional query_hdr_display_state(HMONITOR target_monitor) +{ + const auto active_paths = query_active_display_paths(); + if (!active_paths.has_value()) { + return std::nullopt; + } + + HdrDisplayState state { + .hdr_enabled = false, + .hdr_user_enabled = false, + .active_color_mode = 0, + .sdr_white_nits = kDefaultSdrWhiteNits, + .valid = false, + }; + + if (!target_monitor) { + bool has_valid_path = false; + for (const auto& info : *active_paths) { + if (!info.hdr_state.valid) { + continue; + } + has_valid_path = true; + state.hdr_enabled = state.hdr_enabled || info.hdr_state.hdr_enabled; + state.hdr_user_enabled = state.hdr_user_enabled || info.hdr_state.hdr_user_enabled; + if (info.hdr_state.hdr_enabled) { + state.active_color_mode = info.hdr_state.active_color_mode; + } + } + state.valid = has_valid_path; + return state; + } + + const auto target_device_name = get_monitor_device_name(target_monitor); + if (!target_device_name.has_value()) { + LogWarn << "Failed to query target monitor device name for HDR state" << VAR_VOIDP(target_monitor); + return state; + } + + bool matched_path = false; + bool matched_valid_path = false; + bool white_level_queried = false; + for (const auto& info : *active_paths) { + if (!info.source_device_name.has_value() || !device_name_equals(*info.source_device_name, *target_device_name)) { + continue; + } + matched_path = true; + + if (!info.hdr_state.valid) { + continue; + } + matched_valid_path = true; + + state.valid = true; + state.hdr_enabled = state.hdr_enabled || info.hdr_state.hdr_enabled; + state.hdr_user_enabled = state.hdr_user_enabled || info.hdr_state.hdr_user_enabled; + if (info.hdr_state.hdr_enabled || state.active_color_mode == 0) { + state.active_color_mode = info.hdr_state.active_color_mode; + } + if (state.hdr_enabled && !white_level_queried) { + try_query_sdr_white_level(state, info.path); + white_level_queried = true; + } + } + + if (!matched_path) { + LogInfo << "No active display path matched target monitor for HDR state query" << VAR_VOIDP(target_monitor); + } + else if (!matched_valid_path) { + LogWarn << "Matched display path lacked valid HDR state; caller may fall back to global query" << VAR_VOIDP(target_monitor); + } + + return state; +} + +cv::Mat compensate_hdr_sdr_capture(const cv::Mat& bgra, float sdr_white_nits) +{ + if (bgra.empty() || bgra.type() != CV_8UC4) { + return {}; + } + + const float hdr_white_scale = std::max(1.0f, sdr_white_nits / kDefaultSdrWhiteNits); + cv::Mat bgr8(bgra.rows, bgra.cols, CV_8UC3); + + cv::parallel_for_(cv::Range(0, bgra.rows), [&](const cv::Range& range) { + for (int y = range.start; y < range.end; ++y) { + const auto* src_row = bgra.ptr(y); + auto* dst_row = bgr8.ptr(y); + + for (int x = 0; x < bgra.cols; ++x) { + const float b = srgb_to_linear(src_row[x][0] / 255.0f) / hdr_white_scale; + const float g = srgb_to_linear(src_row[x][1] / 255.0f) / hdr_white_scale; + const float r = srgb_to_linear(src_row[x][2] / 255.0f) / hdr_white_scale; + + dst_row[x] = cv::Vec3b(linear_to_srgb_u8(b), linear_to_srgb_u8(g), linear_to_srgb_u8(r)); + } + } + }); + + return bgr8; +} + +MAA_CTRL_UNIT_NS_END diff --git a/source/MaaWin32ControlUnit/Screencap/HdrDisplayUtils.hpp b/source/MaaWin32ControlUnit/Screencap/HdrDisplayUtils.hpp new file mode 100644 index 0000000000..78d1401a36 --- /dev/null +++ b/source/MaaWin32ControlUnit/Screencap/HdrDisplayUtils.hpp @@ -0,0 +1,24 @@ +#pragma once + +#include + +#include "Common/Conf.h" +#include "MaaUtils/NoWarningCV.hpp" +#include "MaaUtils/SafeWindows.hpp" + +MAA_CTRL_UNIT_NS_BEGIN + +struct HdrDisplayState +{ + bool hdr_enabled = false; + bool hdr_user_enabled = false; + int active_color_mode = 0; + float sdr_white_nits = 80.0f; + bool valid = false; +}; + +std::optional query_windows_hdr_enabled_go_compatible(); +std::optional query_hdr_display_state(HMONITOR target_monitor); +cv::Mat compensate_hdr_sdr_capture(const cv::Mat& bgra, float sdr_white_nits); + +MAA_CTRL_UNIT_NS_END diff --git a/source/MaaWin32ControlUnit/Screencap/HwndUtils.hpp b/source/MaaWin32ControlUnit/Screencap/HwndUtils.hpp index 5b70e9ab6c..e539ed1c73 100644 --- a/source/MaaWin32ControlUnit/Screencap/HwndUtils.hpp +++ b/source/MaaWin32ControlUnit/Screencap/HwndUtils.hpp @@ -79,7 +79,9 @@ inline bool ensure_window_on_screen(HWND hwnd) return false; } - RECT monitor_rect = mi.rcWork; + // Borderless/fullscreen windows are allowed to cover the whole monitor, + // including the taskbar area, so use rcMonitor instead of rcWork there. + RECT monitor_rect = is_fullscreen(hwnd) ? mi.rcMonitor : mi.rcWork; int monitor_w = monitor_rect.right - monitor_rect.left; int monitor_h = monitor_rect.bottom - monitor_rect.top; diff --git a/source/MaaWin32ControlUnit/Screencap/PrintWindowScreencap.cpp b/source/MaaWin32ControlUnit/Screencap/PrintWindowScreencap.cpp index 363a70d042..1d54e8f4e4 100644 --- a/source/MaaWin32ControlUnit/Screencap/PrintWindowScreencap.cpp +++ b/source/MaaWin32ControlUnit/Screencap/PrintWindowScreencap.cpp @@ -1,5 +1,8 @@ #include "PrintWindowScreencap.h" +#include + +#include "HdrDisplayUtils.hpp" #include "HwndUtils.hpp" #include "MaaUtils/Logger.h" @@ -9,6 +12,7 @@ std::optional PrintWindowScreencap::screencap() { if (!hwnd_) { LogError << "hwnd_ is nullptr"; + last_screencap_info_ = {}; return std::nullopt; } @@ -17,6 +21,7 @@ std::optional PrintWindowScreencap::screencap() RECT rect = { 0 }; if (!GetClientRect(hwnd_, &rect)) { LogError << "GetClientRect failed, error code: " << GetLastError(); + last_screencap_info_ = {}; return std::nullopt; } @@ -25,6 +30,7 @@ std::optional PrintWindowScreencap::screencap() if (width <= 0 || height <= 0) { LogError << "Invalid window size" << VAR(width) << VAR(height); + last_screencap_info_ = {}; return std::nullopt; } @@ -52,24 +58,28 @@ std::optional PrintWindowScreencap::screencap() hdc = GetDC(hwnd_); if (!hdc) { LogError << "GetDC failed, error code: " << GetLastError(); + last_screencap_info_ = {}; return std::nullopt; } mem_dc = CreateCompatibleDC(hdc); if (!mem_dc) { LogError << "CreateCompatibleDC failed, error code: " << GetLastError(); + last_screencap_info_ = {}; return std::nullopt; } bitmap = CreateCompatibleBitmap(hdc, width, height); if (!bitmap) { LogError << "CreateCompatibleBitmap failed, error code: " << GetLastError(); + last_screencap_info_ = {}; return std::nullopt; } old_obj = SelectObject(mem_dc, bitmap); if (!old_obj) { LogError << "SelectObject failed, error code: " << GetLastError(); + last_screencap_info_ = {}; return std::nullopt; } @@ -80,6 +90,7 @@ std::optional PrintWindowScreencap::screencap() constexpr UINT nFlags = PW_CLIENTONLY | PW_RENDERFULLCONTENT; if (!PrintWindow(hwnd_, mem_dc, nFlags)) { LogError << "PrintWindow failed, error code: " << GetLastError(); + last_screencap_info_ = {}; return std::nullopt; } @@ -95,11 +106,34 @@ std::optional PrintWindowScreencap::screencap() cv::Mat mat(height, width, CV_8UC4); if (!GetDIBits(mem_dc, bitmap, 0, height, mat.data, &bmi, DIB_RGB_COLORS)) { LogError << "GetDIBits failed, error code: " << GetLastError(); + last_screencap_info_ = {}; return std::nullopt; } + const HMONITOR target_monitor = MonitorFromWindow(hwnd_, MONITOR_DEFAULTTONEAREST); + const std::optional hdr_state = query_hdr_display_state(target_monitor); + if (hdr_state.has_value() && hdr_state->valid && hdr_state->hdr_enabled) { + cv::Mat compensated = compensate_hdr_sdr_capture(mat, hdr_state->sdr_white_nits); + if (!compensated.empty()) { + last_screencap_info_ = { + .hdr_capture_active = false, + .hdr_preprocessed = true, + .gpu_processed = false, + .display_hdr_active = true, + .display_hdr_compensated = true, + }; + static auto last_log_time = std::chrono::steady_clock::time_point {}; + const auto now = std::chrono::steady_clock::now(); + if (last_log_time == std::chrono::steady_clock::time_point {} || now - last_log_time > std::chrono::seconds(5)) { + last_log_time = now; + LogInfo << "PrintWindow HDR display compensation applied" << VAR(hdr_state->sdr_white_nits); + } + return compensated; + } + } + + last_screencap_info_ = {}; return bgra_to_bgr(mat); } MAA_CTRL_UNIT_NS_END - diff --git a/source/MaaWin32ControlUnit/Screencap/PrintWindowScreencap.h b/source/MaaWin32ControlUnit/Screencap/PrintWindowScreencap.h index c655af9d02..b040be9abc 100644 --- a/source/MaaWin32ControlUnit/Screencap/PrintWindowScreencap.h +++ b/source/MaaWin32ControlUnit/Screencap/PrintWindowScreencap.h @@ -19,10 +19,11 @@ class PrintWindowScreencap : public ScreencapBase public: // from ScreencapBase virtual std::optional screencap() override; + virtual ScreencapInfo last_screencap_info() const override { return last_screencap_info_; } private: HWND hwnd_ = nullptr; + ScreencapInfo last_screencap_info_ {}; }; MAA_CTRL_UNIT_NS_END - diff --git a/source/MaaWin32ControlUnit/Screencap/PrintWindowWithPseudoMinimizeScreencap.h b/source/MaaWin32ControlUnit/Screencap/PrintWindowWithPseudoMinimizeScreencap.h index 54bd16aee3..b5c11344f1 100644 --- a/source/MaaWin32ControlUnit/Screencap/PrintWindowWithPseudoMinimizeScreencap.h +++ b/source/MaaWin32ControlUnit/Screencap/PrintWindowWithPseudoMinimizeScreencap.h @@ -26,6 +26,8 @@ class PrintWindowWithPseudoMinimizeScreencap : public ScreencapBase return inner_.screencap(); } + virtual ScreencapInfo last_screencap_info() const override { return inner_.last_screencap_info(); } + virtual void inactive() override { if (helper_.is_pseudo_minimized()) { diff --git a/source/MaaWin32ControlUnit/Screencap/ScreenDCScreencap.cpp b/source/MaaWin32ControlUnit/Screencap/ScreenDCScreencap.cpp index 536b7be3fa..3e9582cd9e 100644 --- a/source/MaaWin32ControlUnit/Screencap/ScreenDCScreencap.cpp +++ b/source/MaaWin32ControlUnit/Screencap/ScreenDCScreencap.cpp @@ -1,5 +1,8 @@ #include "ScreenDCScreencap.h" +#include + +#include "HdrDisplayUtils.hpp" #include "HwndUtils.hpp" #include "MaaUtils/Logger.h" @@ -9,6 +12,7 @@ std::optional ScreenDCScreencap::screencap() { if (!hwnd_) { LogError << "hwnd_ is nullptr"; + last_screencap_info_ = {}; return std::nullopt; } @@ -21,6 +25,7 @@ std::optional ScreenDCScreencap::screencap() RECT window_rect = { 0 }; if (!GetWindowRect(hwnd_, &window_rect)) { LogError << "GetWindowRect failed, error code: " << GetLastError(); + last_screencap_info_ = {}; return std::nullopt; } @@ -28,6 +33,7 @@ std::optional ScreenDCScreencap::screencap() RECT client_rect = { 0 }; if (!GetClientRect(hwnd_, &client_rect)) { LogError << "GetClientRect failed, error code: " << GetLastError(); + last_screencap_info_ = {}; return std::nullopt; } @@ -35,6 +41,7 @@ std::optional ScreenDCScreencap::screencap() POINT client_top_left = { client_rect.left, client_rect.top }; if (!ClientToScreen(hwnd_, &client_top_left)) { LogError << "ClientToScreen failed, error code: " << GetLastError(); + last_screencap_info_ = {}; return std::nullopt; } @@ -43,6 +50,7 @@ std::optional ScreenDCScreencap::screencap() if (width <= 0 || height <= 0) { LogError << "Invalid window size" << VAR(width) << VAR(height); + last_screencap_info_ = {}; return std::nullopt; } @@ -70,41 +78,69 @@ std::optional ScreenDCScreencap::screencap() screen_dc = GetDC(nullptr); if (!screen_dc) { LogError << "GetDC failed, error code: " << GetLastError(); + last_screencap_info_ = {}; return std::nullopt; } mem_dc = CreateCompatibleDC(screen_dc); if (!mem_dc) { LogError << "CreateCompatibleDC failed, error code: " << GetLastError(); + last_screencap_info_ = {}; return std::nullopt; } bitmap = CreateCompatibleBitmap(screen_dc, width, height); if (!bitmap) { LogError << "CreateCompatibleBitmap failed, error code: " << GetLastError(); + last_screencap_info_ = {}; return std::nullopt; } old_obj = SelectObject(mem_dc, bitmap); if (!old_obj) { LogError << "SelectObject failed, error code: " << GetLastError(); + last_screencap_info_ = {}; return std::nullopt; } // 从屏幕DC复制客户区内容 if (!BitBlt(mem_dc, 0, 0, width, height, screen_dc, client_top_left.x, client_top_left.y, SRCCOPY)) { LogError << "BitBlt failed, error code: " << GetLastError(); + last_screencap_info_ = {}; return std::nullopt; } cv::Mat mat(height, width, CV_8UC4); if (!GetBitmapBits(bitmap, width * height * 4, mat.data)) { LogError << "GetBitmapBits failed, error code: " << GetLastError(); + last_screencap_info_ = {}; return std::nullopt; } + const HMONITOR target_monitor = MonitorFromWindow(hwnd_, MONITOR_DEFAULTTONEAREST); + const std::optional hdr_state = query_hdr_display_state(target_monitor); + if (hdr_state.has_value() && hdr_state->valid && hdr_state->hdr_enabled) { + cv::Mat compensated = compensate_hdr_sdr_capture(mat, hdr_state->sdr_white_nits); + if (!compensated.empty()) { + last_screencap_info_ = { + .hdr_capture_active = false, + .hdr_preprocessed = true, + .gpu_processed = false, + .display_hdr_active = true, + .display_hdr_compensated = true, + }; + static auto last_log_time = std::chrono::steady_clock::time_point {}; + const auto now = std::chrono::steady_clock::now(); + if (last_log_time == std::chrono::steady_clock::time_point {} || now - last_log_time > std::chrono::seconds(5)) { + last_log_time = now; + LogInfo << "ScreenDC HDR display compensation applied" << VAR(hdr_state->sdr_white_nits); + } + return compensated; + } + } + + last_screencap_info_ = {}; return bgra_to_bgr(mat); } MAA_CTRL_UNIT_NS_END - diff --git a/source/MaaWin32ControlUnit/Screencap/ScreenDCScreencap.h b/source/MaaWin32ControlUnit/Screencap/ScreenDCScreencap.h index 20d3e89ab6..9c8bb2b709 100644 --- a/source/MaaWin32ControlUnit/Screencap/ScreenDCScreencap.h +++ b/source/MaaWin32ControlUnit/Screencap/ScreenDCScreencap.h @@ -19,10 +19,11 @@ class ScreenDCScreencap : public ScreencapBase public: // from ScreencapBase virtual std::optional screencap() override; + virtual ScreencapInfo last_screencap_info() const override { return last_screencap_info_; } private: HWND hwnd_ = nullptr; + ScreencapInfo last_screencap_info_ {}; }; MAA_CTRL_UNIT_NS_END -