Merge pull request #785 from o-sdn-o/gui-bridge

Enable floating point (pixel-wise) mouse reporting over Win32 Console API

Author: o-sdn-o

doc/apps.md

@@ -38,6 +38,7 @@
   - Enforced ENABLE_PROCESSED_OUTPUT and ENABLE_VIRTUAL_TERMINAL_PROCESSING modes.
   - Disabled ENABLE_QUICK_EDIT_MODE mode.
   - Per process (not per process name) Windows Command Prompt (cmd.exe) input history, aka "Line input" or "Cooked read".
+  - Floating point (pixel-wise) mouse reporting.
 - Stdin/stdout logging.
 
 ### Private control sequences
@@ -141,6 +142,64 @@ Example 4. Output the longest word in the Hindi language 16x1 (G1_00):
 Screenshot:  
   ![image](images/vtm_character_geometry_modifiers_screenshot.png)
 
+### Floating point (pixel-wise) mouse reporting
+
+On Windows, when using the Win32 Console API, vtm reports mouse events with fractional mouse coordinates. Fractional coordinates are 32-bit floating-point numbers that represent the position of the cursor relative to the console's grid of text cells. Screen pixel coordinates can be calculated by multiplying the fractional coordinates by the cell size.
+
+Example:
+```c++
+#include <iostream>
+#include <vector>
+#include <windows.h>
+
+static constexpr auto custom_type = 0b1000'0000'0000'0000;
+static constexpr auto fp2d_mouse = 3;
+struct fp2d_mouse_input : MENU_EVENT_RECORD // MENU_EVENT_RECORD structure extension.
+{
+    //DWORD EventType   = MENU_EVENT;
+    //DWORD dwCommandId = custom_type | fp2d_mouse;
+    float x; // Floating point mouse x coord.
+    float y; // Floating point mouse y coord.
+};
+int main()
+{
+    auto inp = ::GetStdHandle(STD_INPUT_HANDLE);
+    ::SetConsoleMode(inp, ENABLE_MOUSE_INPUT);
+    auto r = INPUT_RECORD{};
+    auto count = DWORD{};
+    auto x = std::numeric_limits<float>::quiet_NaN();
+    auto y = std::numeric_limits<float>::quiet_NaN();
+    auto mouse_out = false;
+    std::cout << "Press any mouse button to exit\n";
+    while (true)
+    {
+        ::ReadConsoleInputW(inp, &r, 1, &count);
+        if (r.EventType == MENU_EVENT)
+        {
+            if (r.Event.MenuEvent.dwCommandId == (custom_type | fp2d_mouse)) // The floating point coordinates message always precedes the classic mouse report.
+            {
+                x = reinterpret_cast<fp2d_mouse_input*>(&r.Event.MenuEvent)->x;
+                y = reinterpret_cast<fp2d_mouse_input*>(&r.Event.MenuEvent)->y;
+                mouse_out = std::isnan(x); // NaN is a sign that the mouse has gone away or is disconnected.
+                if (mouse_out) std::cout << "The mouse has left the window\n";
+            }
+        }
+        else if (r.EventType == MOUSE_EVENT && !mouse_out)
+        {
+            if (r.Event.MouseEvent.dwButtonState) return 0;
+            if (std::isnan(x)) // Classical behavior.
+            {
+                std::cout << "MOUSE_EVENT coord: " << r.Event.MouseEvent.dwMousePosition.X << "," << r.Event.MouseEvent.dwMousePosition.Y << "\n";
+            }
+            else // Floating point mouse coordinates.
+            {
+                std::cout << "MOUSE_EVENT coord: " << x << "," << y << "\n";
+            }
+        }
+    }
+}
+```
+
 ### Window menu
 
 It is possible to create your own terminal window menu from scratch by configuring own menu items in the `<config/terminal/menu/>` subsection of the configuration file. See (`doc/settings.md#event-scripting`)[https://github.com/directvt/vtm/blob/master/doc/settings.md#event-scripting] for details.

doc/vt-input-mode.md

@@ -78,7 +78,7 @@ Note: By enabling `vt-input-mode`, all current terminal modes are automatically
   ```
 - Mouse
   ```
-  ESC _ event=mouse ; id=0 ; kbmods=<KeyMods> ; coord=<X>,<Y> ; buttons=<ButtonState> ; wheel=<DeltaY>[,<DeltaX>] ESC \
+  ESC _ event=mouse ; id=0 ; kbmods=<KeyMods> ; coord=<X>,<Y> ; buttons=<ButtonState> ; scroll=<DeltaX>,<DeltaY> ESC \
   ```
 - Focus
   ```
@@ -351,7 +351,7 @@ Key ID | Name               | Generic Name       | Scan Code | Notes
 ### Mouse
 
 ```
-ESC _ event=mouse ; id=0 ; kbmods=<KeyMods> ; coord=<X>,<Y> ; buttons=<ButtonState> ; wheel=<DeltaY>[,<DeltaX>] ESC \
+ESC _ event=mouse ; id=0 ; kbmods=<KeyMods> ; coord=<X>,<Y> ; buttons=<ButtonState> ; scroll=<DeltaX>,<DeltaY> ESC \
 ```
 
 Attribute                   | Description
@@ -360,7 +360,7 @@ Attribute                   | Description
 `kbmods=<KeyMods>`          | Keyboard modifiers (see Keyboard event).
 `coord=<X>,<Y>`             | Pixel-wise coordinates of the mouse pointer. Each coordinate is represented in the form of a floating point value of the sum of the integer coordinate of the cell in the terminal window grid and the relative offset within the cell in the range `[0.0f, 1.0f)`.
 `buttons=<ButtonState>`     | Mouse button state.
-`wheel=<DeltaY>[,<DeltaX>]` | Vertical and horizontal high-resolution wheel delta integer value.
+`scroll=<DeltaX>,<DeltaY>`  | Integer value of high resolution horizontal and vertical scroll delta in integer 1/120 units.
 
 In response to the activation of `mouse` tracking, the application receives a vt-sequence containing current mouse state:
 ```
@@ -378,6 +378,8 @@ Bit | Active button
 2   | Middle
 3   | 4th
 4   | 5th
+... | ...
+N-1 | Nth
 
 Note: Mouse tracking will continue outside the terminal window as long as the mouse button pressed inside the window is active. In this case, coordinates with negative values are possible.
 

src/netxs/apps.hpp

@@ -703,7 +703,7 @@ namespace netxs::app::shared
             chord_block->attach_cells({ 5, 3 }, {           {}, label(skin::globals().NsInfoGeneric), label(skin::globals().NsInfoLiteral), label(skin::globals().NsInfoSpecific), label(skin::globals().NsInfoScancodes),
                                                  pressed_label, pressed[0],       pressed[1],       pressed[2],        pressed[3],
                                                 released_label, released[0],      released[1],      released[2],       released[3] });
-            released[0]->set("<Press any keys>")->hidden = faux;
+            released[0]->set(skin::globals().NsInfo_pressanykeys)->hidden = faux;
             auto& update = window_ptr->base::field([pressed, released](auto& boss, hids& gear, bool is_key_event)
             {
                 //log("vkchord=%% keyid=%% hexvkchord=%% hexscchord=%% hexchchord=%%", input::key::kmap::to_string(gear.vkchord, faux),

src/netxs/apps/test.hpp

@@ -58,7 +58,7 @@ namespace netxs::app::test
             {
                 return ansi::mgl(1).wrp(wrap::off).fgc(hdrclr).unc(whitedk).cap(caption).erl().und(unln::none).eol().fgc(txtclr).mgl(3).unc(0).wrp(wrap::on);
             };
-            return ansi::mgl(1).mgr(2).jet(bias::center)
+            auto crop = ansi::mgl(1).mgr(2).jet(bias::center)
                 .add("\n")
                 .wrp(wrap::off).fgc(hdrclr).cap(skin::globals().NsInfoSF, 3, 3, faux).eol()
                 .jet(bias::left)
@@ -260,8 +260,10 @@ namespace netxs::app::test
                 .add(header(skin::globals().NsInfoCharacterHalves))
                 .add("\n")
                 .add("😎", vss<21,11>, " 😃", vss<21,21>, "<VS21_11/VS21_21\n")
-                .add("\n")
-                .add(header(skin::globals().NsInfosRGBBlending))
+                .add("\n");
+            if constexpr (debugmode)
+            {
+                crop.add(header(skin::globals().NsInfosRGBBlending))
                 .add("\n")
                 .add(skin::globals().NsInfoPressCtrlCaps)
                 .add("\n")
@@ -275,6 +277,8 @@ namespace netxs::app::test
                 .bgc(argb{})
                 .fgc(purered).add(" test \n")
                 .fgc(purecyan).add(" test ");
+            }
+            return crop;
         };
 
         auto get_text = []

src/netxs/desktopio/ansivt.hpp

@@ -612,8 +612,9 @@ namespace netxs::ansi
             return *this;
         }
         template<class T>
-        auto& mouse_sgr(T const& gear, twod coor) // escx: Mouse tracking report (SGR).
+        auto& mouse_sgr(T const& gear, fp2d coor) // escx: Mouse tracking report (SGR).
         {
+            if (std::isnan(coor.x)) return *this;
             using hids = T;
             static constexpr auto left     = si32{ 0  };
             static constexpr auto mddl     = si32{ 1  };
@@ -681,13 +682,14 @@ namespace netxs::ansi
             auto count = std::max(1, std::abs(gear.m_sys.wheelsi));
             while (count--)
             {
-                add("\033[<", ctrl, ';', coor.x, ';', coor.y, pressed ? 'M' : 'm');
+                add("\033[<", ctrl, ';', (si32)coor.x, ';', (si32)coor.y, pressed ? 'M' : 'm');
             }
             return *this;
         }
         template<class T>
-        auto& mouse_x11(T const& gear, twod coor, bool utf8) // escx: Mouse tracking report (X11).
+        auto& mouse_x11(T const& gear, fp2d coor, bool utf8) // escx: Mouse tracking report (X11).
         {
+            if (std::isnan(coor.x)) return *this;
             using hids = T;
             static constexpr auto left     = si32{ 0  };
             static constexpr auto mddl     = si32{ 1  };
@@ -739,15 +741,15 @@ namespace netxs::ansi
                 if (utf8)
                 {
                     add("\033[M");
-                    utf::to_utf_from_code(std::clamp(ctrl,   0, si16max - 32) + 32, *this);
-                    utf::to_utf_from_code(std::clamp(coor.x, 1, si16max - 32) + 32, *this);
-                    utf::to_utf_from_code(std::clamp(coor.y, 1, si16max - 32) + 32, *this);
+                    utf::to_utf_from_code(std::clamp(ctrl,         0, si16max - 32) + 32, *this);
+                    utf::to_utf_from_code(std::clamp((si32)coor.x, 1, si16max - 32) + 32, *this);
+                    utf::to_utf_from_code(std::clamp((si32)coor.y, 1, si16max - 32) + 32, *this);
                 }
                 else
                 {
-                    add("\033[M", (char)(std::clamp(ctrl,   0, 127-32) + 32),
-                                  (char)(std::clamp(coor.x, 1, 127-32) + 32),
-                                  (char)(std::clamp(coor.y, 1, 127-32) + 32));
+                    add("\033[M", (char)(std::clamp(ctrl,         0, 127-32) + 32),
+                                  (char)(std::clamp((si32)coor.x, 1, 127-32) + 32),
+                                  (char)(std::clamp((si32)coor.y, 1, 127-32) + 32));
                 }
             }
             return *this;

src/netxs/desktopio/application.hpp

@@ -22,7 +22,7 @@ namespace netxs::app
 
 namespace netxs::app::shared
 {
-    static const auto version = "v2025.08.10";
+    static const auto version = "v2025.08.12";
     static const auto repository = "https://github.com/directvt/vtm";
     static const auto usr_config = "~/.config/vtm/settings.xml"s;
     static const auto sys_config = "/etc/vtm/settings.xml"s;

src/netxs/desktopio/consrv.hpp

@@ -23,7 +23,7 @@ struct consrv
     virtual fd_t watch() = 0;
     virtual bool send(view utf8) = 0;
     virtual void keybd(input::hids& gear, bool decckm) = 0;
-    virtual void mouse(input::hids& gear, bool moved, twod coord, input::mouse::prot encod, input::mouse::mode state) = 0;
+    virtual void mouse(input::hids& gear, bool moved, fp2d coord, input::mouse::prot encod, input::mouse::mode state) = 0;
     virtual void paste(view block) = 0;
     virtual void focus(bool state) = 0;
     virtual void winsz(twod newsz) = 0;
@@ -959,16 +959,22 @@ struct impl : consrv
         void style(si32 format)
         {
             auto lock = std::lock_guard{ locker };
-            auto data = INPUT_RECORD{ .EventType = MENU_EVENT };
-            data.Event.MenuEvent.dwCommandId = nt::console::event::custom | nt::console::event::style;
-            stream.emplace_back(data);
-            data.Event.MenuEvent.dwCommandId = nt::console::event::custom | format;
-            stream.emplace_back(data);
+            auto data = nt::console::style_input{ .format = format };
+            stream.emplace_back(*reinterpret_cast<INPUT_RECORD*>(&data));
             ondata.reset();
             signal.notify_one();
         }
-        void mouse(input::hids& gear, twod coord)
+        void mouse(input::hids& gear, fp2d coord)
         {
+            if (std::isnan(coord.x)) // Forward a mouse halt event.
+            {
+                auto lock = std::lock_guard{ locker };
+                auto r2 = nt::console::fp2d_mouse_input{ .coord = coord };
+                stream.emplace_back(*reinterpret_cast<INPUT_RECORD*>(&r2));
+                ondata.reset();
+                signal.notify_one();
+                return;
+            }
             auto state = os::nt::ms_kbstate(gear.ctlstat);
             auto bttns = gear.m_sys.buttons & 0b00011111;
             auto moved = gear.m_sys.buttons == gear.m_sav.buttons && gear.m_sys.wheelfp == 0.f; // No events means mouse move. MSFT: "MOUSE_EVENT_RECORD::dwEventFlags: If this value is zero, it indicates a mouse button being pressed or released". Far Manager relies on this.
@@ -982,7 +988,7 @@ struct impl : consrv
                 {
                     auto& s = dclick[i];
                     auto fired = gear.m_sys.timecod;
-                    if (fired - s.fired < gear.delay && s.coord == coord) // Set the double-click flag if the delay has not expired and the mouse is in the same position.
+                    if (fired - s.fired < gear.delay && s.coord == twod{ coord }) // Set the double-click flag if the delay has not expired and the mouse is in the same cell.
                     {
                         flags |= DOUBLE_CLICK;
                         s.fired = {};
@@ -1003,6 +1009,8 @@ struct impl : consrv
                 if (gear.m_sys.hzwheel) flags |= MOUSE_HWHEELED;
             }
             auto lock = std::lock_guard{ locker };
+            auto r2 = nt::console::fp2d_mouse_input{ .coord = coord };
+            stream.emplace_back(*reinterpret_cast<INPUT_RECORD*>(&r2));
             stream.emplace_back(INPUT_RECORD
             {
                 .EventType = MOUSE_EVENT,
@@ -1012,8 +1020,8 @@ struct impl : consrv
                     {
                         .dwMousePosition =
                         {
-                            .X = (si16)std::clamp<si32>(coord.x, si16min, si16max),
-                            .Y = (si16)std::clamp<si32>(coord.y, si16min, si16max),
+                            .X = (si16)std::clamp<si32>((si32)coord.x, si16min, si16max),
+                            .Y = (si16)std::clamp<si32>((si32)coord.y, si16min, si16max),
                         },
                         .dwButtonState     = (DWORD)bttns,
                         .dwControlKeyState = state,
@@ -4934,7 +4942,7 @@ struct impl : consrv
             uiterm.mtrack.setmode(input::mouse::prot::w32);
         }
     }
-    void mouse(input::hids& gear, bool /*moved*/, twod coord, input::mouse::prot /*encod*/,
+    void mouse(input::hids& gear, bool /*moved*/, fp2d coord, input::mouse::prot /*encod*/,
                  input::mouse::mode /*state*/) { events.mouse(gear, coord);        }
     void keybd(input::hids& gear, bool decckm) { events.keybd(gear, decckm);       }
     void paste(view block)                     { events.paste(block);              }
@@ -5153,7 +5161,7 @@ struct consrv : ipc::stdcon
     {
         //todo win32-input-mode
     }
-    void mouse(input::hids& /*gear*/, bool /*moved*/, twod /*coord*/, input::mouse::prot /*encod*/, input::mouse::mode /*state*/)
+    void mouse(input::hids& /*gear*/, bool /*moved*/, fp2d /*coord*/, input::mouse::prot /*encod*/, input::mouse::mode /*state*/)
     {
         //todo win32-input-mode
     }

src/netxs/desktopio/geometry.hpp

@@ -283,7 +283,7 @@ namespace netxs
         constexpr twod clamp(twod point) const
         {
             auto [min, max] = twod::sort(coor, coor + size);
-            return std::clamp(point, min, max - dot_11);
+            return std::clamp(point, min, std::max(min, max - dot_11)); // Use std::max for zero size cases.
         }
         // rect: Is the point inside the rect.
         constexpr bool hittest(twod p) const

src/netxs/desktopio/intmath.hpp

@@ -92,6 +92,8 @@ namespace netxs
     static constexpr auto fp64min = std::numeric_limits<fp64>::lowest();
     static constexpr auto fp32epsilon = std::numeric_limits<fp32>::min();
     static constexpr auto fp64epsilon = std::numeric_limits<fp64>::min();
+    static constexpr auto fp32nan = std::numeric_limits<fp32>::quiet_NaN();
+    static constexpr auto fp64nan = std::numeric_limits<fp64>::quiet_NaN();
     static constexpr auto debugmode
         #if defined(DEBUG)
         = true;