First working version of Dockable context menu and tracking the toolwindow's state

bclothier · bclothier · commit 7419dc601d0b · 2019-10-09T23:19:33.000-05:00
diff --git a/Rubberduck.Main/ComClientLibrary/UI/DockableWindowHost.cs b/Rubberduck.Main/ComClientLibrary/UI/DockableWindowHost.cs
@@ -4,6 +4,7 @@
 using System.Diagnostics.CodeAnalysis;
 using System.Drawing;
 using System.Runtime.InteropServices;
+using System.Text;
 using System.Windows.Forms;
 using Rubberduck.VBEditor;
 using Rubberduck.VBEditor.WindowsApi;
@@ -429,46 +430,76 @@ private void AdjustSize()
             }
         }
 
-        private const int WM_SYSCOMMAND = 0x112;
-        private const int MF_BYPOSITION = 0x400;
-        private const int ToggleDockableMenuId = 1000;
+        private static void ToggleDockable(IntPtr hWndVBE)
+        {
+            NativeMethods.SendMessage(hWndVBE, 0x1044, (IntPtr)0xB5, IntPtr.Zero);
+        }
+
+        [ComVisible(false)]
+        public class ParentWindow : SubclassingWindow
+        {
+            private const int MF_BYPOSITION = 0x400;
 
-        [DllImport("user32.dll")]
-        private static extern IntPtr GetSystemMenu(IntPtr hWnd, bool bRevert);
-        [DllImport("user32.dll")]
-        private static extern bool InsertMenu(IntPtr hMenu, int wPosition, int wFlags, int wIDNewItem, string lpNewItem);
+            public event SubClassingWindowEventHandler CallBackEvent;
+            public delegate void SubClassingWindowEventHandler(object sender, SubClassingWindowEventArgs e);
 
-        [DllImport("user32.dll", CharSet = CharSet.Auto)]
-        public static extern IntPtr SendMessage(IntPtr hWnd, int Msg, int wParam, IntPtr lParam);
+            private readonly IntPtr _vbeHwnd;
 
-        private static void InsertDockableToggle(IntPtr handle)
-        {
-            var menuHandle = GetSystemMenu(handle, false);
+            private IntPtr _containerHwnd;
+            private ToolWindowState _windowState;
+            private IntPtr _menuHandle;
 
-            if (menuHandle == IntPtr.Zero)
+            private enum ToolWindowState
             {
-                Debug.Print("No menu handle");
-                return;
+                Unknown,
+                Docked,
+                Floating,
+                Undockable
             }
 
-            if (!InsertMenu(menuHandle, 5, MF_BYPOSITION, ToggleDockableMenuId, "Dockable"))
+            private ToolWindowState GetWindowState(IntPtr containerHwnd)
             {
-                Debug.Print("Failed to insert a menu item for dockable command");
+                var className = new StringBuilder(255);
+                if (NativeMethods.GetClassName(containerHwnd, className, className.Capacity) > 0)
+                {
+                    switch (className.ToString())
+                    {
+                        case "wndclass_desked_gsk":
+                            return ToolWindowState.Docked;
+                        case "VBFloatingPalette":
+                            return ToolWindowState.Floating;
+                        case "DockingView":
+                            return ToolWindowState.Undockable;
+                    }
+                }
+
+                return ToolWindowState.Unknown;
             }
-        }
 
-        private static void ToggleDockable(IntPtr hWndVBE)
-        {
-            SendMessage(hWndVBE, 0x1044, 0xB5, IntPtr.Zero);
-        }
+            private void DisplayUndockableContextMenu(IntPtr handle, IntPtr lParam)
+            {
+                if (_menuHandle == IntPtr.Zero)
+                {
+                    _menuHandle = NativeMethods.CreatePopupMenu();
 
-        [ComVisible(false)]
-        public class ParentWindow : SubclassingWindow
-        {
-            public event SubClassingWindowEventHandler CallBackEvent;
-            public delegate void SubClassingWindowEventHandler(object sender, SubClassingWindowEventArgs e);
+                    if (_menuHandle == IntPtr.Zero)
+                    {
+                        Debug.Print("Cannot create menu handle");
+                        return;
+                    }
 
-            private readonly IntPtr _vbeHwnd;
+                    if (!NativeMethods.InsertMenu(_menuHandle, 0, MF_BYPOSITION, (UIntPtr)WM.RUBBERDUCK_UNDOCKABLE_CONTEXT_MENU, "Dockable" + char.MinValue))
+                    {
+                        Debug.Print("Failed to insert a menu item for dockable command");
+                    }
+                }
+
+                var param = new LParam {Value = (uint)lParam};
+                if (!NativeMethods.TrackPopupMenuEx(_menuHandle, 0x0, param.LowWord, param.HighWord, handle, IntPtr.Zero ))
+                {
+                    Debug.Print("Failed to set the context menu for undockable tool windows");
+                };
+            }
 
             private void OnCallBackEvent(SubClassingWindowEventArgs e)
             {
@@ -485,10 +516,24 @@ public override int SubClassProc(IntPtr hWnd, IntPtr msg, IntPtr wParam, IntPtr
             {
                 switch ((uint)msg)
                 {
-                    case (uint)WM.SYSCOMMAND:
+                    case (uint)WM.WINDOWPOSCHANGED:
+                        var containerHwnd = GetParent(hWnd);
+                        if (containerHwnd != _containerHwnd)
+                        {
+                            _containerHwnd = containerHwnd;
+                            _windowState = GetWindowState(_containerHwnd);
+                        }
+                        break;
+                    case (uint)WM.CONTEXTMENU:
+                        if (_windowState == ToolWindowState.Undockable)
+                        {
+                            DisplayUndockableContextMenu(hWnd, lParam);
+                        }
+                        break;
+                    case (uint)WM.COMMAND:
                         switch (wParam.ToInt32())
                         {
-                            case ToggleDockableMenuId:
+                            case (int)WM.RUBBERDUCK_UNDOCKABLE_CONTEXT_MENU:
                                 ToggleDockable(_vbeHwnd);
                                 break;
                         }
@@ -503,6 +548,15 @@ public override int SubClassProc(IntPtr hWnd, IntPtr msg, IntPtr wParam, IntPtr
                     case (uint)WM.KILLFOCUS:
                         if (!_closing) User32.SendMessage(_vbeHwnd, WM.RUBBERDUCK_CHILD_FOCUS, Hwnd, IntPtr.Zero);
                         break;
+                    case (uint)WM.DESTROY:
+                        if (_menuHandle != IntPtr.Zero)
+                        {
+                            if (!NativeMethods.DestroyMenu(_menuHandle))
+                            {
+                                Debug.Print($"Failed to destroy the menu handle {_menuHandle}");
+                            }
+                        }
+                        break;
                 }
                 return base.SubClassProc(hWnd, msg, wParam, lParam, uIdSubclass, dwRefData);
             }
diff --git a/Rubberduck.VBEEditor/WindowsApi/NativeMethods.cs b/Rubberduck.VBEEditor/WindowsApi/NativeMethods.cs
@@ -13,6 +13,18 @@ namespace Rubberduck.VBEditor.WindowsApi
     /// </remarks>
     public static class NativeMethods
     {
+        [DllImport("user32.dll")]
+        public static extern IntPtr CreatePopupMenu();
+
+        [DllImport("user32.dll")]
+        public static extern bool TrackPopupMenuEx(IntPtr hmenu, uint fuFlags, int x, int y, IntPtr hwnd, IntPtr lptpm);
+
+        [DllImport("user32.dll", EntryPoint = "InsertMenuW", SetLastError = true, CharSet = CharSet.Unicode)]
+        public static extern bool InsertMenu(IntPtr hMenu, uint wPosition, uint wFlags, UIntPtr wIDNewItem, [MarshalAs(UnmanagedType.LPWStr)]string lpNewItem);
+
+        [DllImport("user32.dll")]
+        public static extern bool DestroyMenu(IntPtr hMenu);
+
         /// <summary>   Sends a message to the OS. </summary>
         ///
         /// <param name="hWnd">     The window handle. </param>
@@ -68,6 +80,14 @@ public enum PeekMessageRemoval : uint
         [DllImport("user32", ExactSpelling = true, CharSet = CharSet.Unicode)]
         public static extern int EnumChildWindows(IntPtr parentWindowHandle, EnumChildWindowsDelegate lpEnumFunction, IntPtr lParam);
 
+        /// <summary> Retrieves the name of the class to which the specified window belongs. </summary>
+        /// <param name="hWnd">A handle to the window and, indirectly, the class to which the window belongs.</param>
+        /// <param name="lpClassName">The class name string.</param>
+        /// <param name="nMaxCount">The length of the <see cref="lpClassName" /> buffer, in characters. The buffer must be large enough to include the terminating null character; otherwise, the class name string is truncated to <see cref="nMaxCount" /> characters.</param>
+        /// <returns>If the function succeeds, the return value is the number of characters copied to the buffer, not including the terminating null character. If the function fails, the return value is zero.To get extended error information, call GetLastError.</returns>
+        [DllImport("user32.dll", EntryPoint = "GetClassNameW", SetLastError = true, CharSet = CharSet.Unicode)]
+        public static extern int GetClassName(IntPtr hWnd, StringBuilder lpClassName, int nMaxCount);
+
         /// <summary>   Gets window text. </summary>
         ///
         /// <param name="hWnd">         The window handle. </param>
diff --git a/Rubberduck.VBEEditor/WindowsApi/WM.cs b/Rubberduck.VBEEditor/WindowsApi/WM.cs
@@ -940,6 +940,10 @@ public enum WM : uint
         /// Private message to signal focus RD shutdown. No parameters.
         /// </summary>        
         RUBBERDUCK_SINKING = USER + 0x0D1E,
+        /// <summary>
+        /// Private message to indicate that the toolwindow is undockable and needs its special menu.
+        /// </summary>
+        RUBBERDUCK_UNDOCKABLE_CONTEXT_MENU = USER + 0x0F01
     }
 
 #if (DEBUG && (THIRSTY_DUCK || THIRSTY_DUCK_WM))
