C# Draw Custom Button on Existing Window’s TitleBar

The following will work in XP, I have no Vista machine handy to test it, but I think your issues are steming from an incorrect hWnd somehow. Anyway, on with the poorly commented code.

// The state of our little button
ButtonState _buttState = ButtonState.Normal;
Rectangle _buttPosition = new Rectangle();

[DllImport("user32.dll")]
private static extern IntPtr GetWindowDC(IntPtr hWnd);
[DllImport("user32.dll")]
private static extern int GetWindowRect(IntPtr hWnd, 
                                        ref Rectangle lpRect);
[DllImport("user32.dll")]
private static extern int ReleaseDC(IntPtr hWnd, IntPtr hDC);
protected override void WndProc(ref Message m)
{
    int x, y;
    Rectangle windowRect = new Rectangle();
    GetWindowRect(m.HWnd, ref windowRect);

    switch (m.Msg)
    {
        // WM_NCPAINT
        case 0x85:
        // WM_PAINT
        case 0x0A:
            base.WndProc(ref m);

            DrawButton(m.HWnd);

            m.Result = IntPtr.Zero;

            break;

        // WM_ACTIVATE
        case 0x86:
            base.WndProc(ref m);
            DrawButton(m.HWnd);

            break;

        // WM_NCMOUSEMOVE
        case 0xA0:
            // Extract the least significant 16 bits
            x = ((int)m.LParam << 16) >> 16;
            // Extract the most significant 16 bits
            y = (int)m.LParam >> 16;

            x -= windowRect.Left;
            y -= windowRect.Top;

            base.WndProc(ref m);

            if (!_buttPosition.Contains(new Point(x, y)) && 
                _buttState == ButtonState.Pushed)
            {
                _buttState = ButtonState.Normal;
                DrawButton(m.HWnd);
            }

            break;

        // WM_NCLBUTTONDOWN
        case 0xA1:
            // Extract the least significant 16 bits
            x = ((int)m.LParam << 16) >> 16;
            // Extract the most significant 16 bits
            y = (int)m.LParam >> 16;

            x -= windowRect.Left;
            y -= windowRect.Top;

            if (_buttPosition.Contains(new Point(x, y)))
            {
                _buttState = ButtonState.Pushed;
                DrawButton(m.HWnd);
            }
            else
                base.WndProc(ref m);

            break;

        // WM_NCLBUTTONUP
        case 0xA2:
            // Extract the least significant 16 bits
            x = ((int)m.LParam << 16) >> 16;
            // Extract the most significant 16 bits
            y = (int)m.LParam >> 16;

            x -= windowRect.Left;
            y -= windowRect.Top;

            if (_buttPosition.Contains(new Point(x, y)) &&
                _buttState == ButtonState.Pushed)
            {
                _buttState = ButtonState.Normal;
                // [[TODO]]: Fire a click event for your button 
                //           however you want to do it.
                DrawButton(m.HWnd);
            }
            else
                base.WndProc(ref m);

            break;

        // WM_NCHITTEST
        case 0x84:
            // Extract the least significant 16 bits
            x = ((int)m.LParam << 16) >> 16;
            // Extract the most significant 16 bits
            y = (int)m.LParam >> 16;

            x -= windowRect.Left;
            y -= windowRect.Top;

            if (_buttPosition.Contains(new Point(x, y)))
                m.Result = (IntPtr)18; // HTBORDER
            else
                base.WndProc(ref m);

            break;

        default:
            base.WndProc(ref m);
            break;
    }
}

private void DrawButton(IntPtr hwnd)
{
    IntPtr hDC = GetWindowDC(hwnd);
    int x, y;

    using (Graphics g = Graphics.FromHdc(hDC))
    {
        // Work out size and positioning
        int CaptionHeight = Bounds.Height - ClientRectangle.Height;
        Size ButtonSize = SystemInformation.CaptionButtonSize;
        x = Bounds.Width - 4 * ButtonSize.Width;
        y = (CaptionHeight - ButtonSize.Height) / 2;
        _buttPosition.Location = new Point(x, y);

        // Work out color
        Brush color;
        if (_buttState == ButtonState.Pushed)
            color = Brushes.LightGreen;
        else
            color = Brushes.Red;

        // Draw our "button"
        g.FillRectangle(color, x, y, ButtonSize.Width, ButtonSize.Height);
    }

    ReleaseDC(hwnd, hDC);
}

private void Form1_Load(object sender, EventArgs e)
{
    _buttPosition.Size = SystemInformation.CaptionButtonSize;
}

WPF doesn't have a built-in property to hide the title bar's Close button, but you can do it with a few lines of P/Invoke.

First, add these declarations to your Window class:

private const int GWL_STYLE = -16;
private const int WS_SYSMENU = 0x80000;
[DllImport("user32.dll", SetLastError = true)]
private static extern int GetWindowLong(IntPtr hWnd, int nIndex);
[DllImport("user32.dll")]
private static extern int SetWindowLong(IntPtr hWnd, int nIndex, int dwNewLong);

Then put this code in the Window's Loaded event:

var hwnd = new WindowInteropHelper(this).Handle;
SetWindowLong(hwnd, GWL_STYLE, GetWindowLong(hwnd, GWL_STYLE) & ~WS_SYSMENU);

And there you go: no more Close button. You also won't have a window icon on the left side of the title bar, which means no system menu, even when you right-click the title bar - they all go together.

Important note: all this does is hide the button. The user can still close the window! If the user presses Alt+F4, or closes the app via the taskbar, the window will still close.

If you don't want to allow the window to close before the background thread is done, then you could also override OnClosing and set Cancel to true, as Gabe suggested.