Frame (MenuBar) corrupt inside Windows app

[Grinchman]

Hello.
I build Qt application as DLL. That dll is used in Windows app (3ds max 2012). To make my Qt app as child window for 3ds max I do:

void MainWindow::setParentWindowHandle(fVoid *parent)
{
	if(parent) 
	{	
		SetParent(winId(), (HWND)parent);	
 
		QEvent e(QEvent::EmbeddingControl);
		QApplication::sendEvent(this, &e);
	}
}

void MainWindow::setParentWindowHandle(fVoid *parent)
{
	if(parent) 
	{	
		SetParent(winId(), (HWND)parent);	

		QEvent e(QEvent::EmbeddingControl);
		QApplication::sendEvent(this, &e);
	}
}

To copy to clipboard, switch view to plain text mode 

But, when my window is showed, border frame(menu bar with close, maximize buttons) isn't painted, it's corrupt (or something like this, see the image below).
Frameless.jpg

But close, maximize buttons is there! I may click them.

Don't know why it isn't painted...

P.S.: When my app isn't child of 3ds max then all frames, buttons are showed fine

[ChrisW67]

How is your Qt event loop being called?

[Grinchman]

Sorry for missing for long time.
I've fixed the code but just partialy. To make right processing of events I used code from QtWinMigrate project. So, what I've done?
First I added to class MyApplication such method:

bool MyApplication::pluginInstance(Qt::HANDLE plugin)
{
	if (qApp)
		return FALSE;
 
	QT_WA({
		hhook = SetWindowsHookExW(WH_GETMESSAGE, QtFilterProc, 0, GetCurrentThreadId());
	}, {
		hhook = SetWindowsHookExA(WH_GETMESSAGE, QtFilterProc, 0, GetCurrentThreadId());
	});
 
	if (plugin) {
		char filename[256];
		if (GetModuleFileNameA((HINSTANCE)plugin, filename, 255))
			LoadLibraryA(filename);
	}
 
	return TRUE;
}

bool MyApplication::pluginInstance(Qt::HANDLE plugin)
{
	if (qApp)
		return FALSE;

	QT_WA({
		hhook = SetWindowsHookExW(WH_GETMESSAGE, QtFilterProc, 0, GetCurrentThreadId());
	}, {
		hhook = SetWindowsHookExA(WH_GETMESSAGE, QtFilterProc, 0, GetCurrentThreadId());
	});

	if (plugin) {
		char filename[256];
		if (GetModuleFileNameA((HINSTANCE)plugin, filename, 255))
			LoadLibraryA(filename);
	}

	return TRUE;
}

To copy to clipboard, switch view to plain text mode 

It's called from here:

BOOL APIENTRY DllMain(HINSTANCE hinstDLL, DWORD ul_reason_for_call, LPVOID lpReserved)
{
	BOOL ret = TRUE;	
	hInstance = hinstDLL;				// Hang on to this DLL's instance handle.
 
	switch(ul_reason_for_call) 
	{
	case DLL_PROCESS_ATTACH:
		{					
			VFBApplication::pluginInstance(hinstDLL);
                }
...
}

BOOL APIENTRY DllMain(HINSTANCE hinstDLL, DWORD ul_reason_for_call, LPVOID lpReserved)
{
	BOOL ret = TRUE;	
	hInstance = hinstDLL;				// Hang on to this DLL's instance handle.

	switch(ul_reason_for_call) 
	{
	case DLL_PROCESS_ATTACH:
		{					
			VFBApplication::pluginInstance(hinstDLL);
                }
...
}

To copy to clipboard, switch view to plain text mode 

Second, at MyApplication constructor I called:

QAbstractEventDispatcher::instance()->setEventFilter(qmfc_eventFilter);

QAbstractEventDispatcher::instance()->setEventFilter(qmfc_eventFilter);

To copy to clipboard, switch view to plain text mode 

qmfc_evenFilter looks like this (it's static standalone function):

static bool qmfc_eventFilter(void *message)
{
	long result = 0;
	return static_cast<MyApplication*>(qApp)->winEventFilter((MSG*)message, &result);
}

static bool qmfc_eventFilter(void *message)
{
	long result = 0;
	return static_cast<MyApplication*>(qApp)->winEventFilter((MSG*)message, &result);
}

To copy to clipboard, switch view to plain text mode 

MyApplication::winEventFilter:

bool MyApplication::winEventFilter ( MSG * msg, long * result )
{	
	static bool recursion = false;
	if (recursion)
		return false;
 
	recursion = true;
 
	QWidget *widget = QWidget::find(msg->hwnd);
	HWND toplevel = 0;
	if (widget) {
		HWND parent = widget->winId();
		while(parent) {
			toplevel = parent;
			parent = GetParent(parent);
		}
		HMENU menu = toplevel ? GetMenu(toplevel) : 0;
		if (menu && GetFocus() == msg->hwnd) {
			if (msg->message == WM_SYSKEYUP && msg->wParam == VK_MENU) {
				// activate menubar on Alt-up and move focus away
				SetFocus(toplevel);
				SendMessage(toplevel, msg->message, msg->wParam, msg->lParam);
				widget->setFocus();
				recursion = false;
				QApplication::processEvents();
				return TRUE;
			} else if (msg->message == WM_SYSKEYDOWN && msg->wParam != VK_MENU) {
				SendMessage(toplevel, msg->message, msg->wParam, msg->lParam);
				SendMessage(toplevel, WM_SYSKEYUP, VK_MENU, msg->lParam);
				recursion = false;
				QApplication::processEvents();
				return TRUE;
			}
		}
 
		QApplication::processEvents();
	}
 
 	recursion = false;	
	return QApplication::winEventFilter(msg, result);
}

bool MyApplication::winEventFilter ( MSG * msg, long * result )
{	
	static bool recursion = false;
	if (recursion)
		return false;

	recursion = true;

	QWidget *widget = QWidget::find(msg->hwnd);
	HWND toplevel = 0;
	if (widget) {
		HWND parent = widget->winId();
		while(parent) {
			toplevel = parent;
			parent = GetParent(parent);
		}
		HMENU menu = toplevel ? GetMenu(toplevel) : 0;
		if (menu && GetFocus() == msg->hwnd) {
			if (msg->message == WM_SYSKEYUP && msg->wParam == VK_MENU) {
				// activate menubar on Alt-up and move focus away
				SetFocus(toplevel);
				SendMessage(toplevel, msg->message, msg->wParam, msg->lParam);
				widget->setFocus();
				recursion = false;
				QApplication::processEvents();
				return TRUE;
			} else if (msg->message == WM_SYSKEYDOWN && msg->wParam != VK_MENU) {
				SendMessage(toplevel, msg->message, msg->wParam, msg->lParam);
				SendMessage(toplevel, WM_SYSKEYUP, VK_MENU, msg->lParam);
				recursion = false;
				QApplication::processEvents();
				return TRUE;
			}
		}

		QApplication::processEvents();
	}
 
 	recursion = false;	
	return QApplication::winEventFilter(msg, result);
}

To copy to clipboard, switch view to plain text mode 

In additional MyApplication has such method:

bool MyApplication::event(QEvent * e)
{
	QApplication::processEvents();
	return QApplication::event(e);
}

bool MyApplication::event(QEvent * e)
{
	QApplication::processEvents();
	return QApplication::event(e);
}

To copy to clipboard, switch view to plain text mode 

As a result I have almost normal behavior of MyApplication. At start point it's look like this:
NormalWindow.jpg

But several objects still corrupt my window:
CorruptedOneWindow.jpg
CorruptedSecondWindow.jpg