Qt: Prevent entering/exiting fullscreen while the VM is locked

pcsx2-qt/MainWindow.cpp

@@ -3306,23 +3306,30 @@ void MainWindow::rescanFile(const std::string& path)
 
 MainWindow::VMLock::VMLock(QWidget* dialog_parent, bool was_paused, bool was_fullscreen)
 	: m_dialog_parent(dialog_parent)
+	, m_has_lock(true)
 	, m_was_paused(was_paused)
 	, m_was_fullscreen(was_fullscreen)
 {
+	QtHost::LockVMWithDialog();
 }
 
 MainWindow::VMLock::VMLock(VMLock&& lock)
 	: m_dialog_parent(lock.m_dialog_parent)
+	, m_has_lock(lock.m_has_lock)
 	, m_was_paused(lock.m_was_paused)
 	, m_was_fullscreen(lock.m_was_fullscreen)
 {
 	lock.m_dialog_parent = nullptr;
+	lock.m_has_lock = false;
 	lock.m_was_paused = true;
 	lock.m_was_fullscreen = false;
 }
 
 MainWindow::VMLock::~VMLock()
 {
+	if (m_has_lock)
+		QtHost::UnlockVMWithDialog();
+
 	if (m_was_fullscreen)
 	{
 		g_main_window->m_is_temporarily_windowed = false;
@@ -3335,10 +3342,15 @@ MainWindow::VMLock::~VMLock()
 
 MainWindow::VMLock& MainWindow::VMLock::operator=(VMLock&& lock)
 {
+	if (m_has_lock)
+		QtHost::UnlockVMWithDialog();
+
+	m_has_lock = lock.m_has_lock;
 	m_dialog_parent = lock.m_dialog_parent;
 	m_was_paused = lock.m_was_paused;
 	m_was_fullscreen = lock.m_was_fullscreen;
 
+	lock.m_has_lock = false;
 	lock.m_dialog_parent = nullptr;
 	lock.m_was_paused = true;
 	lock.m_was_fullscreen = false;

pcsx2-qt/MainWindow.h

@@ -69,6 +69,7 @@ public:
 		friend MainWindow;
 
 		QWidget* m_dialog_parent;
+		bool m_has_lock;
 		bool m_was_paused;
 		bool m_was_fullscreen;
 	};

pcsx2-qt/QtHost.cpp

@@ -94,6 +94,7 @@ static bool s_test_config_and_exit = false;
 static bool s_run_setup_wizard = false;
 static bool s_cleanup_after_update = false;
 static bool s_boot_and_debug = false;
+static std::atomic_int s_vm_locked_with_dialog = 0;
 
 //////////////////////////////////////////////////////////////////////////
 // CPU Thread
@@ -495,6 +496,11 @@ void EmuThread::setFullscreen(bool fullscreen, bool allow_render_to_main)
 		return;
 	}
 
+	// HACK: Prevent entering/exiting fullscreen mode when a dialog is shown, so
+	// that we don't destroy the dialog while inside its exec function.
+	if (s_vm_locked_with_dialog > 0)
+		return;
+
 	if (!MTGS::IsOpen() || m_is_fullscreen == fullscreen)
 		return;
 
@@ -1713,6 +1719,16 @@ void Host::SetMouseMode(bool relative_mode, bool hide_cursor)
 	emit g_emu_thread->onMouseModeRequested(relative_mode, hide_cursor);
 }
 
+void QtHost::LockVMWithDialog()
+{
+	s_vm_locked_with_dialog++;
+}
+
+void QtHost::UnlockVMWithDialog()
+{
+	s_vm_locked_with_dialog--;
+}
+
 namespace
 {
 	class QtHostProgressCallback final : public BaseProgressCallback

pcsx2-qt/QtHost.h

@@ -290,4 +290,10 @@ namespace QtHost
 
 	/// Compare strings in the locale of the current UI language
 	int LocaleSensitiveCompare(QStringView lhs, QStringView rhs);
+
+	/// Determines whether or not requests to enter/exit fullscreen mode should
+	/// be ignored. This is a hack so that we don't destroy a dialog box while
+	/// inside its exec function, which would cause a crash.
+	void LockVMWithDialog();
+	void UnlockVMWithDialog();
 } // namespace QtHost