diff --git a/Utilities/Config.cpp b/Utilities/Config.cpp
index 0c2a1fa2aa..fabc9755df 100644
--- a/Utilities/Config.cpp
+++ b/Utilities/Config.cpp
@@ -553,19 +553,12 @@ void cfg::node::from_default()
 	}
 }
 
-void cfg::_bool::from_default()
+void cfg::node::restore_defaults()
 {
-	m_value = def;
-}
-
-void cfg::string::from_default()
-{
-	m_value = def;
-}
-
-void cfg::set_entry::from_default()
-{
-	m_set = {};
+	for (auto& node : m_nodes)
+	{
+		node->restore_defaults();
+	}
 }
 
 std::string cfg::map_entry::get_value(std::string_view key)
diff --git a/Utilities/Config.h b/Utilities/Config.h
index bd05c25220..50ac07865a 100644
--- a/Utilities/Config.h
+++ b/Utilities/Config.h
@@ -90,6 +90,9 @@ namespace cfg
 		// Reset defaults
 		virtual void from_default() = 0;
 
+		// Restore default members
+		virtual void restore_defaults() = 0;
+
 		// Convert to string (optional)
 		virtual std::string to_string() const
 		{
@@ -151,11 +154,15 @@ namespace cfg
 
 		// Set default values
 		void from_default() override;
+
+		// Restore default members
+		void restore_defaults() override;
 	};
 
 	class _bool final : public _base
 	{
 		atomic_t<bool> m_value;
+		bool original_def;
 
 	public:
 		bool def;
@@ -163,6 +170,7 @@ namespace cfg
 		_bool(node* owner, std::string name, bool def = false, bool dynamic = false)
 			: _base(type::_bool, owner, std::move(name), dynamic)
 			, m_value(def)
+			, original_def(def)
 			, def(def)
 		{
 		}
@@ -177,7 +185,15 @@ namespace cfg
 			return m_value;
 		}
 
-		void from_default() override;
+		void from_default() override
+		{
+			m_value = def;
+		}
+
+		void restore_defaults() override
+		{
+			def = original_def;
+		}
 
 		std::string to_string() const override
 		{
@@ -220,14 +236,16 @@ namespace cfg
 	class _enum : public _base
 	{
 		atomic_t<T> m_value;
+		T original_def;
 
 	public:
-		const T def;
+		T def;
 
-		_enum(node* owner, const std::string& name, T value = {}, bool dynamic = false)
+		_enum(node* owner, const std::string& name, T def = {}, bool dynamic = false)
 			: _base(type::_enum, owner, name, dynamic)
-			, m_value(value)
-			, def(value)
+			, m_value(def)
+			, original_def(def)
+			, def(def)
 		{
 		}
 
@@ -256,6 +274,11 @@ namespace cfg
 			m_value = def;
 		}
 
+		void restore_defaults() override
+		{
+			def = original_def;
+		}
+
 		std::string to_string() const override
 		{
 			std::string result;
@@ -300,6 +323,7 @@ namespace cfg
 		using int_type = std::conditional_t<Min >= s32{smin} && Max <= s32{smax}, s32, s64>;
 
 		atomic_t<int_type> m_value;
+		int_type original_def;
 
 	public:
 		int_type def;
@@ -311,6 +335,7 @@ namespace cfg
 		_int(node* owner, const std::string& name, int_type def = std::min<int_type>(Max, std::max<int_type>(Min, 0)), bool dynamic = false)
 			: _base(type::_int, owner, name, dynamic)
 			, m_value(def)
+			, original_def(def)
 			, def(def)
 		{
 		}
@@ -330,6 +355,11 @@ namespace cfg
 			m_value = def;
 		}
 
+		void restore_defaults() override
+		{
+			def = original_def;
+		}
+
 		std::string to_string() const override
 		{
 			return std::to_string(m_value);
@@ -372,6 +402,7 @@ namespace cfg
 
 		using float_type = f64;
 		atomic_t<float_type> m_value;
+		float_type original_def;
 
 	public:
 		float_type def;
@@ -383,6 +414,7 @@ namespace cfg
 		_float(node* owner, const std::string& name, float_type def = std::min<float_type>(Max, std::max<float_type>(Min, 0)), bool dynamic = false)
 			: _base(type::_int, owner, name, dynamic)
 			, m_value(def)
+			, original_def(def)
 			, def(def)
 		{
 		}
@@ -402,6 +434,11 @@ namespace cfg
 			m_value = def;
 		}
 
+		void restore_defaults() override
+		{
+			def = original_def;
+		}
+
 		std::string to_string() const override
 		{
 			std::string result;
@@ -464,6 +501,7 @@ namespace cfg
 		using int_type = std::conditional_t<Max <= u32{umax}, u32, u64>;
 
 		atomic_t<int_type> m_value;
+		int_type original_def;
 
 	public:
 		int_type def;
@@ -475,6 +513,7 @@ namespace cfg
 		uint(node* owner, const std::string& name, int_type def = std::max<int_type>(Min, 0), bool dynamic = false)
 			: _base(type::uint, owner, name, dynamic)
 			, m_value(def)
+			, original_def(def)
 			, def(def)
 		{
 		}
@@ -494,6 +533,11 @@ namespace cfg
 			m_value = def;
 		}
 
+		void restore_defaults() override
+		{
+			def = original_def;
+		}
+
 		std::string to_string() const override
 		{
 			return std::to_string(m_value);
@@ -538,6 +582,7 @@ namespace cfg
 	class string : public _base
 	{
 		atomic_ptr<std::string> m_value;
+		std::string original_def;
 
 	public:
 		std::string def;
@@ -545,6 +590,7 @@ namespace cfg
 		string(node* owner, std::string name, std::string def = {}, bool dynamic = false)
 			: _base(type::string, owner, std::move(name), dynamic)
 			, m_value(def)
+			, original_def(def)
 			, def(std::move(def))
 		{
 		}
@@ -554,7 +600,15 @@ namespace cfg
 			return *m_value.load().get();
 		}
 
-		void from_default() override;
+		void from_default() override
+		{
+			m_value = def;
+		}
+
+		void restore_defaults() override
+		{
+			def = original_def;
+		}
 
 		std::string to_string() const override
 		{
@@ -595,7 +649,14 @@ namespace cfg
 			m_set = std::move(set);
 		}
 
-		void from_default() override;
+		void from_default() override
+		{
+			m_set = {};
+		}
+
+		void restore_defaults() override
+		{
+		}
 
 		std::vector<std::string> to_list() const override
 		{
@@ -636,6 +697,10 @@ namespace cfg
 		void erase(std::string_view key);
 
 		void from_default() override;
+
+		void restore_defaults() override
+		{
+		}
 	};
 
 	class node_map_entry final : public map_entry
@@ -665,6 +730,10 @@ namespace cfg
 		void set_map(map_of_type<logs::level>&& map);
 
 		void from_default() override;
+
+		void restore_defaults() override
+		{
+		}
 	};
 
 	struct device_info
@@ -702,5 +771,9 @@ namespace cfg
 		void set_map(map_of_type<device_info>&& map);
 
 		void from_default() override;
+
+		void restore_defaults() override
+		{
+		}
 	};
 }
diff --git a/rpcs3/Emu/Io/PadHandler.cpp b/rpcs3/Emu/Io/PadHandler.cpp
index c4c636cfa7..def089f9dd 100644
--- a/rpcs3/Emu/Io/PadHandler.cpp
+++ b/rpcs3/Emu/Io/PadHandler.cpp
@@ -200,6 +200,10 @@ void PadHandlerBase::init_configs()
 {
 	for (u32 i = 0; i < MAX_GAMEPADS; i++)
 	{
+		// We need to restore the original defaults first.
+		m_pad_configs[i].restore_defaults();
+
+		// Set and apply actual defaults depending on pad handler
 		init_config(&m_pad_configs[i]);
 	}
 }
diff --git a/rpcs3/Input/gui_pad_thread.cpp b/rpcs3/Input/gui_pad_thread.cpp
index 3aa81efb29..e3401c94a9 100644
--- a/rpcs3/Input/gui_pad_thread.cpp
+++ b/rpcs3/Input/gui_pad_thread.cpp
@@ -234,14 +234,18 @@ std::shared_ptr<PadHandlerBase> gui_pad_thread::GetHandler(pad_handler type)
 
 void gui_pad_thread::InitPadConfig(cfg_pad& cfg, pad_handler type, std::shared_ptr<PadHandlerBase>& handler)
 {
+	// We need to restore the original defaults first.
+	cfg.restore_defaults();
+
 	if (!handler)
 	{
 		handler = GetHandler(type);
+	}
 
-		if (handler)
-		{
-			handler->init_config(&cfg);
-		}
+	if (handler)
+	{
+		// Set and apply actual defaults depending on pad handler
+		handler->init_config(&cfg);
 	}
 }
 
diff --git a/rpcs3/Input/pad_thread.cpp b/rpcs3/Input/pad_thread.cpp
index 5073e19b94..afc5a73fce 100644
--- a/rpcs3/Input/pad_thread.cpp
+++ b/rpcs3/Input/pad_thread.cpp
@@ -883,12 +883,17 @@ std::shared_ptr<PadHandlerBase> pad_thread::GetHandler(pad_handler type)
 
 void pad_thread::InitPadConfig(cfg_pad& cfg, pad_handler type, std::shared_ptr<PadHandlerBase>& handler)
 {
+	// We need to restore the original defaults first.
+	cfg.restore_defaults();
+
 	if (!handler)
 	{
 		handler = GetHandler(type);
 	}
 
 	ensure(!!handler);
+
+	// Set and apply actual defaults depending on pad handler
 	handler->init_config(&cfg);
 }