From aa09f9cde3ae72d9ae1bab8d8a6ee467a8fe15de Mon Sep 17 00:00:00 2001 From: sina Date: Sat, 6 Jun 2026 23:45:18 +0200 Subject: [PATCH] check to unload the trace module before unloading vmm --- .../hyperkd/code/debugger/core/Debugger.c | 5 -- hyperdbg/hyperkd/code/driver/Loader.c | 62 ++++++++++------- hyperdbg/hypertrace/code/api/TraceApi.c | 9 +++ hyperdbg/libhyperdbg/code/app/libhyperdbg.cpp | 41 +++++++++-- .../commands/debugging-commands/unload.cpp | 69 +++++++++++++++---- .../commands/extension-commands/lbr.cpp | 11 +++ .../commands/extension-commands/lbrdump.cpp | 6 ++ .../code/debugger/core/debugger.cpp | 12 ++-- hyperdbg/libhyperdbg/code/export/export.cpp | 2 +- hyperdbg/libhyperdbg/header/common.h | 2 + 10 files changed, 163 insertions(+), 56 deletions(-) diff --git a/hyperdbg/hyperkd/code/debugger/core/Debugger.c b/hyperdbg/hyperkd/code/debugger/core/Debugger.c index 5b4ce2fe..81dfc63c 100644 --- a/hyperdbg/hyperkd/code/debugger/core/Debugger.c +++ b/hyperdbg/hyperkd/code/debugger/core/Debugger.c @@ -356,11 +356,6 @@ DebuggerUninitialize() ProcessorsCount = KeQueryActiveProcessorCount(0); - // - // Uninitialize the HyperTrace (if it was initialized) - // - HyperTraceUninit(); - // // Free the Pool manager // diff --git a/hyperdbg/hyperkd/code/driver/Loader.c b/hyperdbg/hyperkd/code/driver/Loader.c index d2af8125..977eb4eb 100644 --- a/hyperdbg/hyperkd/code/driver/Loader.c +++ b/hyperdbg/hyperkd/code/driver/Loader.c @@ -334,30 +334,6 @@ LoaderInitDebuggerAndVmm(PDEBUGGER_INIT_VMM_PACKET InitVmmPacket) return TRUE; } -/** - * @brief Uninitialize the VMM - * - * @return VOID - */ -VOID -LoaderUninitVmm() -{ - // - // Mark VMM as uninitialized before uninitializing it to avoid any potential reentrancy issues during the uninitialization process - // - g_VmmInitialized = FALSE; - - // - // First remove all VMM related state from the debugger - // - DebuggerUninitializeVmmOperations(); - - // - // Terminate VMM and its sub-mechanisms - // - VmFuncUninitVmm(); -} - /** * @brief Uninitialize the hyper trace module * @@ -377,6 +353,44 @@ LoaderUninitHyperTrace() HyperTraceUninit(); } +/** + * @brief Uninitialize the VMM + * + * @return VOID + */ +VOID +LoaderUninitVmm() +{ + // + // Mark VMM as uninitialized before uninitializing it to avoid any potential reentrancy issues during the uninitialization process + // + g_VmmInitialized = FALSE; + + // + // Uninitialize the HyperTrace (if it was initialized) + // + // If the trace module is currently loaded, it must be unloaded before the VMM module can be unloaded + // HyperTrace can operate both with and without the VMM module. When loaded after the VMM module, HyperTrace can make + // use of hypervisor-specific features. Otherwise, it will operate normally, but those features will not be available + // The trace module will be unloaded automatically and may be reloaded later if needed + // + // Note: The user mode should automatically request to unload the 'trace' module if it is already loaded + // however, here we also unload it just in case if this function is directly called or the user mode + // code did not unload it + // + LoaderUninitHyperTrace(); + + // + // First remove all VMM related state from the debugger + // + DebuggerUninitializeVmmOperations(); + + // + // Terminate VMM and its sub-mechanisms + // + VmFuncUninitVmm(); +} + /** * @brief Uninitialize the debugger * diff --git a/hyperdbg/hypertrace/code/api/TraceApi.c b/hyperdbg/hypertrace/code/api/TraceApi.c index 0dbf78f8..709f5c6f 100644 --- a/hyperdbg/hypertrace/code/api/TraceApi.c +++ b/hyperdbg/hypertrace/code/api/TraceApi.c @@ -61,6 +61,7 @@ HyperTraceInitCallback(HYPERTRACE_CALLBACKS * HyperTraceCallbacks, // allocate ToPA / output / overflow buffers on first use per core. // g_PtStateList = (PT_PER_CPU *)PlatformMemAllocateZeroedNonPagedPool(sizeof(PT_PER_CPU) * ProcessorsCount); + if (g_PtStateList != NULL) { UINT32 i; @@ -102,6 +103,14 @@ HyperTraceInitCallback(HYPERTRACE_CALLBACKS * HyperTraceCallbacks, VOID HyperTraceUninit() { + // + // Check if the callbacks are initialized, if not, we don't need to handle anymore + // + if (!g_HyperTraceCallbacksInitialized) + { + return; + } + // // Disable LBR tracing if it is still enabled // diff --git a/hyperdbg/libhyperdbg/code/app/libhyperdbg.cpp b/hyperdbg/libhyperdbg/code/app/libhyperdbg.cpp index 470795c6..144b15f7 100644 --- a/hyperdbg/libhyperdbg/code/app/libhyperdbg.cpp +++ b/hyperdbg/libhyperdbg/code/app/libhyperdbg.cpp @@ -394,6 +394,21 @@ HyperDbgUnloadVmm() ShowMessages("start terminating vmm...\n"); + // + // Check if HyperTrace module is loaded, if so we need to unload it before unloading VMM + // + if (g_IsHyperTraceModuleLoaded) + { + ShowMessages( + "the trace module is currently loaded and will be unloaded before the vmm module\nnote that hypertrace (trace) " + "use the hypervisor (vmm) features when it is loaded after vmm, however, hypertrace can also operate without the vmm " + "module, although hypervisor-specific features will not be available\n" + "the 'trace' module will now be unloaded automatically. You can reload it later " + "using the command 'load trace', which will load the trace module again without enabling hypervisor-dependent features\n"); + + HyperDbgUnloadHyperTrace(); + } + // // Uninitialize the user debugger if it's initialized // @@ -570,17 +585,17 @@ HyperDbgUnloadAllModules() INT RetVal = 0; // - // Unload VMM module if loaded + // Unload HyperTrace module if loaded // - if (g_IsVmmModuleLoaded && HyperDbgUnloadVmm() != 0) + if (g_IsHyperTraceModuleLoaded && HyperDbgUnloadHyperTrace() != 0) { return 1; } // - // Unload HyperTrace module if loaded + // Unload VMM module if loaded // - if (g_IsHyperTraceModuleLoaded && HyperDbgUnloadHyperTrace() != 0) + if (g_IsVmmModuleLoaded && HyperDbgUnloadVmm() != 0) { return 1; } @@ -689,6 +704,20 @@ HyperDbgLoadVmmModule() return 0; } + // + // Check if the HyperTrace module is loaded or not + // + if (g_IsHyperTraceModuleLoaded) + { + ShowMessages( + "err, the trace module is currently loaded and should be unloaded before loading the vmm module\n" + "Note that HyperTrace (trace) uses the hypervisor (vmm) features when it is loaded after the vmm module, " + "however, HyperTrace can also operate without the vmm module, although hypervisor-specific features will not be available\n" + "to solve this problem, first unload the trace module using the 'unload trace' command, next, load " + "the vmm module and then load the trace module again. This way, the trace module is reloaded with hypervisor APIs\n"); + return 1; + } + // // Enable Debug privilege to the current token // @@ -806,7 +835,7 @@ HyperDbgLoadHyperTraceModule() // if (HyperDbgInitHyperTraceModule() == 1) { - ShowMessages("err, initializing HyperTrace module\n"); + ShowMessages("err, initializing hypertrace module\n"); return 1; } @@ -816,7 +845,7 @@ HyperDbgLoadHyperTraceModule() // g_IsHyperTraceModuleLoaded = TRUE; - ShowMessages("hypertrace module is running...\n"); + ShowMessages("hypertrace (trace) module is running...\n"); return 0; } diff --git a/hyperdbg/libhyperdbg/code/debugger/commands/debugging-commands/unload.cpp b/hyperdbg/libhyperdbg/code/debugger/commands/debugging-commands/unload.cpp index 2379cf9f..2302a01d 100644 --- a/hyperdbg/libhyperdbg/code/debugger/commands/debugging-commands/unload.cpp +++ b/hyperdbg/libhyperdbg/code/debugger/commands/debugging-commands/unload.cpp @@ -17,6 +17,7 @@ extern BOOLEAN g_IsConnectedToHyperDbgLocally; extern BOOLEAN g_IsKdModuleLoaded; extern BOOLEAN g_IsVmmModuleLoaded; +extern BOOLEAN g_IsHyperTraceModuleLoaded; extern BOOLEAN g_IsSerialConnectedToRemoteDebuggee; extern BOOLEAN g_IsSerialConnectedToRemoteDebugger; @@ -39,6 +40,34 @@ CommandUnloadHelp() ShowMessages("\t\te.g : unload remove vmm\n"); } +/** + * @brief check the environment for unload command + * + * @return BOOLEAN + */ +BOOLEAN +CommandUnloadCheckEnvironment() +{ + if (!g_IsConnectedToHyperDbgLocally) + { + ShowMessages("you're not connected to any instance of HyperDbg, did you " + "use '.connect'? \n"); + return FALSE; + } + + // + // Check to avoid using this command in debugger-mode + // + if (g_IsSerialConnectedToRemoteDebuggee || g_IsSerialConnectedToRemoteDebugger) + { + ShowMessages("you're connected to a an instance of HyperDbg, please use " + "'.debug close' command\n"); + return FALSE; + } + + return TRUE; +} + /** * @brief unload command handler * @@ -64,20 +93,11 @@ CommandUnload(vector CommandTokens, string Command) if (CommandTokens.size() == 2 && (CompareLowerCaseStrings(CommandTokens.at(1), "vmm") || CompareLowerCaseStrings(CommandTokens.at(1), "vm"))) { - if (!g_IsConnectedToHyperDbgLocally) - { - ShowMessages("you're not connected to any instance of HyperDbg, did you " - "use '.connect'? \n"); - return; - } - // - // Check to avoid using this command in debugger-mode + // Check the environment // - if (g_IsSerialConnectedToRemoteDebuggee || g_IsSerialConnectedToRemoteDebugger) + if (!CommandUnloadCheckEnvironment()) { - ShowMessages("you're connected to a an instance of HyperDbg, please use " - "'.debug close' command\n"); return; } @@ -91,12 +111,25 @@ CommandUnload(vector CommandTokens, string Command) { ShowMessages("the vmm module is not loadedd\n"); } - + } + else if (CommandTokens.size() == 2 && + (CompareLowerCaseStrings(CommandTokens.at(1), "trace") || CompareLowerCaseStrings(CommandTokens.at(1), "hypertrace"))) + { // - // Check to remove the driver + // Check the environment // - if (CompareLowerCaseStrings(CommandTokens.at(1), "remove")) + if (!CommandUnloadCheckEnvironment()) { + return; + } + + if (g_IsHyperTraceModuleLoaded) + { + HyperDbgUnloadHyperTrace(); + } + else + { + ShowMessages("the trace (hypertrace) module is not loadedd\n"); } } else if (CommandTokens.size() == 3 && CompareLowerCaseStrings(CommandTokens.at(1), "remove") && @@ -104,6 +137,14 @@ CommandUnload(vector CommandTokens, string Command) CompareLowerCaseStrings(CommandTokens.at(2), "vmm") || CompareLowerCaseStrings(CommandTokens.at(2), "vm"))) { + // + // Check the environment + // + if (!CommandUnloadCheckEnvironment()) + { + return; + } + // // Unload all modules before removing the driver // diff --git a/hyperdbg/libhyperdbg/code/debugger/commands/extension-commands/lbr.cpp b/hyperdbg/libhyperdbg/code/debugger/commands/extension-commands/lbr.cpp index fde287ee..f716f752 100644 --- a/hyperdbg/libhyperdbg/code/debugger/commands/extension-commands/lbr.cpp +++ b/hyperdbg/libhyperdbg/code/debugger/commands/extension-commands/lbr.cpp @@ -11,6 +11,12 @@ */ #include "pch.h" +// +// Global Variables +// +extern HANDLE g_DeviceHandle; +extern BOOLEAN g_IsHyperTraceModuleLoaded; + /** * @brief help of the !lbr command * @@ -359,6 +365,11 @@ CommandLbr(vector CommandTokens, string Command) return; } + // + // Check if the HyperTrace module is loaded, as it is required for LBR operations + // + AssertShowMessageReturnStmt(g_IsHyperTraceModuleLoaded, ASSERT_MESSAGE_HYPERTRACE_NOT_LOADED, AssertReturn); + // // Send the LBR operation request // diff --git a/hyperdbg/libhyperdbg/code/debugger/commands/extension-commands/lbrdump.cpp b/hyperdbg/libhyperdbg/code/debugger/commands/extension-commands/lbrdump.cpp index 64e8ee8d..d19db3b9 100644 --- a/hyperdbg/libhyperdbg/code/debugger/commands/extension-commands/lbrdump.cpp +++ b/hyperdbg/libhyperdbg/code/debugger/commands/extension-commands/lbrdump.cpp @@ -15,6 +15,7 @@ // Global Variables // extern BOOLEAN g_IsSerialConnectedToRemoteDebuggee; +extern BOOLEAN g_IsHyperTraceModuleLoaded; /** * @brief Send LBR dump requests @@ -249,6 +250,11 @@ CommandLbrdump(vector CommandTokens, string Command) return; } + // + // Check if the HyperTrace module is loaded, as it is required for LBR operations + // + AssertShowMessageReturnStmt(g_IsHyperTraceModuleLoaded, ASSERT_MESSAGE_HYPERTRACE_NOT_LOADED, AssertReturn); + if (CommandTokens.size() == 3) { if (CompareLowerCaseStrings(CommandTokens.at(1), "core")) diff --git a/hyperdbg/libhyperdbg/code/debugger/core/debugger.cpp b/hyperdbg/libhyperdbg/code/debugger/core/debugger.cpp index 72b1a3ff..5c1d6b9d 100644 --- a/hyperdbg/libhyperdbg/code/debugger/core/debugger.cpp +++ b/hyperdbg/libhyperdbg/code/debugger/core/debugger.cpp @@ -570,13 +570,13 @@ ShowErrorMessage(UINT32 Error) break; case DEBUGGER_ERROR_HYPERTRACE_NOT_INITIALIZED: - ShowMessages("err, the HyperTrace module is not loaded and initialized, " - "use the 'load trace' command to load the HyperTrace module (%x)\n", + ShowMessages("err, the hypertrace module is not loaded and initialized, " + "use the 'load trace' command to load the hypertrace module (%x)\n", Error); break; case DEBUGGER_ERROR_INVALID_HYPERTRACE_OPERATION_TYPE: - ShowMessages("err, invalid HyperTrace operation type is specified (%x)\n", + ShowMessages("err, invalid hypertrace operation type is specified (%x)\n", Error); break; @@ -617,9 +617,9 @@ ShowErrorMessage(UINT32 Error) break; case DEBUGGER_ERROR_VMM_CANNOT_BE_INITIALIZED_IF_HYPERTRACE_IS_LOADED: - ShowMessages("err, HyperTrace is already loaded, please unload HyperTrace module using the " - "'unload' command and then load the 'VMM' module. Then you can load HyperTrace " - "after loading the VMM as it is because HyperTrace behaves differently to sync " + ShowMessages("err, hypertrace is already loaded, please unload hypertrace module using the " + "'unload' command and then load the 'VMM' module. Then you can load hypertrace " + "after loading the VMM as it is because hypertrace behaves differently to sync " "with VMM modules; it needs to have this notion that it is running within the " "hypervisor, so that is why it needs to be initialized again if the VMM module " "is loaded (%x)\n", diff --git a/hyperdbg/libhyperdbg/code/export/export.cpp b/hyperdbg/libhyperdbg/code/export/export.cpp index 85d5f3e8..feefff0b 100644 --- a/hyperdbg/libhyperdbg/code/export/export.cpp +++ b/hyperdbg/libhyperdbg/code/export/export.cpp @@ -163,7 +163,7 @@ hyperdbg_u_load_kd_module() } /** - * @brief Load the HyperTrace module + * @brief Load the hypertrace module * * @return INT Returns 0 if it was successful and 1 if it was failed */ diff --git a/hyperdbg/libhyperdbg/header/common.h b/hyperdbg/libhyperdbg/header/common.h index eda44a98..db234588 100644 --- a/hyperdbg/libhyperdbg/header/common.h +++ b/hyperdbg/libhyperdbg/header/common.h @@ -26,6 +26,8 @@ #define ASSERT_MESSAGE_DRIVER_NOT_LOADED "handle of the driver not found, probably the driver is not loaded. Did you use 'load' command?\n" +#define ASSERT_MESSAGE_HYPERTRACE_NOT_LOADED "the trace (hypertrace) module is not loaded. Did you use 'load trace' command?\n" + #define ASSERT_MESSAGE_BUILD_SIGNATURE_DOESNT_MATCH "the handshaking process was successful; however, there is a mismatch between " \ "the version/build of the debuggee and the debugger. please use the same " \ "version/build for both the debuggee and debugger\n"