Question Analysis
A Windows host shows registry modifications in its boot configuration store enabling test signing mode. Investigate which process made this change and identify the exact command it ran to turn on test signing.
bb6b51e1-ab92-45b5-aeea-e410d06405f8
Question ID
5/23
Models Correct
1
Valid Answer
22%
Success Rate
Reference Answer
Human-validated correct answer for this question
bcdedit /set testsigning on
Model Performance Comparison
See how different AI models performed on this question
Model | Answer | KQL Query | Status | Attempts | Time | Cost |
---|---|---|---|---|---|---|
gemini-2.5-flash-preview-04-17 | bcdedit /set testsigning on | DeviceProcessEvents
| where ActionType == "ProcessCreated"
| where ProcessCommandLine contains "bcdedit" and ProcessCommandLine contains "testsigning" and ProcessCommandLine contains "on"
| project Timestamp, DeviceName, ProcessCommandLine, InitiatingProcessFileName, InitiatingProcessCommandLine
| order by Timestamp desc | Correct | 4 | 48.39s | 0.0244 |
gpt-35-turbo | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0118 |
gpt-4-turbo-2024-04-09 | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.2336 |
gpt-4.1 | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0598 |
gpt-4.1-finetuned | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0660 |
gpt-4.1-mini | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0097 |
gpt-4.1-nano | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0017 |
gpt-4o | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0414 |
gpt-5-high | bcdedit.exe | let window = 30m;
// 1) bcdedit invocations that enable test signing
let BcdEditEnableCmds =
DeviceProcessEvents
| where tolower(FileName) in ("bcdedit.exe", "bcdedit") or ProcessCommandLine has "bcdedit"
| where ProcessCommandLine matches regex @"(?i)\bbcdedit(?:\.exe)?\b.*?(?:/|-)?set\b.*?\btestsigning\b.*?\b(on|yes|true|1)\b"
| project DeviceId, DeviceName,
BcdEditTime=TimeGenerated,
BcdEditPid=ProcessId,
BcdEditFileName=FileName,
BcdEditFolderPath=FolderPath,
BcdEditCmd=ProcessCommandLine,
BcdEditElevation=ProcessTokenElevation,
InitiatingProcessFileName,
InitiatingProcessId,
InitiatingProcessCommandLine,
InitiatingProcessAccountDomain,
InitiatingProcessAccountName,
InitiatingProcessAccountSid,
InitiatingProcessParentFileName,
InitiatingProcessParentId;
// 2) BCD hive registry activity (evidence that BCD changed)
let BcdRegWrites =
DeviceRegistryEvents
| where tolower(RegistryKey) has "\\bcd00000000\\objects\\" and tolower(RegistryKey) has "\\elements\\"
| where ActionType in ("RegistryValueSet","RegistryValueCreate","RegistryKeyCreated","RegistryKeyModified","RegistryKeyRestored","RegistryValueDeleted","RegistryKeyDeleted")
| extend ElementId = tostring(extract(@"\\elements\\([0-9a-f]{8})\\", 1, tolower(RegistryKey)))
| project RegTime=TimeGenerated, DeviceId, DeviceName, ActionType, RegistryKey, ElementId, RegistryValueName, RegistryValueData,
WriterProcessFileName=InitiatingProcessFileName,
WriterProcessId=InitiatingProcessId,
WriterProcessCommandLine=InitiatingProcessCommandLine,
WriterProcessParentFileName=InitiatingProcessParentFileName,
WriterProcessParentId=InitiatingProcessParentId,
WriterAccountDomain=InitiatingProcessAccountDomain,
WriterAccountName=InitiatingProcessAccountName,
WriterAccountSid=InitiatingProcessAccountSid;
// 3) Correlate bcdedit commands with nearby BCD writes; also include fallback when only registry writer is visible
union
(
BcdEditEnableCmds
| join kind=leftouter (BcdRegWrites) on DeviceId
| where isnull(RegTime) or RegTime between (BcdEditTime - window .. BcdEditTime + window)
| extend BootEntry = tostring(extract(@"(?i)\{(current|default|globalsettings|[0-9a-f-]{36})\}", 1, BcdEditCmd)),
RequestedState = tostring(extract(@"(?i)\\btestsigning\\b.*?\\b(on|yes|true|1)\\b", 1, BcdEditCmd))
| summarize RegEvents=countif(isnotnull(RegTime)),
FirstRegTime=minif(RegTime, isnotnull(RegTime)),
LastRegTime=maxif(RegTime, isnotnull(RegTime)),
SampleRegKeys=make_set(RegistryKey, 5)
by DeviceId, DeviceName, BcdEditTime, BcdEditPid, BcdEditFileName, BcdEditFolderPath, BcdEditCmd, BcdEditElevation,
InitiatingProcessFileName, InitiatingProcessId, InitiatingProcessCommandLine,
InitiatingProcessAccountDomain, InitiatingProcessAccountName, InitiatingProcessAccountSid,
InitiatingProcessParentFileName, InitiatingProcessParentId, BootEntry, RequestedState
),
(
// Fallback: attribute from the registry writer if its command line shows enabling Test Signing
BcdRegWrites
| where tostring(WriterProcessCommandLine) matches regex @"(?i)\bbcdedit(?:\.exe)?\b.*?(?:/|-)?set\b.*?\btestsigning\b.*?\b(on|yes|true|1)\b"
| extend BcdEditTime=RegTime,
BcdEditPid=WriterProcessId,
BcdEditFileName=WriterProcessFileName,
BcdEditFolderPath="",
BcdEditCmd=WriterProcessCommandLine,
BcdEditElevation="",
InitiatingProcessFileName=WriterProcessParentFileName,
InitiatingProcessId=WriterProcessParentId,
InitiatingProcessCommandLine="",
InitiatingProcessAccountDomain=WriterAccountDomain,
InitiatingProcessAccountName=WriterAccountName,
InitiatingProcessAccountSid=WriterAccountSid,
BootEntry=tostring(extract(@"(?i)\{(current|default|globalsettings|[0-9a-f-]{36})\}", 1, WriterProcessCommandLine)),
RequestedState=tostring(extract(@"(?i)\\btestsigning\\b.*?\\b(on|yes|true|1)\\b", 1, WriterProcessCommandLine)),
RegEvents=1,
FirstRegTime=RegTime,
LastRegTime=RegTime,
SampleRegKeys=pack_array(RegistryKey)
| project DeviceId, DeviceName, BcdEditTime, BcdEditPid, BcdEditFileName, BcdEditFolderPath, BcdEditCmd, BcdEditElevation,
InitiatingProcessFileName, InitiatingProcessId, InitiatingProcessCommandLine,
InitiatingProcessAccountDomain, InitiatingProcessAccountName, InitiatingProcessAccountSid,
InitiatingProcessParentFileName=WriterProcessParentFileName, InitiatingProcessParentId=WriterProcessParentId,
BootEntry, RequestedState, RegEvents, FirstRegTime, LastRegTime, SampleRegKeys
)
| summarize arg_max(BcdEditTime, *) by DeviceId, BcdEditCmd
| order by BcdEditTime desc | Incorrect | 5 | 713.52s | 0.3739 |
gpt-5-mini-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0272 |
gpt-5-mini-low | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0223 |
gpt-5-mini-medium | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0259 |
gpt-5-nano-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0076 |
gpt-5-nano-medium | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0091 |
grok-3-beta | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0946 |
grok-3-mini-beta | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0071 |
o1-high | bcdedit.exe | DeviceProcessEvents
| where FileName =~ "bcdedit.exe" or ProcessCommandLine has "bcdedit"
| where ProcessCommandLine has_any ("testsigning", "TestSigning", "TESTSIGNING")
| order by TimeGenerated desc
| project TimeGenerated,
DeviceName,
FileName,
ProcessCommandLine,
FolderPath,
AccountDomain,
AccountName,
InitiatingProcessFileName,
InitiatingProcessCommandLine,
InitiatingProcessAccountDomain,
InitiatingProcessAccountName,
InitiatingProcessTokenElevation
| Correct | 2 | 74.74s | 0.4691 |
o1-low | cmd.exe bcdedit /set testsigning on | // Combine registry events for BCD store changes and processes that ran bcdedit
// to see who or what turned on test signing mode
union
(
DeviceRegistryEvents
| where ActionType in ("SetValue", "ModifyValue", "CreateValue")
// Look for references to the BCD store or test signing in the registry
| where RegistryKey has "BCD" or RegistryValueData has "testsigning"
| project TimeGenerated, DeviceId, DeviceName, ActionType, RegistryKey, RegistryValueName, RegistryValueData,
InitiatingProcessFileName, InitiatingProcessCommandLine, InitiatingProcessAccountDomain, InitiatingProcessAccountName
),
(
DeviceProcessEvents
// Look for a process that actually ran bcdedit with a test signing parameter
| where FileName =~ "bcdedit.exe" or ProcessCommandLine has "bcdedit"
| where ProcessCommandLine has "testsigning" // Focus on enabling test signing
| project TimeGenerated, DeviceId, DeviceName, ActionType, FileName, ProcessCommandLine,
InitiatingProcessFileName, InitiatingProcessCommandLine, InitiatingProcessAccountDomain, InitiatingProcessAccountName
)
| order by TimeGenerated desc | Correct | 2 | 97.97s | 0.5971 |
o3-high | bcdedit /set testsigning on | // ========================================================================
// Test-signing mode investigation: who enabled it and with what command //
// ========================================================================
// (Time range will be applied by the hunting portal; uncomment next line
// if you want to test it manually)
// | where TimeGenerated between (ago(7d) .. now())
// 1. Registry evidence that the TESTSIGNING element was set in the BCD
let RegistryChanges = DeviceRegistryEvents
| where ActionType in ("RegistryValueSet", "RegistryKeyWritten", "SetValue", "CreateKey")
| where RegistryKey has "\\BCD" // Boot Configuration Data hive
| where RegistryValueName has_cs "TESTSIGNING" or RegistryValueData has_cs "TESTSIGNING"
| project DeviceId, DeviceName,
RegistryEventTime = TimeGenerated,
RegistryKey,
RegistryValueName,
RegistryValueData,
RegInitiatingProcessPath = strcat(InitiatingProcessFolderPath, "\\", InitiatingProcessFileName),
RegInitiatingProcessCommandLine = InitiatingProcessCommandLine,
RegInitiatingProcessSHA256 = InitiatingProcessSHA256,
RegInitiatingProcessId = InitiatingProcessId,
RegUser = strcat(InitiatingProcessAccountDomain, "\\", InitiatingProcessAccountName);
// 2. Actual executions of bcdedit intended to enable test-signing
let BcdeditRuns = DeviceProcessEvents
| where FileName =~ "bcdedit.exe" // bcdedit execution
| where ProcessCommandLine contains "testsigning" // mentions TESTSIGNING
| where ProcessCommandLine matches regex @"(?i)\b(on|yes|enable|1)\b" // enabling it
| project DeviceId, DeviceName,
ProcessTime = TimeGenerated,
BcdeditCommandLine = ProcessCommandLine,
BcdeditPath = strcat(FolderPath, "\\", FileName),
ProcSHA256 = SHA256,
ProcessId,
ProcUser = strcat(AccountDomain, "\\", AccountName);
// 3. Correlate the two artefacts (same host, within ±10 minutes)
RegistryChanges
| join kind=fullouter BcdeditRuns on DeviceId
| extend TimeDeltaMin = iif(isnotempty(RegistryEventTime) and isnotempty(ProcessTime),
abs(datetime_diff("minute", RegistryEventTime, ProcessTime)),
long(null))
| where isnull(TimeDeltaMin) or TimeDeltaMin <= 10
| project EventTime = coalesce(RegistryEventTime, ProcessTime),
DeviceName,
DeviceId,
// Registry side
RegistryKey,
RegistryValueName,
RegistryValueData,
RegInitiatingProcessPath,
RegInitiatingProcessCommandLine,
RegInitiatingProcessSHA256,
// Process side
BcdeditPath,
BcdeditCommandLine,
ProcSHA256,
// Who did it
User = coalesce(RegUser, ProcUser),
TimeDeltaMin
| order by EventTime desc | Correct | 2 | 110.87s | 0.0664 |
o3-mini-high | cmd.exe | let RegistryEvents = DeviceRegistryEvents
| where ActionType == "RegistryValueSet"
| where (tolower(RegistryKey) contains "boot" or tolower(RegistryKey) contains "bcd")
| where (tolower(RegistryValueName) contains "testsign" or tolower(RegistryValueData) contains "on")
| where isnotempty(InitiatingProcessCommandLine)
| project TimeGenerated, DeviceName, RegistryKey, RegistryValueName, RegistryValueData, InitiatingProcessFileName, InitiatingProcessCommandLine;
let ProcessEvents = DeviceProcessEvents
| where tolower(ProcessCommandLine) contains "bcdedit"
| where tolower(ProcessCommandLine) contains "testsign"
| project TimeGenerated, DeviceName, ProcessCommandLine, InitiatingProcessFileName, InitiatingProcessCommandLine;
RegistryEvents
| union ProcessEvents
| order by TimeGenerated desc | Incorrect | 4 | 77.16s | 0.0522 |
o3-mini-low | cmd.exe | union withsource = TableName DeviceRegistryEvents, DeviceProcessEvents
| where ((InitiatingProcessCommandLine contains "bcdedit" or InitiatingProcessCommandLine contains "testsign")
or (ProcessCommandLine contains "bcdedit" and ProcessCommandLine contains "testsign"))
| extend CommandLine = coalesce(InitiatingProcessCommandLine, ProcessCommandLine)
| project TimeGenerated, DeviceName, TableName, CommandLine, InitiatingProcessId, ProcessId, RegistryKey, RegistryValueName, RegistryValueData
| sort by TimeGenerated desc | Incorrect | 3 | 78.54s | 0.0556 |
o4-mini-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0588 |
o4-mini-low | bcdedit.exe | let regEvents = DeviceRegistryEvents
| where ActionType == "SetValue"
and RegistryKey contains "BCD00000000"
and tolower(RegistryValueName) contains "testsign"
| project TimeGenerated,
DeviceName,
InitiatingProcessAccountName,
InitiatingProcessAccountDomain,
InitiatingProcessFileName,
InitiatingProcessId,
InitiatingProcessCommandLine,
RegistryKey,
RegistryValueName,
RegistryValueData;
let procEvents = DeviceProcessEvents
| where tolower(FileName) endswith "bcdedit.exe"
and tolower(ProcessCommandLine) contains "testsign"
| project TimeGenerated,
DeviceName,
InitiatingProcessAccountName,
InitiatingProcessAccountDomain,
InitiatingProcessFileName = FileName,
InitiatingProcessId = ProcessId,
InitiatingProcessCommandLine = ProcessCommandLine,
RegistryKey = "(n/a)",
RegistryValueName = "(n/a)",
RegistryValueData = "(n/a)";
regEvents
| union procEvents
| sort by TimeGenerated desc | Correct | 2 | 69.20s | 0.0161 |