// question
On a Windows device, examine registry event logs for modifications under the System policies path. Determine which registry value name was altered to disable the shutdown button at login.
Reference KQL answer and model-by-model comparison.
6e0d1131-2d7e-4905-8ca5-d6172f05d03d
question id
9/23
models correct
1
valid answer
39%
success rate
// reference answer
Human-validated correct answer for this question.
shutdownwithoutlogon
// model comparison
| Model | Answer | KQL query | Status | Tries | Time | Cost |
|---|---|---|---|---|---|---|
| gemini-2.5-flash-preview-04-17 | shutdownwithoutlogon | DeviceRegistryEvents | where ActionType == "RegistryValueSet" | where RegistryKey contains @"Policies\System" | project Timestamp, DeviceName, RegistryKey, RegistryValueName, RegistryValueData, ActionType | pass | 3 | 23.70s | 0.0100 |
| gpt-35-turbo | no answer | no query generated | fail | 5 | 0.00s | 0.0114 |
| gpt-4-turbo-2024-04-09 | no answer | no query generated | fail | 5 | 0.00s | 0.2317 |
| gpt-4.1 | shutdownwithoutlogon | DeviceRegistryEvents
| where tolower(RegistryKey) has @"policies\system"
| where ActionType in ("SetValue", "ModifyValue", "ValueModified", "RegistryValueSet")
| where tolower(RegistryValueName) == "shutdownwithoutlogon"
| project TimeGenerated, DeviceName, InitiatingProcessAccountName, RegistryKey, RegistryValueName, RegistryValueData, ActionType | pass | 2 | 8.73s | 0.0215 |
| gpt-4.1-finetuned | no answer | no query generated | fail | 5 | 0.00s | 0.0428 |
| gpt-4.1-mini | no answer | no query generated | fail | 5 | 0.00s | 0.0077 |
| gpt-4.1-nano | no answer | no query generated | fail | 5 | 0.00s | 0.0017 |
| gpt-4o | no answer | no query generated | fail | 5 | 0.00s | 0.0477 |
| gpt-5-high | shutdownwithoutlogon | DeviceRegistryEvents
| where ActionType in ("RegistryValueSet", "RegistryValueModified")
// System policies locations
| where RegistryKey has_any ("\\Microsoft\\Windows\\CurrentVersion\\Policies\\System", "\\Policies\\Microsoft\\Windows\\System")
// Value that controls the shutdown button on the logon screen
| where RegistryValueName =~ "ShutdownWithoutLogon"
// Detect it being set to 0 (disabled)
| extend NewValDec = tolong(extract(@"\d+", 0, tostring(RegistryValueData))),
OldValDec = tolong(extract(@"\d+", 0, tostring(PreviousRegistryValueData)))
| where NewValDec == 0 or RegistryValueData has "0x00000000"
| project TimeGenerated, DeviceName, RegistryKey, RegistryValueName, PreviousRegistryValueData, RegistryValueData,
InitiatingProcessAccountDomain, InitiatingProcessAccountName, InitiatingProcessFileName, InitiatingProcessCommandLine, InitiatingProcessId, InitiatingProcessSHA256
| order by TimeGenerated desc | pass | 1 | 116.03s | 0.0508 |
| gpt-5-mini-high | shutdownwithoutlogon | DeviceRegistryEvents | where (tolower(RegistryKey) contains "\\policies\\system" or tolower(PreviousRegistryKey) contains "\\policies\\system") | where isnotempty(RegistryValueName) or isnotempty(PreviousRegistryValueName) | extend ChangedValue = tostring(coalesce(RegistryValueName, PreviousRegistryValueName)) | extend ChangedValueLower = tolower(ChangedValue) | where ChangedValueLower matches regex @".*(shutdown|shutdownwithoutlogon|noclose|hide|power|close|remove).*" | project Timestamp, DeviceName, RegistryKey, PreviousRegistryKey, RegistryValueName, PreviousRegistryValueName, RegistryValueData, PreviousRegistryValueData, InitiatingProcessAccountName, InitiatingProcessFileName, InitiatingProcessCommandLine, ActionType | order by Timestamp desc | take 200 | pass | 1 | 22.48s | 0.0061 |
| gpt-5-mini-low | no answer | no query generated | fail | 5 | 0.00s | 0.0221 |
| gpt-5-mini-medium | shutdownwithoutlogon | DeviceRegistryEvents
| where RegistryKey contains "\\Policies\\System" or RegistryKey contains "\\Microsoft\\Windows\\CurrentVersion\\Policies\\System"
| where ActionType in ("SetValue", "Modify", "Create", "Write", "Delete") or isnotempty(RegistryValueName) or isnotempty(RegistryValueData)
| where tolower(RegistryValueName) in ("shutdownwithoutlogon","shutdownwithoutlogonui","noclose","shutdownwithoutlogon") or RegistryValueName has_cs "Shutdown" or RegistryValueName has_cs "NoClose"
| project TimeGenerated, DeviceName, InitiatingProcessAccountName, InitiatingProcessFileName, InitiatingProcessCommandLine, ActionType, RegistryKey, RegistryValueName, RegistryValueType, PreviousRegistryValueData, RegistryValueData
| order by TimeGenerated desc
| summarize Changes = count(), PrevValues = make_set(PreviousRegistryValueData), NewValues = make_set(RegistryValueData), LatestEvent = max(TimeGenerated) by RegistryValueName, DeviceName
| order by LatestEvent desc
| take 50 | pass | 2 | 31.33s | 0.0075 |
| gpt-5-nano-high | no answer | no query generated | fail | 5 | 0.00s | 0.0079 |
| gpt-5-nano-medium | no answer | no query generated | fail | 5 | 0.00s | 0.0064 |
| grok-3-beta | no answer | no query generated | fail | 5 | 0.00s | 0.0889 |
| grok-3-mini-beta | shutdownwithoutlogon | DeviceRegistryEvents
| where RegistryKey contains 'HKEY_LOCAL_MACHINE\\SOFTWARE\\Policies\\System' or RegistryKey contains 'HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\System'
| where ActionType has_any ('Set', 'RegistryValueSet')
| where RegistryValueName in~ ('NoClose', 'ShutdownWithoutLogon', 'DisableShutdown')
| project Timestamp, DeviceName, RegistryKey, RegistryValueName, RegistryValueData, PreviousRegistryValueData
| order by Timestamp desc | pass | 3 | 32.63s | 0.0044 |
| o1-high | no answer | no query generated | fail | 5 | 0.00s | 0.7958 |
| o1-low | shutdownwithoutlogon | DeviceRegistryEvents
| where Timestamp >= ago(30d)
| where RegistryKey has "HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\System"
| where ActionType in ("RegistryValueSet", "RegistryValueModified", "SetValue", "ModifyValue")
| where RegistryValueName in~ ("DisableShutdownButton", "ShutdownWithoutLogon", "HideShutdownButton")
| project TimeGenerated,
DeviceName,
ActionType,
RegistryKey,
RegistryValueName,
PreviousRegistryValueData,
RegistryValueData,
InitiatingProcessAccountName,
InitiatingProcessCommandLine
| order by TimeGenerated desc | pass | 2 | 46.08s | 0.3536 |
| o3-high | no answer | no query generated | fail | 5 | 0.00s | 0.0752 |
| o3-mini-high | shutdownwithoutlogon | DeviceRegistryEvents | where RegistryKey has "Policies\\System" | where RegistryValueData == "0" | summarize EventCount = count() by RegistryValueName, DeviceName | order by EventCount desc | pass | 1 | 11.92s | 0.0123 |
| o3-mini-low | no answer | no query generated | fail | 5 | 0.00s | 0.0417 |
| o4-mini-high | no answer | no query generated | fail | 5 | 0.00s | 0.0348 |
| o4-mini-low | shutdownwithoutlogon | DeviceRegistryEvents | where ActionType == "RegistryValueSet" | where RegistryKey startswith "HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\System" | summarize arg_max(TimeGenerated, *) by RegistryValueName | project TimeGenerated, DeviceName, RegistryValueName, PreviousRegistryValueData, RegistryValueData | sort by TimeGenerated desc | pass | 4 | 125.10s | 0.0380 |