Question Analysis
A Windows host shows a suspicious registry change under the LSA hive. Review recent registry events to locate any new entries under Authentication Packages and determine the name of the DLL the attacker added.
be2590e8-4ac3-47ac-b4b5-945820f2fbe9
Question ID
7/23
Models Correct
1
Valid Answer
30%
Success Rate
Reference Answer
Human-validated correct answer for this question
package.dll
Model Performance Comparison
See how different AI models performed on this question
Model | Answer | KQL Query | Status | Attempts | Time | Cost |
---|---|---|---|---|---|---|
gemini-2.5-flash-preview-04-17 | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0353 |
gpt-35-turbo | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0108 |
gpt-4-turbo-2024-04-09 | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.2224 |
gpt-4.1 | package.dll | DeviceRegistryEvents
| where RegistryKey has "Control\\Lsa"
| where RegistryValueName contains_cs "Authenticat"
| where ActionType in ("RegistryValueSet", "RegistryKeyValueSet", "RegistryKeyWritten", "SetValue", "RegValueSet", "Modified", "Value Set")
| project TimeGenerated, DeviceName, InitiatingProcessAccountName, InitiatingProcessFileName, RegistryKey, RegistryValueName, RegistryValueData
| order by TimeGenerated desc | Correct | 3 | 11.44s | 0.0322 |
gpt-4.1-finetuned | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0490 |
gpt-4.1-mini | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0073 |
gpt-4.1-nano | msv1_0\package.dll | DeviceRegistryEvents
| where RegistryKey has_any ('HKEY_LOCAL_MACHINE\\System\\CurrentControlSet\\Control\\Lsa', 'HKEY_LOCAL_MACHINE\\System\\ControlSet001\\Control\\Lsa')
| where RegistryValueName == 'Authentication Packages'
| project TimeGenerated, RegistryKey, RegistryValueName, RegistryValueData
| order by TimeGenerated desc
| where RegistryValueData contains '.dll'
| summarize arg_max(TimeGenerated, *) by RegistryValueData | Correct | 3 | 4.83s | 0.0009 |
gpt-4o | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0531 |
gpt-5-high | package.dll | let lsaKeyRegex = @"(?i).*(\\system\\(currentcontrolset|controlset\d{3})\\control\\lsa)$";
let normalize = (s:string) {
tolower(trim(" ;,\t\r\n", replace_regex(replace_regex(replace_regex(tostring(s), @"\u0000", ";"), @"[\r\n]+", ";"), @",", ";")))
};
DeviceRegistryEvents
| where RegistryValueName =~ "Authentication Packages"
| where RegistryKey matches regex lsaKeyRegex
| extend OldRaw = tostring(PreviousRegistryValueData), NewRaw = tostring(RegistryValueData)
| where isnotempty(NewRaw)
| extend NewArr = split(normalize(NewRaw), ";")
| extend OldArr = iif(isempty(OldRaw), dynamic([]), split(normalize(OldRaw), ";"))
| extend AddedPackages = set_difference(NewArr, OldArr)
| extend AddedPackages = set_difference(AddedPackages, dynamic([""]))
| where array_length(AddedPackages) > 0
| project TimeGenerated, DeviceName, DeviceId, RegistryKey, RegistryValueName,
AddedPackages,
RegistryValueData = NewRaw,
PreviousRegistryValueData = OldRaw,
InitiatingProcessAccountDomain, InitiatingProcessAccountName,
InitiatingProcessFileName, InitiatingProcessCommandLine,
InitiatingProcessId, InitiatingProcessSHA256
| order by TimeGenerated desc | Correct | 1 | 93.10s | 0.0521 |
gpt-5-mini-high | msv1_0\x00package.dll | let regs = DeviceRegistryEvents
| where RegistryKey contains_cs "Control\\Lsa" or RegistryKey contains_cs "CurrentControlSet\\Control\\Lsa" or RegistryKey contains_cs "HKLM\\SYSTEM\\CurrentControlSet\\Control\\Lsa" or RegistryKey contains_cs "\\Registry\\Machine\\System\\CurrentControlSet\\Control\\Lsa"
| where tolower(tostring(RegistryValueName)) contains "authentication package" or tolower(tostring(RegistryValueName)) contains "auth package" or tolower(tostring(RegistryValueName)) contains "auth packages"
| extend CurrRaw = tostring(RegistryValueData), PrevRaw = tostring(PreviousRegistryValueData)
| extend CurrClean = trim(" ;", replace_regex(CurrRaw, @"\\x00+", ";")), PrevClean = trim(" ;", replace_regex(PrevRaw, @"\\x00+", ";"))
| extend CurrList = split(CurrClean, ";"), PrevList = split(PrevClean, ";")
| extend EventId = strcat(DeviceId, "|", tostring(TimeGenerated))
| project EventId, TimeGenerated, DeviceName, DeviceId, InitiatingProcessFileName, InitiatingProcessCommandLine, RegistryKey, RegistryValueName, CurrList, PrevList, ActionType, ReportId;
let curr = regs
| mv-expand Entry = CurrList to typeof(string)
| extend Entry = trim(" \"\n\r\t", Entry)
| where Entry != ""
| project EventId, TimeGenerated, DeviceName, DeviceId, InitiatingProcessFileName, InitiatingProcessCommandLine, RegistryKey, RegistryValueName, Entry;
let prev = regs
| mv-expand Entry = PrevList to typeof(string)
| extend Entry = trim(" \"\n\r\t", Entry)
| where Entry != ""
| project EventId, Entry;
curr
| join kind=leftanti (prev) on EventId, Entry
| extend DLL = tostring(Entry)
| where tolower(DLL) endswith ".dll"
| summarize FirstSeen=min(TimeGenerated), LastSeen=max(TimeGenerated), Devices=make_set(DeviceName), DeviceIds=make_set(DeviceId), InitiatingProcesses=make_set(InitiatingProcessFileName), InitiatingCommands=make_set(InitiatingProcessCommandLine), RegistryKeys=make_set(RegistryKey), Events=make_set(EventId) by DLL, RegistryValueName
| order by LastSeen desc
| project DLL, RegistryValueName, FirstSeen, LastSeen, Devices, DeviceIds, InitiatingProcesses, InitiatingCommands, RegistryKeys, Events | Correct | 3 | 66.38s | 0.0176 |
gpt-5-mini-low | package.dll | // Search for recent LSA registry changes (Authentication Packages)
DeviceRegistryEvents
| where TimeGenerated >= ago(7d) // remove or adjust if caller sets timerange externally
| where RegistryKey has_cs "CurrentControlSet\\Control\\Lsa" or RegistryKey has_cs "\\Control\\Lsa" or RegistryKey has_cs "\\Lsa"
| where tolower(RegistryValueName) has "authentication" or tolower(RegistryValueName) has "authenticationpackages" or tolower(RegistryValueName) has "authentication packages" or tolower(RegistryValueName) has "security packages" or tolower(RegistryValueName) has "securitypackages"
| where ActionType has_any ("SetValue", "Create", "Set", "ValueSet", "RegistryValueSet", "Modify", "Modified", "Update")
| extend LsaKey = RegistryKey, ValueName = RegistryValueName, ValueDataStr = tostring(RegistryValueData), PrevValue = tostring(PreviousRegistryValueData)
| extend ValueDataClean = replace_regex(ValueDataStr, "[\r\n\t]+", " ")
| mv-expand entry = split(ValueDataClean, ";")
| mv-expand entry = split(entry, ",")
| mv-expand entry = split(entry, " ")
| extend entry = trim(' \"', tostring(entry))
| where entry != ""
| extend entryLower = tolower(entry)
| where entryLower endswith ".dll" or entryLower contains ".dll"
| summarize FirstSeen = min(TimeGenerated), LastSeen = max(TimeGenerated), Count = count(), ExampleAction = any(ActionType), ExampleProcess = any(InitiatingProcessFileName), ExampleCmd = any(InitiatingProcessCommandLine), PrevExample = any(PrevValue) by DeviceName, DeviceId, LsaKey, ValueName, AddedDll = entry
| order by LastSeen desc
// Enrich with file events where possible
| join kind=leftouter (
DeviceFileEvents
| project FileTime=TimeGenerated, DeviceId, FileName, FolderPath, FileFullPath = strcat(FolderPath, "\\", FileName), FileEventCmd=InitiatingProcessCommandLine
) on DeviceId
| where AddedDll contains tolower(FileName) or tolower(FileFullPath) contains tolower(AddedDll) or tolower(AddedDll) contains tolower(FileName)
| project DeviceName, DeviceId, LsaKey, ValueName, AddedDll, FirstSeen, LastSeen, Count, ExampleAction, ExampleProcess, ExampleCmd, FileTime, FileFullPath, FileName, FileEventCmd
| order by LastSeen desc | Correct | 2 | 23.85s | 0.0052 |
gpt-5-mini-medium | package.dll | DeviceRegistryEvents
| where ActionType in ("SetValue", "Create", "Update", "Delete", "RegistryValueSet")
| where TimeGenerated >= ago(30d)
| extend rk = tolower(tostring(RegistryKey)), rvn = tolower(tostring(RegistryValueName)), rvd = tostring(RegistryValueData), prev_rvd = tostring(PreviousRegistryValueData)
// Focus on the LSA path or the Authentication Packages value name
| where (rk contains "currentcontrolset" and rk contains "control" and rk contains "lsa") or rvn contains "authentication packages" or rvn contains "authenticationpackages"
| where isnotempty(rvd) or isnotempty(rvn)
// Extract DLL names from value data or value name. Pattern allows backslashes and drive letters.
| extend dll_from_data = extract(@"([A-Za-z0-9_\\:\\\.\-]+\.dll)", 1, rvd)
| extend dll_from_name = extract(@"([A-Za-z0-9_\\:\\\.\-]+\.dll)", 1, rvn)
| extend AddedDLL = tolower(coalesce(dll_from_data, dll_from_name))
| where isnotempty(AddedDLL)
// If previous value exists and already contained the DLL, it's not a new addition
| extend prev_contains = iff(isnotempty(prev_rvd) and tolower(prev_rvd) contains AddedDLL, true, false)
| where prev_contains == false
// Keep the most recent record per device and DLL
| summarize arg_max(TimeGenerated, *) by DeviceId, AddedDLL
| project TimeGenerated, DeviceName, DeviceId, AddedDLL, RegistryKey, RegistryValueName, RegistryValueType, RegistryValueData, PreviousRegistryValueData, InitiatingProcessAccountName, InitiatingProcessFileName, InitiatingProcessCommandLine, ActionType
| order by TimeGenerated desc | Correct | 5 | 46.57s | 0.0137 |
gpt-5-nano-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0090 |
gpt-5-nano-medium | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0114 |
grok-3-beta | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0858 |
grok-3-mini-beta | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0070 |
o1-high | package.dll | DeviceRegistryEvents
| where RegistryKey has "\\Control\\Lsa"
| where RegistryValueName =~ "Authentication Packages"
| where ActionType in ("RegistryValueSet", "RegistryValueCreated", "RegistryValueModified")
| project TimeGenerated, DeviceName, ActionType, RegistryKey, RegistryValueName, RegistryValueData, PreviousRegistryValueData, InitiatingProcessAccountName, InitiatingProcessCommandLine
| order by TimeGenerated desc | Correct | 1 | 27.39s | 0.2260 |
o1-low | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.7242 |
o3-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0903 |
o3-mini-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0348 |
o3-mini-low | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0397 |
o4-mini-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0420 |
o4-mini-low | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0594 |