A Windows host shows a suspicious registry change under the LSA hive. Review recent registry events to locate any new entries under Authentication Packages and determine the name of the DLL the attacker added.
KQL answer reference and model-by-model benchmark comparison
be2590e8-4ac3-47ac-b4b5-945820f2fbe9
Question ID
7/23
Models Correct
1
Valid Answer
30%
Success Rate
Reference Answer
Human-validated correct answer for this question
package.dll
Model Performance Comparison
See how different AI models performed on this question
| Model | Answer | KQL Query | Status | Attempts | Time | Cost |
|---|---|---|---|---|---|---|
| gemini-2.5-flash-preview-04-17 | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0353 |
| gpt-35-turbo | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0108 |
| gpt-4-turbo-2024-04-09 | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.2224 |
| gpt-4.1 | package.dll | DeviceRegistryEvents
| where RegistryKey has "Control\\Lsa"
| where RegistryValueName contains_cs "Authenticat"
| where ActionType in ("RegistryValueSet", "RegistryKeyValueSet", "RegistryKeyWritten", "SetValue", "RegValueSet", "Modified", "Value Set")
| project TimeGenerated, DeviceName, InitiatingProcessAccountName, InitiatingProcessFileName, RegistryKey, RegistryValueName, RegistryValueData
| order by TimeGenerated desc | Correct | 3 | 11.44s | 0.0322 |
| gpt-4.1-finetuned | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0490 |
| gpt-4.1-mini | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0073 |
| gpt-4.1-nano | msv1_0\package.dll | DeviceRegistryEvents
| where RegistryKey has_any ('HKEY_LOCAL_MACHINE\\System\\CurrentControlSet\\Control\\Lsa', 'HKEY_LOCAL_MACHINE\\System\\ControlSet001\\Control\\Lsa')
| where RegistryValueName == 'Authentication Packages'
| project TimeGenerated, RegistryKey, RegistryValueName, RegistryValueData
| order by TimeGenerated desc
| where RegistryValueData contains '.dll'
| summarize arg_max(TimeGenerated, *) by RegistryValueData | Correct | 3 | 4.83s | 0.0009 |
| gpt-4o | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0531 |
| gpt-5-high | package.dll | let lsaKeyRegex = @"(?i).*(\\system\\(currentcontrolset|controlset\d{3})\\control\\lsa)$";
let normalize = (s:string) {
tolower(trim(" ;,\t\r\n", replace_regex(replace_regex(replace_regex(tostring(s), @"\u0000", ";"), @"[\r\n]+", ";"), @",", ";")))
};
DeviceRegistryEvents
| where RegistryValueName =~ "Authentication Packages"
| where RegistryKey matches regex lsaKeyRegex
| extend OldRaw = tostring(PreviousRegistryValueData), NewRaw = tostring(RegistryValueData)
| where isnotempty(NewRaw)
| extend NewArr = split(normalize(NewRaw), ";")
| extend OldArr = iif(isempty(OldRaw), dynamic([]), split(normalize(OldRaw), ";"))
| extend AddedPackages = set_difference(NewArr, OldArr)
| extend AddedPackages = set_difference(AddedPackages, dynamic([""]))
| where array_length(AddedPackages) > 0
| project TimeGenerated, DeviceName, DeviceId, RegistryKey, RegistryValueName,
AddedPackages,
RegistryValueData = NewRaw,
PreviousRegistryValueData = OldRaw,
InitiatingProcessAccountDomain, InitiatingProcessAccountName,
InitiatingProcessFileName, InitiatingProcessCommandLine,
InitiatingProcessId, InitiatingProcessSHA256
| order by TimeGenerated desc | Correct | 1 | 93.10s | 0.0521 |
| gpt-5-mini-high | msv1_0\x00package.dll | let regs = DeviceRegistryEvents
| where RegistryKey contains_cs "Control\\Lsa" or RegistryKey contains_cs "CurrentControlSet\\Control\\Lsa" or RegistryKey contains_cs "HKLM\\SYSTEM\\CurrentControlSet\\Control\\Lsa" or RegistryKey contains_cs "\\Registry\\Machine\\System\\CurrentControlSet\\Control\\Lsa"
| where tolower(tostring(RegistryValueName)) contains "authentication package" or tolower(tostring(RegistryValueName)) contains "auth package" or tolower(tostring(RegistryValueName)) contains "auth packages"
| extend CurrRaw = tostring(RegistryValueData), PrevRaw = tostring(PreviousRegistryValueData)
| extend CurrClean = trim(" ;", replace_regex(CurrRaw, @"\\x00+", ";")), PrevClean = trim(" ;", replace_regex(PrevRaw, @"\\x00+", ";"))
| extend CurrList = split(CurrClean, ";"), PrevList = split(PrevClean, ";")
| extend EventId = strcat(DeviceId, "|", tostring(TimeGenerated))
| project EventId, TimeGenerated, DeviceName, DeviceId, InitiatingProcessFileName, InitiatingProcessCommandLine, RegistryKey, RegistryValueName, CurrList, PrevList, ActionType, ReportId;
let curr = regs
| mv-expand Entry = CurrList to typeof(string)
| extend Entry = trim(" \"\n\r\t", Entry)
| where Entry != ""
| project EventId, TimeGenerated, DeviceName, DeviceId, InitiatingProcessFileName, InitiatingProcessCommandLine, RegistryKey, RegistryValueName, Entry;
let prev = regs
| mv-expand Entry = PrevList to typeof(string)
| extend Entry = trim(" \"\n\r\t", Entry)
| where Entry != ""
| project EventId, Entry;
curr
| join kind=leftanti (prev) on EventId, Entry
| extend DLL = tostring(Entry)
| where tolower(DLL) endswith ".dll"
| summarize FirstSeen=min(TimeGenerated), LastSeen=max(TimeGenerated), Devices=make_set(DeviceName), DeviceIds=make_set(DeviceId), InitiatingProcesses=make_set(InitiatingProcessFileName), InitiatingCommands=make_set(InitiatingProcessCommandLine), RegistryKeys=make_set(RegistryKey), Events=make_set(EventId) by DLL, RegistryValueName
| order by LastSeen desc
| project DLL, RegistryValueName, FirstSeen, LastSeen, Devices, DeviceIds, InitiatingProcesses, InitiatingCommands, RegistryKeys, Events | Correct | 3 | 66.38s | 0.0176 |
| gpt-5-mini-low | package.dll | // Search for recent LSA registry changes (Authentication Packages)
DeviceRegistryEvents
| where TimeGenerated >= ago(7d) // remove or adjust if caller sets timerange externally
| where RegistryKey has_cs "CurrentControlSet\\Control\\Lsa" or RegistryKey has_cs "\\Control\\Lsa" or RegistryKey has_cs "\\Lsa"
| where tolower(RegistryValueName) has "authentication" or tolower(RegistryValueName) has "authenticationpackages" or tolower(RegistryValueName) has "authentication packages" or tolower(RegistryValueName) has "security packages" or tolower(RegistryValueName) has "securitypackages"
| where ActionType has_any ("SetValue", "Create", "Set", "ValueSet", "RegistryValueSet", "Modify", "Modified", "Update")
| extend LsaKey = RegistryKey, ValueName = RegistryValueName, ValueDataStr = tostring(RegistryValueData), PrevValue = tostring(PreviousRegistryValueData)
| extend ValueDataClean = replace_regex(ValueDataStr, "[\r\n\t]+", " ")
| mv-expand entry = split(ValueDataClean, ";")
| mv-expand entry = split(entry, ",")
| mv-expand entry = split(entry, " ")
| extend entry = trim(' \"', tostring(entry))
| where entry != ""
| extend entryLower = tolower(entry)
| where entryLower endswith ".dll" or entryLower contains ".dll"
| summarize FirstSeen = min(TimeGenerated), LastSeen = max(TimeGenerated), Count = count(), ExampleAction = any(ActionType), ExampleProcess = any(InitiatingProcessFileName), ExampleCmd = any(InitiatingProcessCommandLine), PrevExample = any(PrevValue) by DeviceName, DeviceId, LsaKey, ValueName, AddedDll = entry
| order by LastSeen desc
// Enrich with file events where possible
| join kind=leftouter (
DeviceFileEvents
| project FileTime=TimeGenerated, DeviceId, FileName, FolderPath, FileFullPath = strcat(FolderPath, "\\", FileName), FileEventCmd=InitiatingProcessCommandLine
) on DeviceId
| where AddedDll contains tolower(FileName) or tolower(FileFullPath) contains tolower(AddedDll) or tolower(AddedDll) contains tolower(FileName)
| project DeviceName, DeviceId, LsaKey, ValueName, AddedDll, FirstSeen, LastSeen, Count, ExampleAction, ExampleProcess, ExampleCmd, FileTime, FileFullPath, FileName, FileEventCmd
| order by LastSeen desc | Correct | 2 | 23.85s | 0.0052 |
| gpt-5-mini-medium | package.dll | DeviceRegistryEvents
| where ActionType in ("SetValue", "Create", "Update", "Delete", "RegistryValueSet")
| where TimeGenerated >= ago(30d)
| extend rk = tolower(tostring(RegistryKey)), rvn = tolower(tostring(RegistryValueName)), rvd = tostring(RegistryValueData), prev_rvd = tostring(PreviousRegistryValueData)
// Focus on the LSA path or the Authentication Packages value name
| where (rk contains "currentcontrolset" and rk contains "control" and rk contains "lsa") or rvn contains "authentication packages" or rvn contains "authenticationpackages"
| where isnotempty(rvd) or isnotempty(rvn)
// Extract DLL names from value data or value name. Pattern allows backslashes and drive letters.
| extend dll_from_data = extract(@"([A-Za-z0-9_\\:\\\.\-]+\.dll)", 1, rvd)
| extend dll_from_name = extract(@"([A-Za-z0-9_\\:\\\.\-]+\.dll)", 1, rvn)
| extend AddedDLL = tolower(coalesce(dll_from_data, dll_from_name))
| where isnotempty(AddedDLL)
// If previous value exists and already contained the DLL, it's not a new addition
| extend prev_contains = iff(isnotempty(prev_rvd) and tolower(prev_rvd) contains AddedDLL, true, false)
| where prev_contains == false
// Keep the most recent record per device and DLL
| summarize arg_max(TimeGenerated, *) by DeviceId, AddedDLL
| project TimeGenerated, DeviceName, DeviceId, AddedDLL, RegistryKey, RegistryValueName, RegistryValueType, RegistryValueData, PreviousRegistryValueData, InitiatingProcessAccountName, InitiatingProcessFileName, InitiatingProcessCommandLine, ActionType
| order by TimeGenerated desc | Correct | 5 | 46.57s | 0.0137 |
| gpt-5-nano-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0090 |
| gpt-5-nano-medium | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0114 |
| grok-3-beta | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0858 |
| grok-3-mini-beta | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0070 |
| o1-high | package.dll | DeviceRegistryEvents
| where RegistryKey has "\\Control\\Lsa"
| where RegistryValueName =~ "Authentication Packages"
| where ActionType in ("RegistryValueSet", "RegistryValueCreated", "RegistryValueModified")
| project TimeGenerated, DeviceName, ActionType, RegistryKey, RegistryValueName, RegistryValueData, PreviousRegistryValueData, InitiatingProcessAccountName, InitiatingProcessCommandLine
| order by TimeGenerated desc | Correct | 1 | 27.39s | 0.2260 |
| o1-low | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.7242 |
| o3-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0903 |
| o3-mini-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0348 |
| o3-mini-low | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0397 |
| o4-mini-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0420 |
| o4-mini-low | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0594 |