Question Analysis

A security investigator suspects that someone attempted to dump stored web credentials on a Windows system using an in-built command-line tool. Review process creation logs to determine which executable was called to list the Web Credentials vault.

bc071188-459f-44d5-901a-f8f2625b2d2e
Question ID
12/23
Models Correct
1
Valid Answer
52%
Success Rate
Reference Answer

Human-validated correct answer for this question

vaultcmd.exe
Model Performance Comparison

See how different AI models performed on this question

ModelAnswerKQL QueryStatusAttemptsTimeCost
gemini-2.5-flash-preview-04-17
VaultCmd.exe
DeviceProcessEvents | where FileName =~ "vaultcmd.exe" | project Timestamp, DeviceName, AccountName, InitiatingProcessFileName, InitiatingProcessCommandLine, FileName, ProcessCommandLine
Correct
3
31.05s
0.0164
gpt-35-turbo
No answer provided
No query generated
Incorrect
5
0.00s
0.0112
gpt-4-turbo-2024-04-09
cmdkey.exe
DeviceProcessEvents | where ActionType == 'ProcessCreated' | where ProcessCommandLine has 'cmdkey' and ProcessCommandLine has '/list' | project DeviceName, AccountName, ProcessCommandLine, ProcessCreationTime | order by ProcessCreationTime desc
Incorrect
1
10.99s
0.0531
gpt-4.1
VaultCmd.exe
DeviceProcessEvents | where FileName in~ ("cmdkey.exe", "vaultcmd.exe") or ProcessCommandLine has_any ("cmdkey.exe", "vaultcmd.exe", "Web Credentials", "list") | project TimeGenerated, DeviceName, AccountDomain, AccountName, FileName, ProcessCommandLine, FolderPath, InitiatingProcessAccountName, InitiatingProcessCommandLine | order by TimeGenerated desc
Correct
1
5.71s
0.0297
gpt-4.1-finetuned
VaultCmd.exe
DeviceProcessEvents | where (FileName contains "cmdkey.exe" or FileName contains "cmdkey" or ProcessCommandLine contains "/list") or (FileName contains "rundll32.exe" or ProcessCommandLine contains "keymgr.dll,KRShowKeyMgr")
Correct
1
8.26s
0.0307
gpt-4.1-mini
VaultCmd.exe
DeviceProcessEvents | where FileName in~ ("cmdkey.exe", "vaultcmd.exe") or InitiatingProcessFileName in~ ("cmdkey.exe", "vaultcmd.exe") or ProcessCommandLine contains_cs "vaultcmd" | project TimeGenerated, DeviceName, AccountName, FileName, ProcessId, InitiatingProcessFileName, ProcessCommandLine | order by TimeGenerated desc
Correct
1
5.81s
0.0024
gpt-4.1-nano
No answer provided
No query generated
Incorrect
5
0.00s
0.0013
gpt-4o
VaultCmd.exe
DeviceProcessEvents | where ProcessCommandLine contains "vaultcmd" or FileName == "vaultcmd.exe" | project TimeGenerated, DeviceName, AccountName, FileName, ProcessCommandLine, InitiatingProcessAccountName, InitiatingProcessCommandLine
Correct
1
3.45s
0.0155
gpt-5-high
VaultCmd.exe
let webVaultGuid = "4bf4c442-9b8a-41a0-b380-dd4a704ddb28"; let suspects = DeviceProcessEvents | where ActionType == "ProcessCreated" | extend cmd = tolower(coalesce(ProcessCommandLine, "")), fn = tolower(FileName) | extend IsWebVaultMention = (cmd has webVaultGuid or cmd has "web credentials") | where (fn == "vaultcmd.exe" and (cmd has "/listcreds" or cmd has "/listproperties") and IsWebVaultMention) or (fn == "cmdkey.exe" and cmd has "/list") or (fn == "rundll32.exe" and cmd has "keymgr.dll,krshowkeymgr") or (IsWebVaultMention and fn in ("powershell.exe","pwsh.exe","cmd.exe")); // Summary: which executable(s) were used to list Web Credentials suspects | summarize FirstSeen=min(TimeGenerated), LastSeen=max(TimeGenerated), EventCount=count(), ExampleCommand=arg_max(TimeGenerated, ProcessCommandLine) by DeviceName, AccountDomain, AccountName, FileName, FolderPath | order by LastSeen desc // Details (uncomment to view raw events) // suspects // | project TimeGenerated, DeviceName, AccountDomain, AccountName, FileName, FolderPath, ProcessCommandLine, InitiatingProcessFileName, InitiatingProcessCommandLine, InitiatingProcessAccountDomain, InitiatingProcessAccountName, SHA256 // | order by TimeGenerated desc
Correct
1
99.84s
0.0572
gpt-5-mini-high
VaultCmd.exe
union isfuzzy=true DeviceProcessEvents, DeviceEvents | where FileName in ('cmdkey.exe','vaultcmd.exe','rundll32.exe') or tolower(ProcessCommandLine) contains 'cmdkey' or tolower(ProcessCommandLine) contains 'vaultcmd' or tolower(ProcessCommandLine) contains 'keymgr.dll' or tolower(ProcessCommandLine) contains 'krshowkeymgr' or tolower(InitiatingProcessCommandLine) contains 'cmdkey' or tolower(InitiatingProcessCommandLine) contains 'vaultcmd' or tolower(InitiatingProcessCommandLine) contains 'keymgr.dll' or tolower(InitiatingProcessCommandLine) contains 'krshowkeymgr' | project TimeGenerated, DeviceName, FileName, ProcessId, ProcessCreationTime, ProcessCommandLine, InitiatingProcessFileName, InitiatingProcessCommandLine, InitiatingProcessAccountName, AccountName | order by TimeGenerated desc // Summarize by executable and device to quickly see which executable was used | summarize FirstSeen=min(TimeGenerated), LastSeen=max(TimeGenerated), Count=count(), ExampleCommand=any(ProcessCommandLine), InitiatingProcess=any(InitiatingProcessFileName) by FileName, DeviceName | order by LastSeen desc
Correct
1
26.61s
0.0070
gpt-5-mini-low
VaultCmd.exe
// Query to find attempts to list Windows Web Credentials (Credential Manager) using built-in tools let indicators = dynamic(["cmdkey.exe","vaultcmd.exe","vaultcmd","rundll32.exe","rundll32","Get-StoredCredential","Get-StoredCredentials","Get-StoredCred","Get-Credential","CredentialManager","KRShowKeyMgr","keymgr.dll","cmdkey /list","/listcreds","/list"]); let procEvents = DeviceProcessEvents | where isnotempty(FileName) | where tolower(FileName) has_any (pack_array("cmdkey.exe","vaultcmd.exe","rundll32.exe","powershell.exe","pwsh.exe")) | where (tolower(ProcessCommandLine) has "cmdkey" or tolower(ProcessCommandLine) has "vaultcmd" or tolower(ProcessCommandLine) has "krshowkeymgr" or tolower(ProcessCommandLine) has "keymgr.dll" or tolower(ProcessCommandLine) has "get-stored" or tolower(ProcessCommandLine) has "credentialmanager" or tolower(ProcessCommandLine) has "get-credential" or tolower(ProcessCommandLine) has "/list" or tolower(ProcessCommandLine) has "listcreds") | extend SourceTable = "DeviceProcessEvents", Executable = FileName, CmdLine = ProcessCommandLine, Time = TimeGenerated | project Time, DeviceName, DeviceId, AccountDomain, AccountName, Executable, ProcessId, CmdLine, InitiatingProcessFileName, InitiatingProcessCommandLine, InitiatingProcessId, InitiatingProcessAccountName, SourceTable; let otherEvents = DeviceEvents | where isnotempty(FileName) or isnotempty(ProcessCommandLine) | where (tolower(FileName) has_any (pack_array("cmdkey.exe","vaultcmd.exe","rundll32.exe","powershell.exe","pwsh.exe")) or tolower(ProcessCommandLine) has_any (pack_array("cmdkey","vaultcmd","krshowkeymgr","keymgr.dll","get-stored","credentialmanager","get-credential","/list","listcreds"))) | extend SourceTable = "DeviceEvents", Executable = FileName, CmdLine = iff(isnotempty(ProcessCommandLine), ProcessCommandLine, InitiatingProcessCommandLine), Time = TimeGenerated | project Time, DeviceName, DeviceId, AccountDomain, AccountName, Executable, ProcessId, CmdLine, InitiatingProcessFileName, InitiatingProcessCommandLine, InitiatingProcessId, InitiatingProcessAccountName, SourceTable; procEvents | union kind=inner otherEvents | extend MatchDetail = strcat("matched executable=", tostring(Executable), " cmdline=", tostring(CmdLine)) | sort by Time desc | distinct Time, DeviceName, DeviceId, AccountDomain, AccountName, Executable, ProcessId, CmdLine, InitiatingProcessFileName, InitiatingProcessCommandLine, InitiatingProcessId, InitiatingProcessAccountName, SourceTable, MatchDetail | take 500
Correct
1
620.06s
0.0039
gpt-5-mini-medium
No answer provided
No query generated
Incorrect
5
0.00s
0.0178
gpt-5-nano-high
VaultCmd.exe
let Timerange = 30d; DeviceProcessEvents | where TimeGenerated >= ago(Timerange) | where tolower(InitiatingProcessFileName) endswith ".exe" | where ( tolower(InitiatingProcessFileName) in ('rundll32.exe','vaultcmd.exe','cmdkey.exe') or tolower(InitiatingProcessCommandLine) contains 'krshowkeymgr' or tolower(InitiatingProcessCommandLine) contains 'vaultcmd' or tolower(InitiatingProcessCommandLine) contains 'cmdkey' or (tolower(InitiatingProcessFileName) == 'rundll32.exe' and tolower(InitiatingProcessCommandLine) contains 'keymgr.dll,krshowkeymgr') ) | project TimeGenerated, DeviceName, DeviceId, InitiatingProcessFileName, InitiatingProcessCommandLine, ProcessCommandLine, ProcessId, InitiatingProcessId, InitiatingProcessFolderPath, InitiatingProcessParentFileName, InitiatingProcessParentId, InitiatingProcessCreationTime, AccountDomain, AccountName | sort by TimeGenerated asc
Correct
5
104.42s
0.0069
gpt-5-nano-medium
No answer provided
No query generated
Incorrect
5
0.00s
0.0076
grok-3-beta
VaultCmd.exe
DeviceProcessEvents | where FileName =~ "vaultcmd.exe" | project DeviceName, AccountName, ProcessCreationTime, FileName, ProcessCommandLine, FolderPath | order by ProcessCreationTime desc | limit 100
Correct
1
8.28s
0.0203
grok-3-mini-beta
cmdkey.exe
DeviceProcessEvents | where ProcessCommandLine contains 'cmdkey' and ProcessCommandLine contains '/list' | project TimeGenerated, DeviceName, AccountName, ProcessCommandLine, FileName | order by TimeGenerated desc
Incorrect
1
11.57s
0.0016
o1-high
VaultCmd.exe
DeviceProcessEvents | where ActionType == "ProcessCreated" | where FileName in~ ("vaultcmd.exe", "cmdkey.exe") or ProcessCommandLine has_any ("vaultcmd", "cmdkey", "/listcreds", "/list") | project TimeGenerated, DeviceName, FolderPath, FileName, ProcessCommandLine, AccountDomain, AccountName, AccountSid, InitiatingProcessFileName, InitiatingProcessCommandLine, InitiatingProcessAccountName | order by TimeGenerated desc
Correct
1
49.82s
0.1931
o1-low
cmdkey.exe
DeviceProcessEvents | where Timestamp between (startofday(ago(7d)) .. now()) | where FileName in~ ("cmdkey.exe", "vaultcmd.exe", "rundll32.exe") | where ProcessCommandLine has_any ("/list", "KRShowKeyMgr") | project Timestamp, DeviceName, FileName, ProcessCommandLine, AccountDomain, AccountName, InitiatingProcessAccountDomain, InitiatingProcessAccountName | order by Timestamp desc
Incorrect
1
19.70s
0.1764
o3-high
cmdkey.exe
// Process creations that attempted to enumerate the Web Credentials vault DeviceProcessEvents | where ActionType == "ProcessCreated" // --- Candidate built-in executables -------------------------------------- | where FileName has_cs "vaultcmd.exe" or FileName has_cs "cmdkey.exe" // --- Look for listing operations ----------------------------------------- | where ProcessCommandLine has_cs "/list" or ProcessCommandLine has_cs "/listcreds:" // --- Narrow down to the Web Credentials vault if explicitly referenced --- | extend TargetVault = iff(ProcessCommandLine matches regex @"(?i)web\s*credentials", "Web Credentials", "Other / Unspecified") // ------------------------------------------------------------------------- | project TimeGenerated, DeviceName, AccountName, FileName, ProcessCommandLine, TargetVault, InitiatingProcessFileName, InitiatingProcessCommandLine | order by TimeGenerated desc // Quick summary – which executable was used the most | summarize Executions = count() by FileName, TargetVault | order by Executions desc
Incorrect
1
31.76s
0.0202
o3-mini-high
VaultCmd.exe
let searchKeyword = "vaultcmd"; let listKeyword = "list"; // Search in DeviceProcessEvents let processEvents = DeviceProcessEvents | where ActionType == "ProcessCreated" | where (tolower(InitiatingProcessCommandLine) contains searchKeyword or tolower(ProcessCommandLine) contains searchKeyword) and (tolower(InitiatingProcessCommandLine) contains listKeyword or tolower(ProcessCommandLine) contains listKeyword) | project TimeGenerated, DeviceName, InitiatingProcessFileName, InitiatingProcessCommandLine, ProcessCommandLine, ProcessId; // Also check DeviceEvents in case the event is logged there let deviceEvents = DeviceEvents | where ActionType == "ProcessCreated" | where (tolower(InitiatingProcessCommandLine) contains searchKeyword or tolower(ProcessCommandLine) contains searchKeyword) and (tolower(InitiatingProcessCommandLine) contains listKeyword or tolower(ProcessCommandLine) contains listKeyword) | project TimeGenerated, DeviceName, InitiatingProcessFileName, InitiatingProcessCommandLine, ProcessCommandLine, ProcessId; processEvents | union deviceEvents | order by TimeGenerated desc
Correct
3
59.89s
0.0325
o3-mini-low
cmdkey.exe
DeviceProcessEvents | where ProcessCommandLine has "cmdkey" and ProcessCommandLine has "/list" | project TimeGenerated, DeviceName, FileName, ProcessCommandLine, ProcessId, InitiatingProcessId | order by TimeGenerated desc
Incorrect
1
32.85s
0.0109
o4-mini-high
No answer provided
No query generated
Incorrect
5
0.00s
0.0519
o4-mini-low
No answer provided
No query generated
Incorrect
5
0.00s
0.0499