Question Analysis
A security investigator suspects that someone attempted to dump stored web credentials on a Windows system using an in-built command-line tool. Review process creation logs to determine which executable was called to list the Web Credentials vault.
bc071188-459f-44d5-901a-f8f2625b2d2e
Question ID
12/23
Models Correct
1
Valid Answer
52%
Success Rate
Reference Answer
Human-validated correct answer for this question
vaultcmd.exe
Model Performance Comparison
See how different AI models performed on this question
Model | Answer | KQL Query | Status | Attempts | Time | Cost |
---|---|---|---|---|---|---|
gemini-2.5-flash-preview-04-17 | VaultCmd.exe | DeviceProcessEvents
| where FileName =~ "vaultcmd.exe"
| project Timestamp, DeviceName, AccountName, InitiatingProcessFileName, InitiatingProcessCommandLine, FileName, ProcessCommandLine | Correct | 3 | 31.05s | 0.0164 |
gpt-35-turbo | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0112 |
gpt-4-turbo-2024-04-09 | cmdkey.exe | DeviceProcessEvents
| where ActionType == 'ProcessCreated'
| where ProcessCommandLine has 'cmdkey' and ProcessCommandLine has '/list'
| project DeviceName, AccountName, ProcessCommandLine, ProcessCreationTime
| order by ProcessCreationTime desc | Incorrect | 1 | 10.99s | 0.0531 |
gpt-4.1 | VaultCmd.exe | DeviceProcessEvents
| where FileName in~ ("cmdkey.exe", "vaultcmd.exe") or ProcessCommandLine has_any ("cmdkey.exe", "vaultcmd.exe", "Web Credentials", "list")
| project TimeGenerated, DeviceName, AccountDomain, AccountName, FileName, ProcessCommandLine, FolderPath, InitiatingProcessAccountName, InitiatingProcessCommandLine
| order by TimeGenerated desc | Correct | 1 | 5.71s | 0.0297 |
gpt-4.1-finetuned | VaultCmd.exe | DeviceProcessEvents
| where (FileName contains "cmdkey.exe" or FileName contains "cmdkey" or ProcessCommandLine contains "/list") or (FileName contains "rundll32.exe" or ProcessCommandLine contains "keymgr.dll,KRShowKeyMgr")
| Correct | 1 | 8.26s | 0.0307 |
gpt-4.1-mini | VaultCmd.exe | DeviceProcessEvents
| where FileName in~ ("cmdkey.exe", "vaultcmd.exe") or InitiatingProcessFileName in~ ("cmdkey.exe", "vaultcmd.exe") or ProcessCommandLine contains_cs "vaultcmd"
| project TimeGenerated, DeviceName, AccountName, FileName, ProcessId, InitiatingProcessFileName, ProcessCommandLine
| order by TimeGenerated desc | Correct | 1 | 5.81s | 0.0024 |
gpt-4.1-nano | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0013 |
gpt-4o | VaultCmd.exe | DeviceProcessEvents
| where ProcessCommandLine contains "vaultcmd" or FileName == "vaultcmd.exe"
| project TimeGenerated, DeviceName, AccountName, FileName, ProcessCommandLine, InitiatingProcessAccountName, InitiatingProcessCommandLine | Correct | 1 | 3.45s | 0.0155 |
gpt-5-high | VaultCmd.exe | let webVaultGuid = "4bf4c442-9b8a-41a0-b380-dd4a704ddb28";
let suspects = DeviceProcessEvents
| where ActionType == "ProcessCreated"
| extend cmd = tolower(coalesce(ProcessCommandLine, "")), fn = tolower(FileName)
| extend IsWebVaultMention = (cmd has webVaultGuid or cmd has "web credentials")
| where
(fn == "vaultcmd.exe" and (cmd has "/listcreds" or cmd has "/listproperties") and IsWebVaultMention)
or (fn == "cmdkey.exe" and cmd has "/list")
or (fn == "rundll32.exe" and cmd has "keymgr.dll,krshowkeymgr")
or (IsWebVaultMention and fn in ("powershell.exe","pwsh.exe","cmd.exe"));
// Summary: which executable(s) were used to list Web Credentials
suspects
| summarize FirstSeen=min(TimeGenerated), LastSeen=max(TimeGenerated), EventCount=count(), ExampleCommand=arg_max(TimeGenerated, ProcessCommandLine) by DeviceName, AccountDomain, AccountName, FileName, FolderPath
| order by LastSeen desc
// Details (uncomment to view raw events)
// suspects
// | project TimeGenerated, DeviceName, AccountDomain, AccountName, FileName, FolderPath, ProcessCommandLine, InitiatingProcessFileName, InitiatingProcessCommandLine, InitiatingProcessAccountDomain, InitiatingProcessAccountName, SHA256
// | order by TimeGenerated desc | Correct | 1 | 99.84s | 0.0572 |
gpt-5-mini-high | VaultCmd.exe | union isfuzzy=true DeviceProcessEvents, DeviceEvents
| where FileName in ('cmdkey.exe','vaultcmd.exe','rundll32.exe')
or tolower(ProcessCommandLine) contains 'cmdkey'
or tolower(ProcessCommandLine) contains 'vaultcmd'
or tolower(ProcessCommandLine) contains 'keymgr.dll'
or tolower(ProcessCommandLine) contains 'krshowkeymgr'
or tolower(InitiatingProcessCommandLine) contains 'cmdkey'
or tolower(InitiatingProcessCommandLine) contains 'vaultcmd'
or tolower(InitiatingProcessCommandLine) contains 'keymgr.dll'
or tolower(InitiatingProcessCommandLine) contains 'krshowkeymgr'
| project TimeGenerated, DeviceName, FileName, ProcessId, ProcessCreationTime, ProcessCommandLine, InitiatingProcessFileName, InitiatingProcessCommandLine, InitiatingProcessAccountName, AccountName
| order by TimeGenerated desc
// Summarize by executable and device to quickly see which executable was used
| summarize FirstSeen=min(TimeGenerated), LastSeen=max(TimeGenerated), Count=count(), ExampleCommand=any(ProcessCommandLine), InitiatingProcess=any(InitiatingProcessFileName) by FileName, DeviceName
| order by LastSeen desc | Correct | 1 | 26.61s | 0.0070 |
gpt-5-mini-low | VaultCmd.exe | // Query to find attempts to list Windows Web Credentials (Credential Manager) using built-in tools
let indicators = dynamic(["cmdkey.exe","vaultcmd.exe","vaultcmd","rundll32.exe","rundll32","Get-StoredCredential","Get-StoredCredentials","Get-StoredCred","Get-Credential","CredentialManager","KRShowKeyMgr","keymgr.dll","cmdkey /list","/listcreds","/list"]);
let procEvents = DeviceProcessEvents
| where isnotempty(FileName)
| where tolower(FileName) has_any (pack_array("cmdkey.exe","vaultcmd.exe","rundll32.exe","powershell.exe","pwsh.exe"))
| where (tolower(ProcessCommandLine) has "cmdkey" or tolower(ProcessCommandLine) has "vaultcmd" or tolower(ProcessCommandLine) has "krshowkeymgr" or tolower(ProcessCommandLine) has "keymgr.dll" or tolower(ProcessCommandLine) has "get-stored" or tolower(ProcessCommandLine) has "credentialmanager" or tolower(ProcessCommandLine) has "get-credential" or tolower(ProcessCommandLine) has "/list" or tolower(ProcessCommandLine) has "listcreds")
| extend
SourceTable = "DeviceProcessEvents",
Executable = FileName,
CmdLine = ProcessCommandLine,
Time = TimeGenerated
| project Time, DeviceName, DeviceId, AccountDomain, AccountName, Executable, ProcessId, CmdLine, InitiatingProcessFileName, InitiatingProcessCommandLine, InitiatingProcessId, InitiatingProcessAccountName, SourceTable;
let otherEvents = DeviceEvents
| where isnotempty(FileName) or isnotempty(ProcessCommandLine)
| where (tolower(FileName) has_any (pack_array("cmdkey.exe","vaultcmd.exe","rundll32.exe","powershell.exe","pwsh.exe"))
or tolower(ProcessCommandLine) has_any (pack_array("cmdkey","vaultcmd","krshowkeymgr","keymgr.dll","get-stored","credentialmanager","get-credential","/list","listcreds")))
| extend
SourceTable = "DeviceEvents",
Executable = FileName,
CmdLine = iff(isnotempty(ProcessCommandLine), ProcessCommandLine, InitiatingProcessCommandLine),
Time = TimeGenerated
| project Time, DeviceName, DeviceId, AccountDomain, AccountName, Executable, ProcessId, CmdLine, InitiatingProcessFileName, InitiatingProcessCommandLine, InitiatingProcessId, InitiatingProcessAccountName, SourceTable;
procEvents
| union kind=inner otherEvents
| extend MatchDetail = strcat("matched executable=", tostring(Executable), " cmdline=", tostring(CmdLine))
| sort by Time desc
| distinct Time, DeviceName, DeviceId, AccountDomain, AccountName, Executable, ProcessId, CmdLine, InitiatingProcessFileName, InitiatingProcessCommandLine, InitiatingProcessId, InitiatingProcessAccountName, SourceTable, MatchDetail
| take 500 | Correct | 1 | 620.06s | 0.0039 |
gpt-5-mini-medium | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0178 |
gpt-5-nano-high | VaultCmd.exe | let Timerange = 30d;
DeviceProcessEvents
| where TimeGenerated >= ago(Timerange)
| where tolower(InitiatingProcessFileName) endswith ".exe"
| where (
tolower(InitiatingProcessFileName) in ('rundll32.exe','vaultcmd.exe','cmdkey.exe')
or tolower(InitiatingProcessCommandLine) contains 'krshowkeymgr'
or tolower(InitiatingProcessCommandLine) contains 'vaultcmd'
or tolower(InitiatingProcessCommandLine) contains 'cmdkey'
or (tolower(InitiatingProcessFileName) == 'rundll32.exe' and tolower(InitiatingProcessCommandLine) contains 'keymgr.dll,krshowkeymgr')
)
| project TimeGenerated, DeviceName, DeviceId, InitiatingProcessFileName, InitiatingProcessCommandLine, ProcessCommandLine, ProcessId, InitiatingProcessId, InitiatingProcessFolderPath, InitiatingProcessParentFileName, InitiatingProcessParentId, InitiatingProcessCreationTime, AccountDomain, AccountName
| sort by TimeGenerated asc | Correct | 5 | 104.42s | 0.0069 |
gpt-5-nano-medium | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0076 |
grok-3-beta | VaultCmd.exe | DeviceProcessEvents
| where FileName =~ "vaultcmd.exe"
| project DeviceName, AccountName, ProcessCreationTime, FileName, ProcessCommandLine, FolderPath
| order by ProcessCreationTime desc
| limit 100 | Correct | 1 | 8.28s | 0.0203 |
grok-3-mini-beta | cmdkey.exe | DeviceProcessEvents
| where ProcessCommandLine contains 'cmdkey' and ProcessCommandLine contains '/list'
| project TimeGenerated, DeviceName, AccountName, ProcessCommandLine, FileName
| order by TimeGenerated desc | Incorrect | 1 | 11.57s | 0.0016 |
o1-high | VaultCmd.exe | DeviceProcessEvents
| where ActionType == "ProcessCreated"
| where FileName in~ ("vaultcmd.exe", "cmdkey.exe")
or ProcessCommandLine has_any ("vaultcmd", "cmdkey", "/listcreds", "/list")
| project TimeGenerated, DeviceName, FolderPath, FileName, ProcessCommandLine,
AccountDomain, AccountName, AccountSid, InitiatingProcessFileName,
InitiatingProcessCommandLine, InitiatingProcessAccountName
| order by TimeGenerated desc | Correct | 1 | 49.82s | 0.1931 |
o1-low | cmdkey.exe | DeviceProcessEvents
| where Timestamp between (startofday(ago(7d)) .. now())
| where FileName in~ ("cmdkey.exe", "vaultcmd.exe", "rundll32.exe")
| where ProcessCommandLine has_any ("/list", "KRShowKeyMgr")
| project Timestamp, DeviceName, FileName, ProcessCommandLine, AccountDomain, AccountName, InitiatingProcessAccountDomain, InitiatingProcessAccountName
| order by Timestamp desc | Incorrect | 1 | 19.70s | 0.1764 |
o3-high | cmdkey.exe | // Process creations that attempted to enumerate the Web Credentials vault
DeviceProcessEvents
| where ActionType == "ProcessCreated"
// --- Candidate built-in executables --------------------------------------
| where FileName has_cs "vaultcmd.exe" or FileName has_cs "cmdkey.exe"
// --- Look for listing operations -----------------------------------------
| where ProcessCommandLine has_cs "/list" or ProcessCommandLine has_cs "/listcreds:"
// --- Narrow down to the Web Credentials vault if explicitly referenced ---
| extend TargetVault = iff(ProcessCommandLine matches regex @"(?i)web\s*credentials", "Web Credentials", "Other / Unspecified")
// -------------------------------------------------------------------------
| project TimeGenerated,
DeviceName,
AccountName,
FileName,
ProcessCommandLine,
TargetVault,
InitiatingProcessFileName,
InitiatingProcessCommandLine
| order by TimeGenerated desc
// Quick summary – which executable was used the most
| summarize Executions = count() by FileName, TargetVault
| order by Executions desc | Incorrect | 1 | 31.76s | 0.0202 |
o3-mini-high | VaultCmd.exe | let searchKeyword = "vaultcmd";
let listKeyword = "list";
// Search in DeviceProcessEvents
let processEvents = DeviceProcessEvents
| where ActionType == "ProcessCreated"
| where (tolower(InitiatingProcessCommandLine) contains searchKeyword or tolower(ProcessCommandLine) contains searchKeyword)
and (tolower(InitiatingProcessCommandLine) contains listKeyword or tolower(ProcessCommandLine) contains listKeyword)
| project TimeGenerated, DeviceName, InitiatingProcessFileName, InitiatingProcessCommandLine, ProcessCommandLine, ProcessId;
// Also check DeviceEvents in case the event is logged there
let deviceEvents = DeviceEvents
| where ActionType == "ProcessCreated"
| where (tolower(InitiatingProcessCommandLine) contains searchKeyword or tolower(ProcessCommandLine) contains searchKeyword)
and (tolower(InitiatingProcessCommandLine) contains listKeyword or tolower(ProcessCommandLine) contains listKeyword)
| project TimeGenerated, DeviceName, InitiatingProcessFileName, InitiatingProcessCommandLine, ProcessCommandLine, ProcessId;
processEvents
| union deviceEvents
| order by TimeGenerated desc
| Correct | 3 | 59.89s | 0.0325 |
o3-mini-low | cmdkey.exe | DeviceProcessEvents
| where ProcessCommandLine has "cmdkey" and ProcessCommandLine has "/list"
| project TimeGenerated, DeviceName, FileName, ProcessCommandLine, ProcessId, InitiatingProcessId
| order by TimeGenerated desc | Incorrect | 1 | 32.85s | 0.0109 |
o4-mini-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0519 |
o4-mini-low | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0499 |