Question Analysis
A security investigator suspects that someone attempted to dump stored web credentials on a Windows system using an in-built command-line tool. Review process creation logs to determine which executable was called to list the Web Credentials vault.
bc071188-459f-44d5-901a-f8f2625b2d2e
Question ID
8/16
Models Correct
1
Valid Answer
50%
Success Rate
Reference Answer
Human-validated correct answer for this question
vaultcmd.exe
Model Performance Comparison
See how different AI models performed on this question
| Model | Answer | KQL Query | Status | Attempts | Time | Cost |
|---|---|---|---|---|---|---|
| gemini-2.5-flash-preview-04-17 | VaultCmd.exe | DeviceProcessEvents
| where FileName =~ "vaultcmd.exe"
| project Timestamp, DeviceName, AccountName, InitiatingProcessFileName, InitiatingProcessCommandLine, FileName, ProcessCommandLine | Correct | 3 | 31.05s | 0.0164 |
| gpt-35-turbo | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0112 |
| gpt-4-turbo-2024-04-09 | cmdkey.exe | DeviceProcessEvents
| where ActionType == 'ProcessCreated'
| where ProcessCommandLine has 'cmdkey' and ProcessCommandLine has '/list'
| project DeviceName, AccountName, ProcessCommandLine, ProcessCreationTime
| order by ProcessCreationTime desc | Incorrect | 1 | 10.99s | 0.0531 |
| gpt-4.1 | VaultCmd.exe | DeviceProcessEvents
| where FileName in~ ("cmdkey.exe", "vaultcmd.exe") or ProcessCommandLine has_any ("cmdkey.exe", "vaultcmd.exe", "Web Credentials", "list")
| project TimeGenerated, DeviceName, AccountDomain, AccountName, FileName, ProcessCommandLine, FolderPath, InitiatingProcessAccountName, InitiatingProcessCommandLine
| order by TimeGenerated desc | Correct | 1 | 5.71s | 0.0297 |
| gpt-4.1-finetuned | VaultCmd.exe | DeviceProcessEvents
| where (FileName contains "cmdkey.exe" or FileName contains "cmdkey" or ProcessCommandLine contains "/list") or (FileName contains "rundll32.exe" or ProcessCommandLine contains "keymgr.dll,KRShowKeyMgr")
| Correct | 1 | 8.26s | 0.0307 |
| gpt-4.1-mini | VaultCmd.exe | DeviceProcessEvents
| where FileName in~ ("cmdkey.exe", "vaultcmd.exe") or InitiatingProcessFileName in~ ("cmdkey.exe", "vaultcmd.exe") or ProcessCommandLine contains_cs "vaultcmd"
| project TimeGenerated, DeviceName, AccountName, FileName, ProcessId, InitiatingProcessFileName, ProcessCommandLine
| order by TimeGenerated desc | Correct | 1 | 5.81s | 0.0024 |
| gpt-4.1-nano | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0013 |
| gpt-4o | VaultCmd.exe | DeviceProcessEvents
| where ProcessCommandLine contains "vaultcmd" or FileName == "vaultcmd.exe"
| project TimeGenerated, DeviceName, AccountName, FileName, ProcessCommandLine, InitiatingProcessAccountName, InitiatingProcessCommandLine | Correct | 1 | 3.45s | 0.0155 |
| grok-3-beta | VaultCmd.exe | DeviceProcessEvents
| where FileName =~ "vaultcmd.exe"
| project DeviceName, AccountName, ProcessCreationTime, FileName, ProcessCommandLine, FolderPath
| order by ProcessCreationTime desc
| limit 100 | Correct | 1 | 8.28s | 0.0203 |
| grok-3-mini-beta | cmdkey.exe | DeviceProcessEvents
| where ProcessCommandLine contains 'cmdkey' and ProcessCommandLine contains '/list'
| project TimeGenerated, DeviceName, AccountName, ProcessCommandLine, FileName
| order by TimeGenerated desc | Incorrect | 1 | 11.57s | 0.0016 |
| o1-high | VaultCmd.exe | DeviceProcessEvents
| where ActionType == "ProcessCreated"
| where FileName in~ ("vaultcmd.exe", "cmdkey.exe")
or ProcessCommandLine has_any ("vaultcmd", "cmdkey", "/listcreds", "/list")
| project TimeGenerated, DeviceName, FolderPath, FileName, ProcessCommandLine,
AccountDomain, AccountName, AccountSid, InitiatingProcessFileName,
InitiatingProcessCommandLine, InitiatingProcessAccountName
| order by TimeGenerated desc | Correct | 1 | 49.82s | 0.1931 |
| o1-low | cmdkey.exe | DeviceProcessEvents
| where Timestamp between (startofday(ago(7d)) .. now())
| where FileName in~ ("cmdkey.exe", "vaultcmd.exe", "rundll32.exe")
| where ProcessCommandLine has_any ("/list", "KRShowKeyMgr")
| project Timestamp, DeviceName, FileName, ProcessCommandLine, AccountDomain, AccountName, InitiatingProcessAccountDomain, InitiatingProcessAccountName
| order by Timestamp desc | Incorrect | 1 | 19.70s | 0.1764 |
| o3-mini-high | VaultCmd.exe | let searchKeyword = "vaultcmd";
let listKeyword = "list";
// Search in DeviceProcessEvents
let processEvents = DeviceProcessEvents
| where ActionType == "ProcessCreated"
| where (tolower(InitiatingProcessCommandLine) contains searchKeyword or tolower(ProcessCommandLine) contains searchKeyword)
and (tolower(InitiatingProcessCommandLine) contains listKeyword or tolower(ProcessCommandLine) contains listKeyword)
| project TimeGenerated, DeviceName, InitiatingProcessFileName, InitiatingProcessCommandLine, ProcessCommandLine, ProcessId;
// Also check DeviceEvents in case the event is logged there
let deviceEvents = DeviceEvents
| where ActionType == "ProcessCreated"
| where (tolower(InitiatingProcessCommandLine) contains searchKeyword or tolower(ProcessCommandLine) contains searchKeyword)
and (tolower(InitiatingProcessCommandLine) contains listKeyword or tolower(ProcessCommandLine) contains listKeyword)
| project TimeGenerated, DeviceName, InitiatingProcessFileName, InitiatingProcessCommandLine, ProcessCommandLine, ProcessId;
processEvents
| union deviceEvents
| order by TimeGenerated desc
| Correct | 3 | 59.89s | 0.0325 |
| o3-mini-low | cmdkey.exe | DeviceProcessEvents
| where ProcessCommandLine has "cmdkey" and ProcessCommandLine has "/list"
| project TimeGenerated, DeviceName, FileName, ProcessCommandLine, ProcessId, InitiatingProcessId
| order by TimeGenerated desc | Incorrect | 1 | 32.85s | 0.0109 |
| o4-mini-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0519 |
| o4-mini-low | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0499 |