Question Analysis
On Windows systems, identify when the built-in Shadow Copy utility is used to enumerate existing snapshots. What was the full command executed?
7161b085-816a-491f-bab4-d68e974b7995
Question ID
18/23
Models Correct
1
Valid Answer
78%
Success Rate
Reference Answer
Human-validated correct answer for this question
vssadmin.exe list shadows
Model Performance Comparison
See how different AI models performed on this question
Model | Answer | KQL Query | Status | Attempts | Time | Cost |
---|---|---|---|---|---|---|
gemini-2.5-flash-preview-04-17 | "vssadmin.exe" list shadows | DeviceProcessEvents
| where FileName =~ "vssadmin.exe"
| where ProcessCommandLine has "list shadows"
| project Timestamp, DeviceName, ProcessCommandLine | Correct | 1 | 6.68s | 0.0023 |
gpt-35-turbo | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0109 |
gpt-4-turbo-2024-04-09 | "vssadmin.exe" list shadows | DeviceProcessEvents
| where FileName == 'vssadmin.exe' and ProcessCommandLine contains 'list shadows'
| project TimeGenerated, DeviceName, ProcessCommandLine | Correct | 1 | 8.21s | 0.0503 |
gpt-4.1 | "vssadmin.exe" list shadows | DeviceProcessEvents
| where FileName =~ "vssadmin.exe"
| where ProcessCommandLine has_any ("list shadows", "list shadowstorage")
| project TimeGenerated, DeviceName, AccountName, FileName, ProcessCommandLine | Correct | 1 | 5.42s | 0.0107 |
gpt-4.1-finetuned | "vssadmin.exe" list shadows | DeviceProcessEvents
| where FileName =~ "vssadmin.exe" and (ProcessCommandLine contains "list" and ProcessCommandLine contains "shad") | Correct | 1 | 6.34s | 0.0141 |
gpt-4.1-mini | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0071 |
gpt-4.1-nano | vssadmin.exe list shadows | DeviceProcessEvents
| where ProcessCommandLine has_any ('vssadmin', 'vssadmin list shadows', 'shadowcopy', 'shadow copy')
| where ProcessCommandLine contains 'list shadows' or ProcessCommandLine contains 'list shadows'
| project TimeGenerated, DeviceName, InitiatingProcessCommandLine, InitiatingProcessFileName | Correct | 1 | 3.04s | 0.0006 |
gpt-4o | vssadmin.exe list shadows | DeviceProcessEvents
| where FileName == "vssadmin.exe" and ProcessCommandLine contains "list shadows"
| project TimeGenerated, DeviceName, AccountName, FileName, ProcessCommandLine | Correct | 1 | 3.95s | 0.0145 |
gpt-5-high | "vssadmin.exe" list shadows | // Shadow Copy enumeration: vssadmin list shadows (and wmic shadowcopy list)
let vss_list = DeviceProcessEvents
| where ActionType == "ProcessCreated"
| where FileName =~ "vssadmin.exe"
| where ProcessCommandLine matches regex @"(?i)\blist\s+shadows?\b"; // enumerate existing snapshots
let wmic_list = DeviceProcessEvents
| where ActionType == "ProcessCreated"
| where FileName =~ "wmic.exe"
| where ProcessCommandLine matches regex @"(?i)\bwmic(\.exe)?\s+shadowcopy\s+list\b"; // alternate enumeration
union vss_list, wmic_list
| project TimeGenerated, DeviceId, DeviceName, AccountDomain, AccountName, FileName, FolderPath, ProcessCommandLine
| extend FullCommand = coalesce(ProcessCommandLine, strcat(FolderPath, "\\", FileName))
| order by TimeGenerated desc | Correct | 1 | 58.54s | 0.0378 |
gpt-5-mini-high | "vssadmin.exe" list shadows | union isfuzzy=true DeviceProcessEvents, DeviceEvents, DeviceFileEvents
| extend CommandLine = coalesce(ProcessCommandLine, InitiatingProcessCommandLine)
| where isnotempty(CommandLine)
| where (
(tolower(CommandLine) contains "vssadmin" and (tolower(CommandLine) contains "list" or tolower(CommandLine) contains "shadows" or tolower(CommandLine) contains "shadowstorage"))
or tolower(CommandLine) contains "wmic shadowcopy"
or ((tolower(CommandLine) contains "get-wmiobject" or tolower(CommandLine) contains "get-ciminstance" or tolower(CommandLine) contains "gwmi") and tolower(CommandLine) contains "shadow")
)
| project TimeGenerated, DeviceName, AccountName = coalesce(InitiatingProcessAccountName, AccountName), UserPrincipal = coalesce(InitiatingProcessAccountUpn, AccountUpn), ProcessId = coalesce(ProcessId, InitiatingProcessId), FileName = coalesce(FileName, InitiatingProcessFileName), CommandLine, ReportId, TenantId
| sort by TimeGenerated desc | Correct | 1 | 30.54s | 0.0075 |
gpt-5-mini-low | "vssadmin.exe" list shadows | union DeviceProcessEvents, DeviceEvents, DeviceFileEvents, DeviceImageLoadEvents
| where FileName has_any ("vssadmin.exe", "diskshadow.exe", "wmic.exe")
or ProcessCommandLine has_any ("vssadmin", "diskshadow", "list shadows", "list shadow", "shadowcopy", "shadowcopies", "wmic shadowcopy", "shadow copy")
or InitiatingProcessCommandLine has_any ("vssadmin", "diskshadow", "list shadows", "list shadow", "shadowcopy", "shadowcopies", "wmic shadowcopy", "shadow copy")
| extend EventTime = coalesce(TimeGenerated, Timestamp)
| extend FullCommand = coalesce(ProcessCommandLine, InitiatingProcessCommandLine)
// Keep records that are likely to be enumerating snapshots
| where FullCommand has_any ("list shadows", "shadowcopy", "shadowcopies") or FileName has_any ("vssadmin.exe", "diskshadow.exe")
| project EventTime, DeviceName, Account = coalesce(InitiatingProcessAccountName, AccountName), FileName, ProcessId = coalesce(ProcessId, InitiatingProcessId), InitiatingProcessFileName, FullCommand, ReportId
| sort by EventTime desc | Correct | 1 | 38.93s | 0.0097 |
gpt-5-mini-medium | "powershell.exe" & {if(!(vssadmin.exe list shadows | findstr \"\"No items found that satisfy the query.\"\")) { exit 0 } else { exit 1 }} | // Detect usage of vssadmin to enumerate shadows
let lookback = 30d; // actual time range will be set by caller/timepicker
let p1 = DeviceProcessEvents
| where TimeGenerated >= ago(lookback)
| where tolower(InitiatingProcessFileName) has "vssadmin" or tolower(ProcessCommandLine) has "vssadmin"
| where tolower(InitiatingProcessCommandLine) has "list" or tolower(InitiatingProcessCommandLine) has "shadows" or tolower(ProcessCommandLine) has "list" or tolower(ProcessCommandLine) has "shadows"
| extend FullCommand = coalesce(InitiatingProcessCommandLine, ProcessCommandLine)
| project TimeGenerated, DeviceName, InitiatingProcessAccountName, InitiatingProcessId, InitiatingProcessFileName, FullCommand, ReportId, TenantId, SourceSystem, _ResourceId;
let p2 = DeviceEvents
| where TimeGenerated >= ago(lookback)
| where tolower(FileName) has "vssadmin" or tolower(ProcessCommandLine) has "vssadmin" or tolower(InitiatingProcessFileName) has "vssadmin"
| extend FullCommand = coalesce(ProcessCommandLine, InitiatingProcessCommandLine, AdditionalFields.CommandLine, tostring(AdditionalFields.CommandLine))
| where tolower(FullCommand) has "list" or tolower(FullCommand) has "shadows"
| project TimeGenerated, DeviceName, InitiatingProcessAccountName, InitiatingProcessId = coalesce(InitiatingProcessId, ProcessId), InitiatingProcessFileName = coalesce(InitiatingProcessFileName, FileName), FullCommand, ReportId, TenantId, SourceSystem, _ResourceId;
union p1, p2
| sort by TimeGenerated desc
| distinct TimeGenerated, DeviceName, InitiatingProcessAccountName, InitiatingProcessId, InitiatingProcessFileName, FullCommand, ReportId, TenantId, SourceSystem, _ResourceId
| where isnotempty(FullCommand)
| extend CommandLower = tolower(FullCommand)
| where CommandLower has "vssadmin" and (CommandLower has "list" or CommandLower has "shadows" or CommandLower has "shadow")
| project TimeGenerated, DeviceName, InitiatingProcessAccountName, InitiatingProcessId, InitiatingProcessFileName, FullCommand
| take 100
| Correct | 1 | 23.32s | 0.0052 |
gpt-5-nano-high | "powershell.exe" & {if(!(vssadmin.exe list shadows | findstr \\\"No items found that satisfy the query.\\\"")) { exit 0 } else { exit 1 }} | DeviceProcessEvents
| where InitiatingProcessFileName contains "vssadmin" or tolower(InitiatingProcessCommandLine) contains "vssadmin"
| where tolower(InitiatingProcessCommandLine) contains "list shadows"
| extend FullCommand = InitiatingProcessCommandLine
| project TimeGenerated, DeviceName, InitiatingProcessAccountName, InitiatingProcessFileName, InitiatingProcessCommandLine, ProcessCommandLine, FullCommand
| sort by TimeGenerated desc | Correct | 2 | 53.65s | 0.0033 |
gpt-5-nano-medium | vssadmin.exe list shadows | let timeWindow = 1d;
DeviceProcessEvents
| where TimeGenerated >= ago(timeWindow)
| where InitiatingProcessCommandLine has_cs "vssadmin" and InitiatingProcessCommandLine has_cs "list" and InitiatingProcessCommandLine has_cs "shadows"
| project TimeGenerated, DeviceName, DeviceId, InitiatingProcessId, InitiatingProcessFileName, InitiatingProcessCommandLine, ProcessCommandLine | Correct | 1 | 34.04s | 0.0024 |
grok-3-beta | vssadmin.exe list shadows | DeviceProcessEvents
| where FileName == 'vssadmin.exe' and ProcessCommandLine contains 'list shadows'
| project Timestamp, DeviceName, AccountName, ProcessCommandLine, InitiatingProcessFileName, InitiatingProcessAccountName
| order by Timestamp desc | Correct | 1 | 8.48s | 0.0206 |
grok-3-mini-beta | "vssadmin.exe" list shadows | DeviceProcessEvents
| where FileName == "vssadmin.exe"
| where tolower(ProcessCommandLine) contains "list shadows"
| project Timestamp, DeviceName, AccountName, ProcessCommandLine
| order by Timestamp desc | Correct | 1 | 16.17s | 0.0016 |
o1-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.6543 |
o1-low | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.6739 |
o3-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0859 |
o3-mini-high | vssadmin.exe list shadows | DeviceProcessEvents
| where tolower(FileName) == "vssadmin.exe" or tolower(InitiatingProcessFileName) == "vssadmin.exe"
| where tolower(ProcessCommandLine) has "list shadows" or tolower(InitiatingProcessCommandLine) has "list shadows"
| extend fullCommand = iif(isnotempty(ProcessCommandLine), ProcessCommandLine, InitiatingProcessCommandLine)
| project TimeGenerated, DeviceName, fullCommand
| order by TimeGenerated desc | Correct | 2 | 33.34s | 0.0201 |
o3-mini-low | vssadmin.exe list shadows | DeviceProcessEvents
| where (InitiatingProcessFileName =~ "vssadmin.exe" or FileName =~ "vssadmin.exe")
| where tolower(iif(isnotempty(ProcessCommandLine), ProcessCommandLine, InitiatingProcessCommandLine)) contains "list shadows"
| project TimeGenerated, DeviceName, FullCommand = iif(isnotempty(ProcessCommandLine), ProcessCommandLine, InitiatingProcessCommandLine)
| order by TimeGenerated desc | Correct | 2 | 39.17s | 0.0175 |
o4-mini-high | vssadmin.exe list shadows | let ShadowTools = dynamic(["vssadmin.exe","diskshadow.exe","wmic.exe"]);
DeviceProcessEvents
| where FileName in~ (ShadowTools)
| where ProcessCommandLine matches regex @"(?i)\b(list|get|enum)\b.*\b(shadow|shadowcopy)s?\b"
| project TimeGenerated, DeviceName, User = InitiatingProcessAccountName, Executable = FileName, CommandLine = ProcessCommandLine
| sort by TimeGenerated desc | Correct | 4 | 152.83s | 0.0418 |
o4-mini-low | vssadmin.exe list shadows | DeviceProcessEvents
// Filter for built-in Shadow Copy utilities
| where FileName in ("vssadmin.exe", "diskshadow.exe")
// Look for commands that enumerate snapshots
| where tolower(ProcessCommandLine) has "list shadows"
// Project the relevant details
| project TimeGenerated, DeviceName, InitiatingProcessAccountName, FileName, ProcessCommandLine
| sort by TimeGenerated desc | Correct | 1 | 23.91s | 0.0100 |