Question Analysis
Investigate Windows process execution logs for a PowerShell cmdlet used to list group members. Look for entries where a group name is provided after a '-Name' flag and identify which group was queried.
a580462d-2c19-4bc7-8b9a-57a41b7d3ba4
Question ID
10/23
Models Correct
1
Valid Answer
43%
Success Rate
Reference Answer
Human-validated correct answer for this question
Administrators
Model Performance Comparison
See how different AI models performed on this question
Model | Answer | KQL Query | Status | Attempts | Time | Cost |
---|---|---|---|---|---|---|
gemini-2.5-flash-preview-04-17 | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0320 |
gpt-35-turbo | No | DeviceProcessEvents | where ProcessCommandLine contains '-Name' | extend GroupName = tostring(split(ProcessCommandLine, '-Name ')[1]) | project GroupName, DeviceName, TimeGenerated | Incorrect | 3 | 9.10s | 0.0086 |
gpt-4-turbo-2024-04-09 | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.2370 |
gpt-4.1 | Administrators | DeviceProcessEvents
| where ProcessCommandLine has_any ('powershell', 'pwsh')
and (ProcessCommandLine has 'Get-ADGroupMember' or ProcessCommandLine has 'Get-LocalGroupMember')
and ProcessCommandLine has '-Name'
| extend QueriedGroup = extract("-Name ['\"]*([^'\" ]+)['\"]*", 1, ProcessCommandLine)
| where isnotempty(QueriedGroup)
| project TimeGenerated, DeviceName, AccountName, InitiatingProcessAccountName, ProcessCommandLine, QueriedGroup | Correct | 3 | 13.65s | 0.0274 |
gpt-4.1-finetuned | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0893 |
gpt-4.1-mini | Administrators | DeviceProcessEvents
| where ProcessCommandLine has "-Name"
and (ProcessCommandLine has "Get-LocalGroupMember" or ProcessCommandLine has "Get-ADGroupMember" or ProcessCommandLine has "Get-GroupMember" or ProcessCommandLine has "Get-Group")
| extend GroupName = extract("-Name\\s+['\"]?([\\w\\d\\\\._-]+)['\"]?", 1, ProcessCommandLine)
| where isnotempty(GroupName)
| project TimeGenerated, DeviceName, AccountName, ProcessCreationTime, ProcessCommandLine, GroupName
| order by TimeGenerated desc | Correct | 3 | 26.23s | 0.0055 |
gpt-4.1-nano | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0013 |
gpt-4o | Administrators | DeviceProcessEvents
| where ProcessCommandLine has "-Name"
| extend GroupName = extract("-Name\\s+(['\"]?)(.*?)(['\"]?\\s|$)", 2, ProcessCommandLine)
| where isnotempty(GroupName)
| project TimeGenerated, AccountName, DeviceName, ProcessCommandLine, GroupName | Correct | 2 | 9.53s | 0.0390 |
gpt-5-high | Administrators | DeviceProcessEvents
| where FileName in~ ("powershell.exe", "pwsh.exe", "powershell_ise.exe")
| where ProcessCommandLine has_any ("Get-LocalGroupMember","Get-ADGroupMember","Get-DistributionGroupMember","Get-AzureADGroupMember","Get-MgGroupMember","Get-LocalGroup")
| where ProcessCommandLine has "-Name"
| extend GroupName = coalesce(
extract(@'(?i)-name\s*(?::|=)?\s*"([^"]+)"', 1, ProcessCommandLine),
extract(@'(?i)-name\s*(?::|=)?\s*''([^'']+)''', 1, ProcessCommandLine),
extract(@'(?i)-name\s*(?::|=)?\s*([^\s\|\;]+)', 1, ProcessCommandLine)
)
| where isnotempty(GroupName)
| extend Cmdlet = case(
ProcessCommandLine matches regex @'(?i)\bGet-LocalGroupMember\b', 'Get-LocalGroupMember',
ProcessCommandLine matches regex @'(?i)\bGet-ADGroupMember\b', 'Get-ADGroupMember',
ProcessCommandLine matches regex @'(?i)\bGet-AzureADGroupMember\b', 'Get-AzureADGroupMember',
ProcessCommandLine matches regex @'(?i)\bGet-DistributionGroupMember\b', 'Get-DistributionGroupMember',
ProcessCommandLine matches regex @'(?i)\bGet-MgGroupMember\b', 'Get-MgGroupMember',
ProcessCommandLine matches regex @'(?i)\bGet-LocalGroup\b', 'Get-LocalGroup',
'Unknown')
| project TimeGenerated, DeviceName, AccountDomain, AccountName, FileName, Cmdlet, GroupName, ProcessCommandLine, InitiatingProcessParentFileName, InitiatingProcessAccountDomain, InitiatingProcessAccountName
| order by TimeGenerated desc | Correct | 2 | 172.68s | 0.0934 |
gpt-5-mini-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0259 |
gpt-5-mini-low | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0226 |
gpt-5-mini-medium | Administrators | let nameRegex = @"(?i)(?:-Name|-Identity)\s+(?:'([^']+)'|""([^""]+)""|([^\s)]+))";
union DeviceProcessEvents, DeviceEvents
| extend CmdLine = coalesce(ProcessCommandLine, InitiatingProcessCommandLine)
| where isnotempty(CmdLine)
// focus on likely PowerShell usage or explicit cmdlet mentions
| where CmdLine has_any ("Get-ADGroupMember","Get-LocalGroupMember","Get-ADGroup","Get-ADPrincipalGroupMembership","Get-LocalGroup","Get-GroupMember")
or CmdLine has_any ("powershell.exe","pwsh.exe","PowerShell_ISE.exe")
// capture the cmdlet if present
| extend Cmdlet = extract(@"(?i)\b(Get-ADGroupMember|Get-LocalGroupMember|Get-ADGroup|Get-ADPrincipalGroupMembership|Get-LocalGroup|Get-GroupMember)\b", 1, CmdLine)
// extract group name after -Name or -Identity (handles 'quoted', ""double-quoted"", and unquoted tokens)
| extend g1 = extract(nameRegex, 1, CmdLine), g2 = extract(nameRegex, 2, CmdLine), g3 = extract(nameRegex, 3, CmdLine)
| extend GroupName = coalesce(g1, g2, g3)
| where isnotempty(GroupName)
| project TimeGenerated, DeviceName, DeviceId, AccountName, AccountDomain, InitiatingProcessAccountName, FileName, Cmdlet, GroupName, CmdLine, ReportId, TenantId
| sort by TimeGenerated desc
| take 200 | Correct | 4 | 61.62s | 0.0170 |
gpt-5-nano-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0095 |
gpt-5-nano-medium | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0107 |
grok-3-beta | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0907 |
grok-3-mini-beta | Administrators | DeviceProcessEvents
| where FileName endswith "powershell.exe" // More precise match for PowerShell executable
| where ProcessCommandLine contains "-Name" // Look for commands containing the -Name flag
| extend GroupName = extract("-Name\\s+['\"]?([^'\"\\s]+)['\"]?", 1, ProcessCommandLine) // Enhanced regex to capture the group name after -Name, handling optional quotes and avoiding extra spaces
| where isnotempty(GroupName) // Ensure only rows with a detected group name are kept
| project TimeGenerated, DeviceName, AccountName, ProcessCommandLine, GroupName // Project relevant fields
| order by TimeGenerated desc // Order by timestamp descending for latest events first | Correct | 3 | 33.10s | 0.0055 |
o1-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 1.0138 |
o1-low | None | DeviceProcessEvents
| where FileName =~ "powershell.exe"
| where ProcessCommandLine has "-Name"
| extend QueriedGroupQuoted = extract(@'(?)-Name\s+"([^\"]+)"', 1, ProcessCommandLine)
| extend QueriedGroupUnquoted = extract(@'(?)-Name\s+(\S+)', 1, ProcessCommandLine)
| extend QueriedGroup = coalesce(QueriedGroupQuoted, QueriedGroupUnquoted)
| project TimeGenerated, DeviceName, InitiatingProcessAccountName, ProcessCommandLine, QueriedGroup
| summarize QueriedCount = count() by QueriedGroup | Incorrect | 3 | 71.46s | 0.5645 |
o3-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0973 |
o3-mini-high | Administrators | DeviceProcessEvents
| where ProcessCommandLine has "powershell" and ProcessCommandLine has "-Name"
| extend GroupQueried = extract("-Name\\s+['\"']?([^ '\"']+)", 1, ProcessCommandLine)
| where isnotempty(GroupQueried)
| project TimeGenerated, DeviceName, AccountName, ProcessCommandLine, GroupQueried
| order by TimeGenerated desc | Correct | 2 | 14.26s | 0.0172 |
o3-mini-low | Administrators | DeviceProcessEvents
| where ProcessCommandLine contains "powershell" and ProcessCommandLine contains "-Name"
| extend Group1 = extract(@"-Name\s+'([^']+)'", 1, ProcessCommandLine)
| extend Group2 = extract(@"-Name\s+""([^""]+)""", 1, ProcessCommandLine)
| extend Group3 = extract(@"-Name\s+(\S+)", 1, ProcessCommandLine)
| extend GroupQueried = coalesce(Group1, Group2, Group3)
| where isnotempty(GroupQueried)
| project TimeGenerated, DeviceName, AccountName, ProcessCommandLine, GroupQueried | Correct | 3 | 68.08s | 0.0298 |
o4-mini-high | Administrators | DeviceProcessEvents
| where FileName in~ ("powershell.exe","pwsh.exe")
| where ProcessCommandLine matches regex @"-Name\s+\S+"
| extend QueriedGroup = extract(@"-Name\s+(\S+)", 1, ProcessCommandLine)
| where isnotempty(QueriedGroup)
| project TimeGenerated, DeviceName, AccountName = InitiatingProcessAccountName, QueriedGroup, ProcessCommandLine
| sort by TimeGenerated desc | Correct | 1 | 91.05s | 0.0348 |
o4-mini-low | Administrators | DeviceProcessEvents
| where FileName in ("powershell.exe", "pwsh.exe")
| where ProcessCommandLine has "-Name"
| extend QueriedGroup = extract("(?i)-Name\\s+['\\\"]?([^'\\\"\\s]+)", 1, ProcessCommandLine)
| where isnotempty(QueriedGroup)
| project TimeGenerated, DeviceName, AccountName, FileName, ProcessCommandLine, QueriedGroup
| sort by TimeGenerated desc | Correct | 3 | 226.46s | 0.0340 |