Question Analysis
Investigate Windows process execution logs for a PowerShell cmdlet used to list group members. Look for entries where a group name is provided after a '-Name' flag and identify which group was queried.
a580462d-2c19-4bc7-8b9a-57a41b7d3ba4
Question ID
8/16
Models Correct
1
Valid Answer
50%
Success Rate
Reference Answer
Human-validated correct answer for this question
Administrators
Model Performance Comparison
See how different AI models performed on this question
| Model | Answer | KQL Query | Status | Attempts | Time | Cost |
|---|---|---|---|---|---|---|
| gemini-2.5-flash-preview-04-17 | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0320 |
| gpt-35-turbo | No | DeviceProcessEvents | where ProcessCommandLine contains '-Name' | extend GroupName = tostring(split(ProcessCommandLine, '-Name ')[1]) | project GroupName, DeviceName, TimeGenerated | Incorrect | 3 | 9.10s | 0.0086 |
| gpt-4-turbo-2024-04-09 | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.2370 |
| gpt-4.1 | Administrators | DeviceProcessEvents
| where ProcessCommandLine has_any ('powershell', 'pwsh')
and (ProcessCommandLine has 'Get-ADGroupMember' or ProcessCommandLine has 'Get-LocalGroupMember')
and ProcessCommandLine has '-Name'
| extend QueriedGroup = extract("-Name ['\"]*([^'\" ]+)['\"]*", 1, ProcessCommandLine)
| where isnotempty(QueriedGroup)
| project TimeGenerated, DeviceName, AccountName, InitiatingProcessAccountName, ProcessCommandLine, QueriedGroup | Correct | 3 | 13.65s | 0.0274 |
| gpt-4.1-finetuned | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0893 |
| gpt-4.1-mini | Administrators | DeviceProcessEvents
| where ProcessCommandLine has "-Name"
and (ProcessCommandLine has "Get-LocalGroupMember" or ProcessCommandLine has "Get-ADGroupMember" or ProcessCommandLine has "Get-GroupMember" or ProcessCommandLine has "Get-Group")
| extend GroupName = extract("-Name\\s+['\"]?([\\w\\d\\\\._-]+)['\"]?", 1, ProcessCommandLine)
| where isnotempty(GroupName)
| project TimeGenerated, DeviceName, AccountName, ProcessCreationTime, ProcessCommandLine, GroupName
| order by TimeGenerated desc | Correct | 3 | 26.23s | 0.0055 |
| gpt-4.1-nano | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0013 |
| gpt-4o | Administrators | DeviceProcessEvents
| where ProcessCommandLine has "-Name"
| extend GroupName = extract("-Name\\s+(['\"]?)(.*?)(['\"]?\\s|$)", 2, ProcessCommandLine)
| where isnotempty(GroupName)
| project TimeGenerated, AccountName, DeviceName, ProcessCommandLine, GroupName | Correct | 2 | 9.53s | 0.0390 |
| grok-3-beta | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0907 |
| grok-3-mini-beta | Administrators | DeviceProcessEvents
| where FileName endswith "powershell.exe" // More precise match for PowerShell executable
| where ProcessCommandLine contains "-Name" // Look for commands containing the -Name flag
| extend GroupName = extract("-Name\\s+['\"]?([^'\"\\s]+)['\"]?", 1, ProcessCommandLine) // Enhanced regex to capture the group name after -Name, handling optional quotes and avoiding extra spaces
| where isnotempty(GroupName) // Ensure only rows with a detected group name are kept
| project TimeGenerated, DeviceName, AccountName, ProcessCommandLine, GroupName // Project relevant fields
| order by TimeGenerated desc // Order by timestamp descending for latest events first | Correct | 3 | 33.10s | 0.0055 |
| o1-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 1.0138 |
| o1-low | None | DeviceProcessEvents
| where FileName =~ "powershell.exe"
| where ProcessCommandLine has "-Name"
| extend QueriedGroupQuoted = extract(@'(?)-Name\s+"([^\"]+)"', 1, ProcessCommandLine)
| extend QueriedGroupUnquoted = extract(@'(?)-Name\s+(\S+)', 1, ProcessCommandLine)
| extend QueriedGroup = coalesce(QueriedGroupQuoted, QueriedGroupUnquoted)
| project TimeGenerated, DeviceName, InitiatingProcessAccountName, ProcessCommandLine, QueriedGroup
| summarize QueriedCount = count() by QueriedGroup | Incorrect | 3 | 71.46s | 0.5645 |
| o3-mini-high | Administrators | DeviceProcessEvents
| where ProcessCommandLine has "powershell" and ProcessCommandLine has "-Name"
| extend GroupQueried = extract("-Name\\s+['\"']?([^ '\"']+)", 1, ProcessCommandLine)
| where isnotempty(GroupQueried)
| project TimeGenerated, DeviceName, AccountName, ProcessCommandLine, GroupQueried
| order by TimeGenerated desc | Correct | 2 | 14.26s | 0.0172 |
| o3-mini-low | Administrators | DeviceProcessEvents
| where ProcessCommandLine contains "powershell" and ProcessCommandLine contains "-Name"
| extend Group1 = extract(@"-Name\s+'([^']+)'", 1, ProcessCommandLine)
| extend Group2 = extract(@"-Name\s+""([^""]+)""", 1, ProcessCommandLine)
| extend Group3 = extract(@"-Name\s+(\S+)", 1, ProcessCommandLine)
| extend GroupQueried = coalesce(Group1, Group2, Group3)
| where isnotempty(GroupQueried)
| project TimeGenerated, DeviceName, AccountName, ProcessCommandLine, GroupQueried | Correct | 3 | 68.08s | 0.0298 |
| o4-mini-high | Administrators | DeviceProcessEvents
| where FileName in~ ("powershell.exe","pwsh.exe")
| where ProcessCommandLine matches regex @"-Name\s+\S+"
| extend QueriedGroup = extract(@"-Name\s+(\S+)", 1, ProcessCommandLine)
| where isnotempty(QueriedGroup)
| project TimeGenerated, DeviceName, AccountName = InitiatingProcessAccountName, QueriedGroup, ProcessCommandLine
| sort by TimeGenerated desc | Correct | 1 | 91.05s | 0.0348 |
| o4-mini-low | Administrators | DeviceProcessEvents
| where FileName in ("powershell.exe", "pwsh.exe")
| where ProcessCommandLine has "-Name"
| extend QueriedGroup = extract("(?i)-Name\\s+['\\\"]?([^'\\\"\\s]+)", 1, ProcessCommandLine)
| where isnotempty(QueriedGroup)
| project TimeGenerated, DeviceName, AccountName, FileName, ProcessCommandLine, QueriedGroup
| sort by TimeGenerated desc | Correct | 3 | 226.46s | 0.0340 |