Question Analysis
A Windows system’s process logs show a PowerShell execution that altered firewall settings. Which cmdlet was used to add this new rule?
94be7646-25f6-467e-af23-585fb13000c8
Question ID
13/16
Models Correct
1
Valid Answer
81%
Success Rate
Reference Answer
Human-validated correct answer for this question
New-NetFirewallRule
Model Performance Comparison
See how different AI models performed on this question
| Model | Answer | KQL Query | Status | Attempts | Time | Cost |
|---|---|---|---|---|---|---|
| gemini-2.5-flash-preview-04-17 | New-NetFirewallRule | DeviceProcessEvents
| where FileName =~ "powershell.exe"
| where ProcessCommandLine contains "NetFirewallRule"
| project Timestamp, DeviceName, ProcessCommandLine, AccountName | Correct | 1 | 7.82s | 0.0027 |
| gpt-35-turbo | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0109 |
| gpt-4-turbo-2024-04-09 | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.2293 |
| gpt-4.1 | New-NetFirewallRule | DeviceProcessEvents
| where FileName =~ "powershell.exe" or FileName =~ "pwsh.exe"
| where ProcessCommandLine has_any ("New-NetFirewallRule", "Set-NetFirewallRule", "netsh advfirewall firewall add rule")
| extend CmdletUsed = case(ProcessCommandLine has "New-NetFirewallRule", "New-NetFirewallRule",
ProcessCommandLine has "Set-NetFirewallRule", "Set-NetFirewallRule",
ProcessCommandLine has "netsh advfirewall firewall add rule", "netsh advfirewall firewall add rule",
"Other")
| project TimeGenerated, DeviceName, AccountName, ProcessCommandLine, CmdletUsed | Correct | 1 | 5.24s | 0.0114 |
| gpt-4.1-finetuned | New-NetFirewallRule | // Collect DeviceEvents that show PowerShell processes modifying firewall rules
DeviceProcessEvents
| where (InitiatingProcessFileName =~ "powershell.exe" or InitiatingProcessFileName =~ "pwsh.exe")
| where isnotempty(ProcessCommandLine)
| where ProcessCommandLine has "New-NetFirewallRule" or ProcessCommandLine has "netsh" or ProcessCommandLine has "Set-NetFirewallProfile"
| project TimeGenerated, DeviceName, FileName, ProcessCommandLine
| summarize count(), make_set(ProcessCommandLine, 1000) by TimeGenerated, DeviceName, FileName
| sort by TimeGenerated desc
| take 10 | Correct | 1 | 9.17s | 0.0123 |
| gpt-4.1-mini | New-NetFirewallRule | DeviceProcessEvents
| where FileName =~ "powershell.exe" or FileName =~ "pwsh.exe"
| where ProcessCommandLine has_any ("New-NetFirewallRule", "Set-NetFirewallRule", "Add-NetFirewallRule")
| project TimeGenerated, DeviceName, InitiatingProcessAccountName, ProcessCommandLine
| order by TimeGenerated desc | Correct | 1 | 20.28s | 0.0022 |
| gpt-4.1-nano | New-NetFirewallRule | DeviceProcessEvents
| where ProcessCommandLine contains 'New-NetFirewallRule'
| where InitiatingProcessFileName == 'powershell.exe' or InitiatingProcessFileName == 'pwsh.exe' | Correct | 1 | 2.41s | 0.0006 |
| gpt-4o | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0679 |
| grok-3-beta | New-NetFirewallRule | DeviceProcessEvents
| where (InitiatingProcessFileName contains "powershell.exe" or FileName contains "powershell.exe")
| where ProcessCommandLine contains "firewall" or ProcessCommandLine contains "New-NetFirewallRule" or ProcessCommandLine contains "Set-NetFirewallRule" or ProcessCommandLine contains "Enable-NetFirewallRule"
| project DeviceName, AccountName, ProcessCommandLine, TimeGenerated
| order by TimeGenerated desc | Correct | 1 | 12.13s | 0.0236 |
| grok-3-mini-beta | New-NetFirewallRule | DeviceProcessEvents
| where FileName =~ 'powershell.exe'
| where ProcessCommandLine contains 'NetFirewallRule'
| project Cmdlet = extract(@'New-|Set-|Remove-.*FirewallRule', 0, ProcessCommandLine), ProcessCommandLine, TimeGenerated
| summarize arg_max(TimeGenerated, *) by Cmdlet
| project Cmdlet, ProcessCommandLine, TimeGenerated
| order by TimeGenerated desc
| take 10 | Correct | 2 | 26.19s | 0.0030 |
| o1-high | New-NetFirewallRule | DeviceProcessEvents
| where FileName =~ "powershell.exe" or InitiatingProcessFileName =~ "powershell.exe"
| where ProcessCommandLine has "New-NetFirewallRule"
| project TimeGenerated, DeviceName, AccountName, ProcessCommandLine, InitiatingProcessCommandLine
| order by TimeGenerated desc | Correct | 1 | 25.31s | 0.1915 |
| o1-low | New-NetFirewallRule | DeviceProcessEvents
| where FileName =~ "powershell.exe"
or InitiatingProcessFileName =~ "powershell.exe"
| where ProcessCommandLine has_any ("New-NetFirewallRule", "Add-NetFirewallRule", "Set-NetFirewallRule")
| project TimeGenerated, DeviceName, AccountName, ProcessCommandLine
| order by TimeGenerated desc | Correct | 1 | 19.03s | 0.1603 |
| o3-mini-high | New-NetFirewallRule | DeviceProcessEvents
| where FileName has "powershell.exe"
| where ProcessCommandLine contains "NetFirewallRule"
| extend Cmdlet = extract("(?i)(New-NetFirewallRule|Add-NetFirewallRule|Set-NetFirewallRule)", 1, ProcessCommandLine)
| where isnotempty(Cmdlet)
| project TimeGenerated, DeviceName, ProcessCommandLine, Cmdlet
| summarize Count = count() by Cmdlet, DeviceName, bin(TimeGenerated, 1h)
| order by TimeGenerated desc | Correct | 1 | 10.78s | 0.0076 |
| o3-mini-low | New-NetFirewallRule | DeviceProcessEvents
| where FileName has_cs "powershell.exe" or InitiatingProcessFileName has_cs "powershell.exe"
| extend FullCommandLine = coalesce(ProcessCommandLine, InitiatingProcessCommandLine)
| where FullCommandLine has_cs "NetFirewallRule"
| extend CmdletUsed = extract("(?i)(New-NetFirewallRule|Add-NetFirewallRule)", 1, FullCommandLine)
| where isnotempty(CmdletUsed)
| project TimeGenerated, DeviceName, FullCommandLine, CmdletUsed
| order by TimeGenerated desc | Correct | 2 | 24.49s | 0.0170 |
| o4-mini-high | New-NetFirewallRule | DeviceProcessEvents
| where FileName in~ ("powershell.exe", "pwsh.exe")
| where ProcessCommandLine matches regex @"(?i)(New|Add)-NetFirewallRule"
| extend UsedCmdlet = extract(@"(?i)(New|Add)-NetFirewallRule", 0, ProcessCommandLine)
| summarize Count = count(), Examples = make_set(ProcessCommandLine, 5) by UsedCmdlet
| project UsedCmdlet, Count, Examples | Correct | 1 | 27.71s | 0.0109 |
| o4-mini-low | New-NetFirewallRule | DeviceProcessEvents
| where FileName in ("powershell.exe", "pwsh.exe")
| where ProcessCommandLine has "NetFirewallRule"
| extend Cmdlet = extract(@"\b((New|Set|Add)-NetFirewallRule)\b", 1, ProcessCommandLine)
| project TimeGenerated, DeviceName, InitiatingProcessAccountName, Cmdlet, ProcessCommandLine
| sort by TimeGenerated desc | Correct | 1 | 25.21s | 0.0071 |