Question Analysis
A Windows system’s process logs show a PowerShell execution that altered firewall settings. Which cmdlet was used to add this new rule?
94be7646-25f6-467e-af23-585fb13000c8
Question ID
20/23
Models Correct
1
Valid Answer
87%
Success Rate
Reference Answer
Human-validated correct answer for this question
New-NetFirewallRule
Model Performance Comparison
See how different AI models performed on this question
Model | Answer | KQL Query | Status | Attempts | Time | Cost |
---|---|---|---|---|---|---|
gemini-2.5-flash-preview-04-17 | New-NetFirewallRule | DeviceProcessEvents
| where FileName =~ "powershell.exe"
| where ProcessCommandLine contains "NetFirewallRule"
| project Timestamp, DeviceName, ProcessCommandLine, AccountName | Correct | 1 | 7.82s | 0.0027 |
gpt-35-turbo | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0109 |
gpt-4-turbo-2024-04-09 | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.2293 |
gpt-4.1 | New-NetFirewallRule | DeviceProcessEvents
| where FileName =~ "powershell.exe" or FileName =~ "pwsh.exe"
| where ProcessCommandLine has_any ("New-NetFirewallRule", "Set-NetFirewallRule", "netsh advfirewall firewall add rule")
| extend CmdletUsed = case(ProcessCommandLine has "New-NetFirewallRule", "New-NetFirewallRule",
ProcessCommandLine has "Set-NetFirewallRule", "Set-NetFirewallRule",
ProcessCommandLine has "netsh advfirewall firewall add rule", "netsh advfirewall firewall add rule",
"Other")
| project TimeGenerated, DeviceName, AccountName, ProcessCommandLine, CmdletUsed | Correct | 1 | 5.24s | 0.0114 |
gpt-4.1-finetuned | New-NetFirewallRule | // Collect DeviceEvents that show PowerShell processes modifying firewall rules
DeviceProcessEvents
| where (InitiatingProcessFileName =~ "powershell.exe" or InitiatingProcessFileName =~ "pwsh.exe")
| where isnotempty(ProcessCommandLine)
| where ProcessCommandLine has "New-NetFirewallRule" or ProcessCommandLine has "netsh" or ProcessCommandLine has "Set-NetFirewallProfile"
| project TimeGenerated, DeviceName, FileName, ProcessCommandLine
| summarize count(), make_set(ProcessCommandLine, 1000) by TimeGenerated, DeviceName, FileName
| sort by TimeGenerated desc
| take 10 | Correct | 1 | 9.17s | 0.0123 |
gpt-4.1-mini | New-NetFirewallRule | DeviceProcessEvents
| where FileName =~ "powershell.exe" or FileName =~ "pwsh.exe"
| where ProcessCommandLine has_any ("New-NetFirewallRule", "Set-NetFirewallRule", "Add-NetFirewallRule")
| project TimeGenerated, DeviceName, InitiatingProcessAccountName, ProcessCommandLine
| order by TimeGenerated desc | Correct | 1 | 20.28s | 0.0022 |
gpt-4.1-nano | New-NetFirewallRule | DeviceProcessEvents
| where ProcessCommandLine contains 'New-NetFirewallRule'
| where InitiatingProcessFileName == 'powershell.exe' or InitiatingProcessFileName == 'pwsh.exe' | Correct | 1 | 2.41s | 0.0006 |
gpt-4o | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0679 |
gpt-5-high | New-NetFirewallRule | let psHosts = dynamic(["powershell.exe","pwsh.exe","powershell_ise.exe"]);
DeviceProcessEvents
| where FileName in~ (psHosts)
| where isnotempty(ProcessCommandLine)
// Look for PowerShell adding a firewall rule via cmdlet or via netsh
| extend FirewallCmdletRaw = extract(@"(?i)\b(new-netfirewallrule|netsh\s+advfirewall\s+firewall\s+add\s+rule)\b", 1, ProcessCommandLine)
| where isnotempty(FirewallCmdletRaw)
| extend FirewallCmdlet = iff(tolower(FirewallCmdletRaw) startswith "netsh", "netsh advfirewall firewall add rule", tostring(FirewallCmdletRaw))
// If you only want true PowerShell cmdlets that add a rule, keep New-NetFirewallRule
//| where tolower(FirewallCmdlet) == "new-netfirewallrule"
| summarize Executions=count(), SampleCommandLine=arg_max(TimeGenerated, ProcessCommandLine) by DeviceName, AccountName, FirewallCmdlet
| order by Executions desc | Correct | 1 | 83.38s | 0.0464 |
gpt-5-mini-high | New-NetFirewallRule | union DeviceProcessEvents, DeviceEvents, DeviceFileEvents
| extend CmdLine = coalesce(ProcessCommandLine, InitiatingProcessCommandLine)
| where isnotempty(CmdLine) and tolower(CmdLine) has "powershell"
| where CmdLine matches regex @"(?i)\b(New-NetFirewallRule|Set-NetFirewallRule|Add-NetFirewallRule|Remove-NetFirewallRule|Enable-NetFirewallRule|Disable-NetFirewallRule|netsh advfirewall|netsh)\b"
| extend Cmdlet = tostring(extract(@"(?i)\b(New-NetFirewallRule|Set-NetFirewallRule|Add-NetFirewallRule|Remove-NetFirewallRule|Enable-NetFirewallRule|Disable-NetFirewallRule|netsh advfirewall|netsh)\b", 1, CmdLine))
| project TimeGenerated, DeviceName, Account = coalesce(InitiatingProcessAccountName, AccountName, RequestAccountName), Cmdlet, CmdLine, FileName, ProcessId = coalesce(ProcessId, InitiatingProcessId)
| order by TimeGenerated desc
| take 100 | Correct | 1 | 24.73s | 0.0056 |
gpt-5-mini-low | New-NetFirewallRule | // Define firewall-related patterns
let firewallPatterns = dynamic(["New-NetFirewallRule","Set-NetFirewallRule","Add-NetFirewallRule","New-FirewallRule","Add-FirewallRule","netsh advfirewall","netsh.exe advfirewall","advfirewall"]);
// Function to extract matched cmdlet/command
let extractFirewallCmd = (cmdline:string){
extract(@"(?i)(New-NetFirewallRule|Set-NetFirewallRule|Add-NetFirewallRule|New-FirewallRule|Add-FirewallRule|netsh(?:\.exe)?\s+advfirewall|advfirewall)", 1, cmdline)
};
// Search DeviceProcessEvents for PowerShell processes with firewall patterns
let dpe = DeviceProcessEvents
| where ProcessCommandLine has_any (firewallPatterns)
| where FileName has_any ("powershell","pwsh","powershell.exe","pwsh.exe") or ProcessCommandLine has_any ("powershell","pwsh")
| extend MatchedCmd = extractFirewallCmd(ProcessCommandLine)
| project TimeGenerated=Timestamp, SourceTable="DeviceProcessEvents", DeviceName, AccountName, ProcessId, FileName, ProcessCommandLine, MatchedCmd;
// Search DeviceEvents for PowerShell-related entries (additional telemetry)
let dev = DeviceEvents
| where (ProcessCommandLine has_any (firewallPatterns) or AdditionalFields has_any (firewallPatterns) or tostring(AdditionalFields) has_any (firewallPatterns))
| where FileName has_any ("powershell","pwsh","powershell.exe","pwsh.exe") or ProcessCommandLine has_any ("powershell","pwsh")
| extend MatchedCmd = extractFirewallCmd(coalesce(ProcessCommandLine, tostring(AdditionalFields)))
| project TimeGenerated=Timestamp, SourceTable="DeviceEvents", DeviceName, AccountName=InitiatingProcessAccountName, ProcessId=InitiatingProcessId, FileName=InitiatingProcessFileName, ProcessCommandLine=InitiatingProcessCommandLine, MatchedCmd;
// Combine and present results
union isfuzzy=true dpe, dev
| where isnotempty(MatchedCmd)
| sort by TimeGenerated desc
| take 100
| Correct | 2 | 20.08s | 0.0048 |
gpt-5-mini-medium | New-NetFirewallRule | // Find PowerShell processes that executed firewall-related cmdlets and identify which cmdlet added the rule
union isfuzzy=true
(
DeviceEvents
| where FileName has_cs "powershell" or InitiatingProcessFileName has_cs "powershell"
| extend cmdline = coalesce(ProcessCommandLine, InitiatingProcessCommandLine)
| where cmdline has_cs "New-NetFirewallRule" or cmdline has_cs "Set-NetFirewallRule" or cmdline has_cs "Add-NetFirewallRule" or cmdline has_cs "Enable-NetFirewallRule" or cmdline has_cs "netsh advfirewall" or cmdline has_cs "New-NetFirewallPortFilter" or cmdline has_cs "New-NetFirewallApplicationFilter"
| extend detectedCmdlet = case(
cmdline has_cs "New-NetFirewallRule", "New-NetFirewallRule",
cmdline has_cs "Add-NetFirewallRule", "Add-NetFirewallRule",
cmdline has_cs "Set-NetFirewallRule", "Set-NetFirewallRule",
cmdline has_cs "Enable-NetFirewallRule", "Enable-NetFirewallRule",
cmdline has_cs "netsh advfirewall", "netsh advfirewall",
cmdline has_cs "New-NetFirewallPortFilter", "New-NetFirewallPortFilter",
cmdline has_cs "New-NetFirewallApplicationFilter", "New-NetFirewallApplicationFilter",
"unknown"
)
| project TimeGenerated, DeviceName, FileName, ProcessId, InitiatingProcessFileName, InitiatingProcessId, AccountName, InitiatingProcessAccountName, cmdline, detectedCmdlet, ReportId, TenantId
),
(
DeviceProcessEvents
| where FileName has_cs "powershell" or InitiatingProcessFileName has_cs "powershell"
| extend cmdline = coalesce(ProcessCommandLine, InitiatingProcessCommandLine)
| where cmdline has_cs "New-NetFirewallRule" or cmdline has_cs "Set-NetFirewallRule" or cmdline has_cs "Add-NetFirewallRule" or cmdline has_cs "Enable-NetFirewallRule" or cmdline has_cs "netsh advfirewall" or cmdline has_cs "New-NetFirewallPortFilter" or cmdline has_cs "New-NetFirewallApplicationFilter"
| extend detectedCmdlet = case(
cmdline has_cs "New-NetFirewallRule", "New-NetFirewallRule",
cmdline has_cs "Add-NetFirewallRule", "Add-NetFirewallRule",
cmdline has_cs "Set-NetFirewallRule", "Set-NetFirewallRule",
cmdline has_cs "Enable-NetFirewallRule", "Enable-NetFirewallRule",
cmdline has_cs "netsh advfirewall", "netsh advfirewall",
cmdline has_cs "New-NetFirewallPortFilter", "New-NetFirewallPortFilter",
cmdline has_cs "New-NetFirewallApplicationFilter", "New-NetFirewallApplicationFilter",
"unknown"
)
| project TimeGenerated, DeviceName, FileName, ProcessId, InitiatingProcessFileName, InitiatingProcessId, AccountName, InitiatingProcessAccountName, cmdline, detectedCmdlet, ReportId, TenantId
)
| order by TimeGenerated desc
| summarize count() by detectedCmdlet
| join kind=inner (
union isfuzzy=true (
DeviceEvents
| where FileName has_cs "powershell" or InitiatingProcessFileName has_cs "powershell"
| extend cmdline = coalesce(ProcessCommandLine, InitiatingProcessCommandLine)
| where cmdline has_cs "New-NetFirewallRule" or cmdline has_cs "Set-NetFirewallRule" or cmdline has_cs "Add-NetFirewallRule" or cmdline has_cs "Enable-NetFirewallRule" or cmdline has_cs "netsh advfirewall" or cmdline has_cs "New-NetFirewallPortFilter" or cmdline has_cs "New-NetFirewallApplicationFilter"
| extend detectedCmdlet = case(
cmdline has_cs "New-NetFirewallRule", "New-NetFirewallRule",
cmdline has_cs "Add-NetFirewallRule", "Add-NetFirewallRule",
cmdline has_cs "Set-NetFirewallRule", "Set-NetFirewallRule",
cmdline has_cs "Enable-NetFirewallRule", "Enable-NetFirewallRule",
cmdline has_cs "netsh advfirewall", "netsh advfirewall",
cmdline has_cs "New-NetFirewallPortFilter", "New-NetFirewallPortFilter",
cmdline has_cs "New-NetFirewallApplicationFilter", "New-NetFirewallApplicationFilter",
"unknown"
)
| project TimeGenerated, DeviceName, AccountName, InitiatingProcessAccountName, cmdline, detectedCmdlet
),
(
DeviceProcessEvents
| where FileName has_cs "powershell" or InitiatingProcessFileName has_cs "powershell"
| extend cmdline = coalesce(ProcessCommandLine, InitiatingProcessCommandLine)
| where cmdline has_cs "New-NetFirewallRule" or cmdline has_cs "Set-NetFirewallRule" or cmdline has_cs "Add-NetFirewallRule" or cmdline has_cs "Enable-NetFirewallRule" or cmdline has_cs "netsh advfirewall" or cmdline has_cs "New-NetFirewallPortFilter" or cmdline has_cs "New-NetFirewallApplicationFilter"
| extend detectedCmdlet = case(
cmdline has_cs "New-NetFirewallRule", "New-NetFirewallRule",
cmdline has_cs "Add-NetFirewallRule", "Add-NetFirewallRule",
cmdline has_cs "Set-NetFirewallRule", "Set-NetFirewallRule",
cmdline has_cs "Enable-NetFirewallRule", "Enable-NetFirewallRule",
cmdline has_cs "netsh advfirewall", "netsh advfirewall",
cmdline has_cs "New-NetFirewallPortFilter", "New-NetFirewallPortFilter",
cmdline has_cs "New-NetFirewallApplicationFilter", "New-NetFirewallApplicationFilter",
"unknown"
)
| project TimeGenerated, DeviceName, AccountName, InitiatingProcessAccountName, cmdline, detectedCmdlet
)
) on detectedCmdlet
| project detectedCmdlet, count_, TimeGenerated, DeviceName, AccountName, InitiatingProcessAccountName, cmdline
| order by count_ desc, TimeGenerated desc
| Correct | 1 | 19.00s | 0.0052 |
gpt-5-nano-high | New-NetFirewallRule | DeviceProcessEvents
| where InitiatingProcessFileName in ("powershell.exe","pwsh.exe")
| where InitiatingProcessCommandLine has_any ("New-NetFirewallRule","Add-NetFirewallRule","Set-NetFirewallRule","netsh firewall add rule","netsh advfirewall firewall add rule")
| extend CmdletUsed = case(
InitiatingProcessCommandLine has "New-NetFirewallRule", "New-NetFirewallRule",
InitiatingProcessCommandLine has "Add-NetFirewallRule", "Add-NetFirewallRule",
InitiatingProcessCommandLine has "Set-NetFirewallRule", "Set-NetFirewallRule",
InitiatingProcessCommandLine has_any ("netsh firewall add rule","netsh advfirewall firewall add rule"), "netsh firewall add rule",
"Unknown")
| project TimeGenerated, DeviceName, InitiatingProcessFileName, InitiatingProcessCommandLine, CmdletUsed
| sort by TimeGenerated desc | Correct | 3 | 71.20s | 0.0046 |
gpt-5-nano-medium | New-NetFirewallRule | DeviceProcessEvents
| where InitiatingProcessFileName in ('powershell.exe','pwsh.exe')
| where ProcessCommandLine has_any ('New-NetFirewallRule','Add-NetFirewallRule','Set-NetFirewallRule')
| extend CmdletUsed = extract('(New-NetFirewallRule|Add-NetFirewallRule|Set-NetFirewallRule)', 1, ProcessCommandLine)
| project TimeGenerated, DeviceName, InitiatingProcessFileName, InitiatingProcessCommandLine, ProcessCommandLine, CmdletUsed, InitiatingProcessId, ProcessId
| order by TimeGenerated desc | Correct | 1 | 41.20s | 0.0017 |
grok-3-beta | New-NetFirewallRule | DeviceProcessEvents
| where (InitiatingProcessFileName contains "powershell.exe" or FileName contains "powershell.exe")
| where ProcessCommandLine contains "firewall" or ProcessCommandLine contains "New-NetFirewallRule" or ProcessCommandLine contains "Set-NetFirewallRule" or ProcessCommandLine contains "Enable-NetFirewallRule"
| project DeviceName, AccountName, ProcessCommandLine, TimeGenerated
| order by TimeGenerated desc | Correct | 1 | 12.13s | 0.0236 |
grok-3-mini-beta | New-NetFirewallRule | DeviceProcessEvents
| where FileName =~ 'powershell.exe'
| where ProcessCommandLine contains 'NetFirewallRule'
| project Cmdlet = extract(@'New-|Set-|Remove-.*FirewallRule', 0, ProcessCommandLine), ProcessCommandLine, TimeGenerated
| summarize arg_max(TimeGenerated, *) by Cmdlet
| project Cmdlet, ProcessCommandLine, TimeGenerated
| order by TimeGenerated desc
| take 10 | Correct | 2 | 26.19s | 0.0030 |
o1-high | New-NetFirewallRule | DeviceProcessEvents
| where FileName =~ "powershell.exe" or InitiatingProcessFileName =~ "powershell.exe"
| where ProcessCommandLine has "New-NetFirewallRule"
| project TimeGenerated, DeviceName, AccountName, ProcessCommandLine, InitiatingProcessCommandLine
| order by TimeGenerated desc | Correct | 1 | 25.31s | 0.1915 |
o1-low | New-NetFirewallRule | DeviceProcessEvents
| where FileName =~ "powershell.exe"
or InitiatingProcessFileName =~ "powershell.exe"
| where ProcessCommandLine has_any ("New-NetFirewallRule", "Add-NetFirewallRule", "Set-NetFirewallRule")
| project TimeGenerated, DeviceName, AccountName, ProcessCommandLine
| order by TimeGenerated desc | Correct | 1 | 19.03s | 0.1603 |
o3-high | New-NetFirewallRule | DeviceProcessEvents
| where ActionType == "ProcessCreated" // only new processes
| where FileName in~ ("powershell.exe","pwsh.exe","powershell_ise.exe")
| where ProcessCommandLine has_any ("New-NetFirewallRule","Add-NetFirewallRule","Set-NetFirewallRule")
| extend CmdletUsed = tostring(extract(@"(?i)(New-NetFirewallRule|Add-NetFirewallRule|Set-NetFirewallRule)", 1, ProcessCommandLine))
| project TimeGenerated, DeviceName, AccountName, InitiatingProcessAccountName, ProcessCommandLine, CmdletUsed
| summarize FirstSeen=min(TimeGenerated), LastSeen=max(TimeGenerated), Executions=count() by DeviceName, AccountName, CmdletUsed
| order by LastSeen desc | Correct | 2 | 54.28s | 0.0384 |
o3-mini-high | New-NetFirewallRule | DeviceProcessEvents
| where FileName has "powershell.exe"
| where ProcessCommandLine contains "NetFirewallRule"
| extend Cmdlet = extract("(?i)(New-NetFirewallRule|Add-NetFirewallRule|Set-NetFirewallRule)", 1, ProcessCommandLine)
| where isnotempty(Cmdlet)
| project TimeGenerated, DeviceName, ProcessCommandLine, Cmdlet
| summarize Count = count() by Cmdlet, DeviceName, bin(TimeGenerated, 1h)
| order by TimeGenerated desc | Correct | 1 | 10.78s | 0.0076 |
o3-mini-low | New-NetFirewallRule | DeviceProcessEvents
| where FileName has_cs "powershell.exe" or InitiatingProcessFileName has_cs "powershell.exe"
| extend FullCommandLine = coalesce(ProcessCommandLine, InitiatingProcessCommandLine)
| where FullCommandLine has_cs "NetFirewallRule"
| extend CmdletUsed = extract("(?i)(New-NetFirewallRule|Add-NetFirewallRule)", 1, FullCommandLine)
| where isnotempty(CmdletUsed)
| project TimeGenerated, DeviceName, FullCommandLine, CmdletUsed
| order by TimeGenerated desc | Correct | 2 | 24.49s | 0.0170 |
o4-mini-high | New-NetFirewallRule | DeviceProcessEvents
| where FileName in~ ("powershell.exe", "pwsh.exe")
| where ProcessCommandLine matches regex @"(?i)(New|Add)-NetFirewallRule"
| extend UsedCmdlet = extract(@"(?i)(New|Add)-NetFirewallRule", 0, ProcessCommandLine)
| summarize Count = count(), Examples = make_set(ProcessCommandLine, 5) by UsedCmdlet
| project UsedCmdlet, Count, Examples | Correct | 1 | 27.71s | 0.0109 |
o4-mini-low | New-NetFirewallRule | DeviceProcessEvents
| where FileName in ("powershell.exe", "pwsh.exe")
| where ProcessCommandLine has "NetFirewallRule"
| extend Cmdlet = extract(@"\b((New|Set|Add)-NetFirewallRule)\b", 1, ProcessCommandLine)
| project TimeGenerated, DeviceName, InitiatingProcessAccountName, Cmdlet, ProcessCommandLine
| sort by TimeGenerated desc | Correct | 1 | 25.21s | 0.0071 |