Question Analysis
On a Windows device, a new inbound firewall rule was created unexpectedly. Review process execution records to identify the command-line utility responsible for adding the rule.
9636dd6e-7599-40d2-8eee-ac16434f35ed
Question ID
11/23
Models Correct
1
Valid Answer
48%
Success Rate
Reference Answer
Human-validated correct answer for this question
netsh
Model Performance Comparison
See how different AI models performed on this question
Model | Answer | KQL Query | Status | Attempts | Time | Cost |
---|---|---|---|---|---|---|
gemini-2.5-flash-preview-04-17 | netsh.exe | DeviceProcessEvents
| where ActionType == "ProcessCreated"
| where FolderPath endswith @"\System32\netsh.exe" or FileName == "powershell.exe"
| where ProcessCommandLine contains "advfirewall firewall add rule" or ProcessCommandLine contains "New-NetFirewallRule"
| where ProcessCommandLine contains "dir=in" or ProcessCommandLine contains "-Direction Inbound"
| project Timestamp, DeviceName, ProcessCommandLine, InitiatingProcessCommandLine, FileName, InitiatingProcessFileName, AccountName, InitiatingProcessAccountName | Correct | 1 | 9.25s | 0.0041 |
gpt-35-turbo | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0110 |
gpt-4-turbo-2024-04-09 | netsh | DeviceProcessEvents
| where ActionType == 'ProcessCreated'
| where ProcessCommandLine has 'netsh' and ProcessCommandLine has 'advfirewall' and ProcessCommandLine has 'add'
| project TimeGenerated, DeviceName, InitiatingProcessCommandLine, ProcessCommandLine | Correct | 1 | 8.91s | 0.0541 |
gpt-4.1 | netsh.exe | DeviceProcessEvents
| where FileName in~ ("netsh.exe", "powershell.exe", "cmd.exe", "wf.msc")
| where ProcessCommandLine has_any ("firewall", "add rule", "New-NetFirewallRule")
| project TimeGenerated, DeviceName, AccountName, FileName, ProcessCommandLine, InitiatingProcessAccountName, InitiatingProcessFileName
| order by TimeGenerated desc | Correct | 1 | 6.42s | 0.0128 |
gpt-4.1-finetuned | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0422 |
gpt-4.1-mini | netsh | DeviceProcessEvents
| where ProcessCommandLine has_cs "netsh" or ProcessCommandLine has_cs "New-NetFirewallRule" or ProcessCommandLine has_cs "advfirewall" or ProcessCommandLine has_cs "firewall"
| where ActionType == "ProcessCreated"
| project TimeGenerated, DeviceName, InitiatingProcessAccountName, ProcessCommandLine, ProcessCreationTime, FileName
| order by TimeGenerated desc | Correct | 1 | 16.31s | 0.0033 |
gpt-4.1-nano | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0014 |
gpt-4o | cmd.exe | DeviceProcessEvents
| where InitiatingProcessCommandLine has_any ("netsh advfirewall firewall add", "powershell New-NetFirewallRule")
| project TimeGenerated, DeviceName, DeviceId, InitiatingProcessFileName, InitiatingProcessCommandLine, InitiatingProcessId, AccountName, AccountDomain
| order by TimeGenerated desc | Incorrect | 4 | 28.09s | 0.0472 |
gpt-5-high | PowerShell | // Find new/changed inbound firewall rules and correlate to the process that likely added them
let firewall_rule_events =
union
(
DeviceEvents
| where ActionType has_any ("FirewallRule", "FirewallPolicy")
| extend RuleName = coalesce(tostring(AdditionalFields.RuleName), tostring(AdditionalFields.RuleDisplayName), tostring(AdditionalFields.Name)),
DirectionRaw = tostring(AdditionalFields.Direction),
RuleDetails = tostring(AdditionalFields)
| where isempty(DirectionRaw) or DirectionRaw =~ "In" or DirectionRaw =~ "Inbound"
| project DeviceId, DeviceName, EventTime=Timestamp, Source="DeviceEvents", ActionType, RuleName, Direction=iff(isempty(DirectionRaw), "In", DirectionRaw), RuleDetails
),
(
DeviceRegistryEvents
| where ActionType in~ ("RegistryValueSet", "RegistryValueCreate", "RegistryValueCreated", "RegistryKeyCreated")
| where RegistryKey has "\\SYSTEM\\CurrentControlSet\\Services\\SharedAccess\\Parameters\\FirewallPolicy\\FirewallRules"
| where tolower(RegistryValueData) has "dir=in"
| project DeviceId, DeviceName, EventTime=Timestamp, Source="Registry", ActionType,
RuleName=RegistryValueName, Direction="In", RuleDetails=tostring(RegistryValueData)
);
let proc_candidates =
DeviceProcessEvents
| extend ProcTime=Timestamp, lowerFile=tolower(FileName), lowerCmd=tolower(ProcessCommandLine)
| where lowerFile in ("netsh.exe", "powershell.exe", "pwsh.exe", "cmd.exe", "mmc.exe")
or lowerCmd has_any ("netsh", "new-netfirewallrule", "add-netfirewallrule", "set-netfirewallrule", "enable-netfirewallrule", "advfirewall", "firewall add rule", "wf.msc")
| project DeviceId, ProcTime, FileName, FolderPath, ProcessCommandLine,
InitiatingProcessAccountDomain, InitiatingProcessAccountName,
InitiatingProcessParentFileName, InitiatingProcessCommandLine,
InitiatingProcessId, InitiatingProcessParentId, ProcessId, ProcessTokenElevation;
// Correlate rule events with nearby process executions on the same device
let correlated =
firewall_rule_events
| join kind=inner (proc_candidates) on DeviceId
| where ProcTime between (EventTime - 15m .. EventTime + 10m)
| extend proximity = abs(ProcTime - EventTime)
| summarize arg_min(proximity, *) by DeviceId, DeviceName, EventTime, Source, ActionType, RuleName, Direction, RuleDetails
| extend SuspectedUtility = case(
tolower(FileName)=="netsh.exe", "netsh advfirewall",
tolower(FileName) in ("powershell.exe", "pwsh.exe"), "PowerShell (NetSecurity cmdlet)",
tolower(FileName)=="mmc.exe", "Windows Firewall MMC (wf.msc)",
tolower(FileName)=="cmd.exe" and tolower(ProcessCommandLine) has "netsh", "cmd -> netsh",
FileName)
| extend User = iff(isempty(InitiatingProcessAccountName), "", strcat(InitiatingProcessAccountDomain, "\\", InitiatingProcessAccountName))
| project MatchType="CorrelatedToRule", DeviceName, EventTime, RuleName, Direction, ActionType,
SuspectedUtility, CommandLine=ProcessCommandLine, ProcTime,
User, ParentProcess=InitiatingProcessParentFileName, ParentCmd=InitiatingProcessCommandLine,
ProcessId, InitiatingProcessId, ProcessTokenElevation, Source, RuleDetails;
// Fallback: list likely firewall rule addition commands (focus on inbound) within the time range
let process_only =
DeviceProcessEvents
| extend ProcTime=Timestamp, lowerFile=tolower(FileName), lowerCmd=tolower(ProcessCommandLine)
| where (lowerFile=="netsh.exe" and lowerCmd has "advfirewall" and lowerCmd has "firewall" and lowerCmd has "add" and lowerCmd has "dir=in")
or (lowerFile in ("powershell.exe", "pwsh.exe") and lowerCmd has_any ("new-netfirewallrule", "add-netfirewallrule", "set-netfirewallrule") and lowerCmd has "-direction" and lowerCmd has "inbound")
or (lowerFile=="cmd.exe" and lowerCmd has "netsh" and lowerCmd has "advfirewall" and lowerCmd has "add" and lowerCmd has "dir=in")
or (lowerFile=="mmc.exe" and lowerCmd has "wf.msc")
| extend SuspectedUtility = case(
lowerFile=="netsh.exe", "netsh advfirewall",
lowerFile in ("powershell.exe", "pwsh.exe"), "PowerShell (NetSecurity cmdlet)",
lowerFile=="mmc.exe", "Windows Firewall MMC (wf.msc)",
lowerFile=="cmd.exe" and lowerCmd has "netsh", "cmd -> netsh",
FileName)
| extend User = iff(isempty(InitiatingProcessAccountName), "", strcat(InitiatingProcessAccountDomain, "\\", InitiatingProcessAccountName))
| project MatchType="ProcessOnly", DeviceName, EventTime=ProcTime, RuleName=strcat("(unknown - process heuristic)"), Direction="In",
ActionType="", SuspectedUtility, CommandLine=ProcessCommandLine, ProcTime,
User, ParentProcess=InitiatingProcessParentFileName, ParentCmd=InitiatingProcessCommandLine,
ProcessId, InitiatingProcessId, ProcessTokenElevation, Source="Process", RuleDetails="";
union correlated, process_only
| order by EventTime desc, ProcTime asc | Incorrect | 4 | 485.52s | 0.2495 |
gpt-5-mini-high | PowerShell | // Correlate firewall-related registry changes with processes that executed firewall command-line utilities
let proc_cmds = dynamic(['netsh','advfirewall','firewall add rule','New-NetFirewallRule','New-NetFirewallPortRule','Set-NetFirewallRule','netsh firewall','netsh advfirewall','Add-NetFirewallRule']);
// Registry events targeting firewall keys
let regEvents = DeviceRegistryEvents
| where TimeGenerated between (startofday(ago(7d)) .. now())
| where RegistryKey has_any ('FirewallPolicy','WindowsFirewall','FirewallRules','SharedAccess','HKEY_LOCAL_MACHINE\\SOFTWARE\\Policies\\Microsoft\\WindowsFirewall')
| where ActionType in ('SetValue','Create','Modify','Delete')
| project DeviceId, DeviceName, FirewallEventTime = TimeGenerated, RegistryKey, RegistryValueName, RegistryValueData, InitiatingProcessId, InitiatingProcessFileName, InitiatingProcessCommandLine, InitiatingProcessAccountName, InitiatingProcessAccountDomain;
// Process events that may indicate firewall CLI usage
let procEvents = DeviceProcessEvents
| where TimeGenerated between (startofday(ago(7d)) .. now())
| where (ProcessCommandLine has_any (proc_cmds)) or (FileName in ('netsh.exe','powershell.exe','pwsh.exe','cmd.exe','cscript.exe') and ProcessCommandLine contains 'firewall')
| project DeviceId, ProcTime = TimeGenerated, ProcessId, ProcessFileName = FileName, ProcessCommandLine, AccountName = InitiatingProcessAccountName, AccountDomain = InitiatingProcessAccountDomain;
// Correlate within +/- 5 minutes
regEvents
| join kind=leftouter (procEvents) on DeviceId
| where isnotempty(ProcTime) and abs(datetime_diff('second', FirewallEventTime, ProcTime)) <= 300
| project DeviceName, DeviceId, FirewallEventTime, RegistryKey, RegistryValueName, RegistryValueData, InitiatingProcessFileName, InitiatingProcessCommandLine, ProcTime, ProcessId, ProcessFileName, ProcessCommandLine, AccountName, AccountDomain
| summarize EarliestFirewallChange = min(FirewallEventTime), RegistryKeys = make_set(RegistryKey,50), RegistryValues = make_set(RegistryValueName,200), RelatedProcessCommands = make_set(ProcessCommandLine,200), RelatedProcessFiles = make_set(ProcessFileName,50), InitiatingProcessFiles = make_set(InitiatingProcessFileName,20) by DeviceName, DeviceId
| order by EarliestFirewallChange desc
// Fallback: If no registry events found, return process-only matches (firewall-related commands)
| union (
DeviceProcessEvents
| where TimeGenerated between (startofday(ago(7d)) .. now())
| where (ProcessCommandLine has_any (proc_cmds)) or (FileName in ('netsh.exe','powershell.exe','pwsh.exe','cmd.exe','cscript.exe') and ProcessCommandLine contains 'firewall')
| project DeviceName, DeviceId, EarliestFirewallChange = TimeGenerated, RegistryKeys = dynamic([]), RegistryValues = dynamic([]), RelatedProcessCommands = pack_array(ProcessCommandLine), RelatedProcessFiles = pack_array(FileName), InitiatingProcessFiles = pack_array(tostring(''))
)
| order by EarliestFirewallChange desc
| Incorrect | 3 | 57.49s | 0.0148 |
gpt-5-mini-low | powershell.exe | union DeviceProcessEvents, DeviceEvents, DeviceNetworkEvents, DeviceRegistryEvents
| extend cmd = coalesce(ProcessCommandLine, InitiatingProcessCommandLine)
| where isnotempty(cmd)
// Look for common firewall-management utilities and PowerShell cmdlets (case-insensitive)
| where tolower(cmd) has_any ("netsh", "advfirewall", "new-netfirewallrule", "add-netfirewallrule", "set-netfirewallrule", "add rule")
| project TimeGenerated, DeviceName, ReportId, FileName, ProcessId, ProcessCreationTime, cmd, InitiatingProcessFileName, InitiatingProcessId, InitiatingProcessCommandLine, InitiatingProcessAccountName = InitiatingProcessAccountName, AccountName, AccountDomain, ActionType, RegistryKey, RegistryValueName
| summarize FirstSeen = min(TimeGenerated), LastSeen = max(TimeGenerated), Count = count() by DeviceName, FileName, ProcessId, ProcessCreationTime, InitiatingProcessFileName, InitiatingProcessId, cmd, InitiatingProcessCommandLine, InitiatingProcessAccountName, AccountName, AccountDomain
| sort by LastSeen desc
| take 200 | Incorrect | 1 | 31.31s | 0.0066 |
gpt-5-mini-medium | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0237 |
gpt-5-nano-high | netsh | let FirewallRuleTools = dynamic(["netsh", "New-NetFirewallRule", "Add-NetFirewallRule", "Set-NetFirewallRule", "netsh advfirewall firewall add rule", "New-NetFirewallRule -Inbound", "Add-NetFirewallRule -Inbound"]);
DeviceProcessEvents
| where InitiatingProcessCommandLine has_any (FirewallRuleTools)
| project TimeGenerated, DeviceName, DeviceId, InitiatingProcessId, InitiatingProcessSessionId, InitiatingProcessFileName, InitiatingProcessCommandLine, InitiatingProcessCreationTime, InitiatingProcessAccountDomain, InitiatingProcessAccountName, InitiatingProcessVersionInfoProductName
| sort by TimeGenerated desc | Correct | 1 | 43.29s | 0.0027 |
gpt-5-nano-medium | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0067 |
grok-3-beta | netsh.exe | DeviceProcessEvents
| where ProcessCommandLine has_cs 'netsh' or ProcessCommandLine has_cs 'firewall' or ProcessCommandLine has_cs 'advfirewall' or ProcessCommandLine has_cs 'rule'
| project Timestamp, DeviceName, ProcessName = FileName, CommandLine = ProcessCommandLine, AccountName, InitiatingProcessFileName, InitiatingProcessCommandLine
| order by Timestamp desc
| limit 50 | Correct | 4 | 32.70s | 0.0873 |
grok-3-mini-beta | netsh | DeviceProcessEvents
| where ProcessCommandLine has_any ("netsh advfirewall", "New-NetFirewallRule")
| project FileName, ProcessCommandLine, Timestamp
| order by Timestamp desc | Correct | 1 | 18.56s | 0.0016 |
o1-high | netsh | DeviceProcessEvents
| where TimeGenerated >= ago(7d)
| where ActionType == "ProcessCreated"
| where (
ProcessCommandLine has_any ("netsh advfirewall firewall add rule", "Add-NetFirewallRule", "New-NetFirewallRule")
)
| where (
// Filter further to commands specifying inbound direction
ProcessCommandLine has_any ("dir=in", "-Direction In", "-Direction Inbound")
)
| project TimeGenerated,
DeviceName,
FolderPath,
FileName,
ProcessCommandLine,
AccountName,
InitiatingProcessFileName,
InitiatingProcessCommandLine,
InitiatingProcessFolderPath
| order by TimeGenerated desc
| limit 50 | Correct | 4 | 138.01s | 1.1520 |
o1-low | powershell.exe | DeviceProcessEvents
| where isnotempty(ProcessCommandLine)
| where ProcessCommandLine has_any ("netsh", "New-NetFirewallRule")
| where ProcessCommandLine has_any ("add rule", "firewall", "advfirewall") or ProcessCommandLine contains "New-NetFirewallRule"
| project TimeGenerated, DeviceId, DeviceName, FileName, ProcessCommandLine, InitiatingProcessCommandLine, InitiatingProcessFileName, InitiatingProcessId
| order by TimeGenerated desc | Incorrect | 2 | 88.68s | 0.5735 |
o3-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.1296 |
o3-mini-high | netsh | DeviceProcessEvents
| where isnotempty(InitiatingProcessCommandLine)
| where InitiatingProcessCommandLine has 'netsh advfirewall firewall add rule' or InitiatingProcessCommandLine has 'New-NetFirewallRule'
| project TimeGenerated, DeviceName, AccountName, InitiatingProcessFileName, InitiatingProcessCommandLine, InitiatingProcessCreationTime
| sort by TimeGenerated desc
| limit 50 | Correct | 1 | 76.05s | 0.0126 |
o3-mini-low | netsh | DeviceProcessEvents
| where (FileName has_cs "netsh.exe" and ProcessCommandLine has_cs "advfirewall")
or (FileName has_cs "powershell.exe" and ProcessCommandLine has_cs "New-NetFirewallRule")
| project TimeGenerated, DeviceName, FileName, ProcessCommandLine, InitiatingProcessCommandLine, InitiatingProcessFileName
| sort by TimeGenerated desc | Correct | 1 | 27.43s | 0.0196 |
o4-mini-high | netsh | DeviceProcessEvents
| where ProcessCommandLine matches regex @"(?i)(?:netsh\s+advfirewall\s+firewall\s+add\s+rule|New-NetFirewallRule)"
| project TimeGenerated,
DeviceName,
AccountName,
ProcessId,
FileName,
ProcessCommandLine,
InitiatingProcessId,
InitiatingProcessFileName,
InitiatingProcessCommandLine
| order by TimeGenerated desc | Correct | 3 | 88.66s | 0.0299 |
o4-mini-low | powershell.exe | DeviceProcessEvents
// Restrict to process creation events
| where ActionType == "ProcessCreated"
// Filter for netsh.exe adding firewall rules or PowerShell cmdlets
| where (
InitiatingProcessFileName == "netsh.exe" and ProcessCommandLine has_cs "advfirewall firewall add rule"
)
or (
InitiatingProcessFileName has_cs "powershell.exe" and (
ProcessCommandLine has_cs "New-NetFirewallRule" or
ProcessCommandLine has_cs "Set-NetFirewallRule"
)
)
// Project relevant fields for analysis
| project TimeGenerated, DeviceName, InitiatingProcessAccountDomain, InitiatingProcessAccountName,
InitiatingProcessFileName, ProcessId, InitiatingProcessId, ProcessCommandLine
| sort by TimeGenerated asc | Incorrect | 2 | 65.12s | 0.0188 |