Question Analysis
An analyst notices a PowerShell process on a Windows host that appears to be checking SMB connectivity. Which PowerShell cmdlet was executed to perform this outbound port 445 test?
d9c32b3b-7916-45ad-aca5-6c902da80319
Question ID
9/22
Models Correct
1
Valid Answer
41%
Success Rate
Reference Answer
Human-validated correct answer for this question
Test-NetConnection
Model Performance Comparison
See how different AI models performed on this question
Model | Answer | KQL Query | Status | Attempts | Time | Cost |
---|---|---|---|---|---|---|
gemini-2.5-flash-preview-04-17 | Test-NetConnection | DeviceNetworkEvents
| where RemotePort == 445
| where InitiatingProcessFileName =~ "powershell.exe"
| distinct InitiatingProcessCommandLine | Correct | 1 | 12.16s | 0.0057 |
gpt-35-turbo | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0114 |
gpt-4-turbo-2024-04-09 | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.3084 |
gpt-4.1 | New-Object | DeviceProcessEvents
| where (FileName in~ ('powershell.exe', 'pwsh.exe'))
| where ProcessCommandLine has_any ('Test-NetConnection', 'System.Net.Sockets.TcpClient', 'Test-Connection', 'Invoke-Expression', 'New-Object') and ProcessCommandLine has '445'
| project TimeGenerated, DeviceName, AccountName, FileName, ProcessCommandLine
| order by TimeGenerated desc | Incorrect | 2 | 11.38s | 0.0248 |
gpt-4.1-finetuned | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0394 |
gpt-4.1-mini | Test-NetConnection | DeviceProcessEvents
| where FileName in ("powershell.exe", "pwsh.exe")
| join kind=inner (
DeviceNetworkEvents
| where RemotePort == 445
| project InitiatingProcessId, RemotePort
) on $left.ProcessId == $right.InitiatingProcessId
| project TimeGenerated, DeviceName, InitiatingProcessAccountName, ProcessCommandLine
| order by TimeGenerated desc
| take 50 | Correct | 4 | 16.37s | 0.0182 |
gpt-4.1-nano | Test-NetConnection -CommonTCPPort SMB | DeviceProcessEvents
| where ProcessCommandLine contains "PowerShell" or FileName endswith ".ps1"
| where ProcessCommandLine contains "Test-NetConnection" or ProcessCommandLine contains "Invoke-Expression" or ProcessCommandLine contains "net" or ProcessCommandLine contains "smb"
| project TimeGenerated, DeviceName, ProcessCommandLine, ProcessId, InitiatingProcessFileName, InitiatingProcessParentFileName | Correct | 2 | 3.23s | 0.0013 |
gpt-4o | Invoke-AtomicTest | DeviceProcessEvents
| where InitiatingProcessFileName =~ "powershell.exe" and ProcessCommandLine has "445"
| project TimeGenerated, DeviceName, AccountName, InitiatingProcessCommandLine, InitiatingProcessVersionInfoFileDescription | Incorrect | 3 | 9.38s | 0.0319 |
gpt-5-high | Test-NetConnection | let PowerShellCmds =
DeviceEvents
| where ActionType has "PowerShell"
| extend Command = tostring(coalesce(AdditionalFields.Command, AdditionalFields.ScriptCommand, AdditionalFields.ScriptContent, AdditionalFields.ScriptName))
| where isnotempty(Command)
| extend Cmdlet = case(
Command matches regex @"(?i)\b(?:test-netconnection|tnc)\b", "Test-NetConnection",
Command matches regex @"(?i)\btest-connection\b", "Test-Connection",
""
)
| where Cmdlet != ""
// Ensure it was a port 445 check (SMB)
| where Command matches regex @"(?i)(?:-port\s*=?\s*445|-tcpport\s*=?\s*445|-CommonTCPPort\s+SMB|:445\b)"
| project TimeGenerated, DeviceName, AccountName=InitiatingProcessAccountName, Cmdlet, Command, Source="DeviceEvents";
let PowerShellProc =
DeviceProcessEvents
| where FileName in~ ("powershell.exe","pwsh.exe","powershell_ise.exe")
| where isnotempty(ProcessCommandLine)
| extend Cmdlet = case(
ProcessCommandLine matches regex @"(?i)\b(?:test-netconnection|tnc)\b", "Test-NetConnection",
ProcessCommandLine matches regex @"(?i)\btest-connection\b", "Test-Connection",
""
)
| where Cmdlet != ""
// Ensure it was a port 445 check (SMB)
| where ProcessCommandLine matches regex @"(?i)(?:-port\s*=?\s*445|-tcpport\s*=?\s*445|-CommonTCPPort\s+SMB|:445\b)"
| project TimeGenerated, DeviceName, AccountName, Cmdlet, Command=ProcessCommandLine, Source="DeviceProcessEvents";
union PowerShellCmds, PowerShellProc
| summarize FirstSeen=min(TimeGenerated), LastSeen=max(TimeGenerated), Samples=make_set(Command, 3) by DeviceName, AccountName, Cmdlet
| order by LastSeen desc | Correct | 2 | 146.93s | 0.1137 |
gpt-5-mini-high | Test-NetConnection | DeviceNetworkEvents
| where RemotePort == 445
| where tolower(tostring(InitiatingProcessFileName)) contains "powershell"
| project TimeGenerated, DeviceName, RemoteIP, RemotePort, InitiatingProcessId, InitiatingProcessFileName, InitiatingProcessCommandLine
| extend CmdLine = tostring(InitiatingProcessCommandLine)
// capture known SMB/test cmdlets first, otherwise fall back to the first verb-noun token
| extend KnownCmdlet = extract(@"(?i)\b(Test-NetConnection|Test-Connection|New-SmbMapping|Get-SmbConnection|Get-SmbShare|New-PSDrive|Test-Path|Invoke-Command|net use)\b", 1, CmdLine)
| extend FirstVerbNoun = extract(@"(?i)\b([a-z]+-[a-z0-9]+)\b", 1, CmdLine)
| extend Cmdlet = iif(isnotempty(KnownCmdlet), KnownCmdlet, FirstVerbNoun)
| where isnotempty(Cmdlet)
| summarize Count = count(), FirstSeen = min(TimeGenerated), LastSeen = max(TimeGenerated), SampleDevice = any(DeviceName), SampleInitiatingProcessId = any(InitiatingProcessId), SampleCommandLine = any(CmdLine), RemoteIPs = make_set(RemoteIP, 20) by Cmdlet
| order by Count desc | Correct | 2 | 36.56s | 0.0098 |
gpt-5-mini-low | Test-NetConnection | let targetPorts = dynamic([445]);
let psNames = dynamic(['powershell.exe','pwsh.exe']);
DeviceNetworkEvents
| where RemotePort in (targetPorts) and Protocol =~ 'TCP'
| where InitiatingProcessFileName in (psNames) or InitiatingProcessFileName has 'powershell' or InitiatingProcessFileName has 'pwsh'
| project NetTime = TimeGenerated, DeviceName, InitiatingProcessId, InitiatingProcessFileName, InitiatingProcessCommandLine, InitiatingProcessAccountName, RemoteIP, RemotePort, Protocol
| join kind=leftouter (
DeviceProcessEvents
| where FileName in (psNames) or FileName has 'powershell' or ProcessCommandLine has 'powershell'
| project ProcTime = TimeGenerated, DeviceName, ProcessId, ProcessCommandLine, AccountName = InitiatingProcessAccountName, FileName
) on DeviceName, $left.InitiatingProcessId == $right.ProcessId
| extend CmdLine = coalesce(InitiatingProcessCommandLine, ProcessCommandLine)
| project Time = NetTime, DeviceName, RemoteIP, RemotePort, Protocol, InitiatingProcessFileName, CmdLine, InitiatingProcessAccountName
| where CmdLine has_any ('Test-NetConnection','Test-Connection','New-SmbMapping','Test-SmbPath','New-Object System.Net.Sockets.TcpClient','Test-Path')
| distinct Time, DeviceName, RemoteIP, RemotePort, Protocol, InitiatingProcessFileName, CmdLine, InitiatingProcessAccountName
| order by Time desc | Correct | 2 | 32.78s | 0.0054 |
gpt-5-nano-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0090 |
gpt-5-nano-medium | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0066 |
grok-3-beta | ConnectAsync | DeviceNetworkEvents
| where InitiatingProcessFileName =~ 'powershell.exe'
| where RemotePort == 445
| where ActionType == 'ConnectionSuccess' or ActionType == 'OutboundConnection'
| project Timestamp, DeviceName, InitiatingProcessFileName, InitiatingProcessCommandLine, RemoteIP, RemotePort
| order by Timestamp desc
| limit 100 | Incorrect | 1 | 12.62s | 0.0235 |
grok-3-mini-beta | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0074 |
o1-high | Test-NetConnection | DeviceProcessEvents
| where ActionType == "ProcessCreated"
| where FileName in~ ("powershell.exe","pwsh.exe","powershell_ise.exe")
// Look for references to a port test in command line
| where ProcessCommandLine has_any ("Test-NetConnection", "Test-Connection", "-TcpPort", "445")
| project TimeGenerated, DeviceName, AccountName, InitiatingProcessCommandLine, ProcessCommandLine
| order by TimeGenerated desc | Correct | 3 | 89.57s | 0.5518 |
o1-low | Test-NetConnection | // This query checks both DeviceProcessEvents and DeviceNetworkEvents to find
// PowerShell commands that used the 'Test-NetConnection' (or 'tnc') cmdlet
// specifically mentioning port 445.
union DeviceProcessEvents, DeviceNetworkEvents
| where TimeGenerated >= ago(7d) // Adjust time range as needed
| where (FileName in~ ("powershell.exe", "pwsh.exe") or InitiatingProcessFileName in~ ("powershell.exe", "pwsh.exe"))
| where tostring(ProcessCommandLine) has_any ("Test-NetConnection", "tnc") or tostring(InitiatingProcessCommandLine) has_any ("Test-NetConnection", "tnc")
| where tostring(ProcessCommandLine) has "445" or tostring(InitiatingProcessCommandLine) has "445" or RemotePort == 445
| project TimeGenerated, DeviceName, Type,
FileName, ProcessCommandLine, InitiatingProcessFileName, InitiatingProcessCommandLine,
RemoteIP, RemotePort
| order by TimeGenerated desc | Correct | 2 | 52.68s | 0.3693 |
o3-high | Write-Host | // 1. Locate network connections to TCP/445 that were initiated by a PowerShell process
let ps445 = DeviceNetworkEvents
| where RemotePort == 445 // SMB over TCP
| where InitiatingProcessFileName in~ ("powershell.exe","pwsh.exe","powershell_ise.exe")
| project TimeGenerated,
DeviceId,
DeviceName,
AccountName = InitiatingProcessAccountName,
CmdLine = InitiatingProcessCommandLine,
RemoteIP;
// 2. Extract the PowerShell cmdlet from the command-line text
ps445
| extend Cmdlet = coalesce(
tostring(extract(@"(?i)(?:-Command\s+|\s|^)((?:[A-Za-z]+-)+[A-Za-z]+)",1,CmdLine)), // first verb-noun token after -Command or at beginning
"<Unparsed>")
| summarize FirstSeen = min(TimeGenerated),
LastSeen = max(TimeGenerated),
Executions = count() by Cmdlet, DeviceName, AccountName
| order by Executions desc | Incorrect | 1 | 32.92s | 0.0229 |
o3-mini-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0339 |
o3-mini-low | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0396 |
o4-mini-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0421 |
o4-mini-low | Test-NetConnection | DeviceNetworkEvents
| where InitiatingProcessFileName == "powershell.exe" // only PowerShell processes
| where RemotePort == 445 // SMB port
| project Timestamp, DeviceName, InitiatingProcessId, InitiatingProcessCommandLine
| distinct InitiatingProcessCommandLine | Correct | 1 | 48.94s | 0.0142 |