Question Analysis
A Linux host’s Syslog shows a shell-based network discovery script ran multiple commands. One of them listed current TCP connections. Which utility was invoked?
c141bbdb-7fca-4254-9fd6-f47e79447e17
Question ID
11/23
Models Correct
1
Valid Answer
48%
Success Rate
Reference Answer
Human-validated correct answer for this question
netstat
Model Performance Comparison
See how different AI models performed on this question
Model | Answer | KQL Query | Status | Attempts | Time | Cost |
---|---|---|---|---|---|---|
gemini-2.5-flash-preview-04-17 | netstat | DeviceProcessEvents
| where ProcessCommandLine has_any ("netstat", "ss", "lsof") or FileName has_any ("netstat", "ss", "lsof")
| project Timestamp, DeviceName, FileName, ProcessCommandLine
| sort by Timestamp desc | Correct | 2 | 16.10s | 0.0079 |
gpt-35-turbo | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0108 |
gpt-4-turbo-2024-04-09 | netstat | DeviceProcessEvents
| where InitiatingProcessCommandLine has_any ('netstat', 'ss -t', 'ss -a', 'ss -r')
| project DeviceName, InitiatingProcessCommandLine, ProcessCreationTime | Correct | 1 | 10.12s | 0.1152 |
gpt-4.1 | netstat | DeviceProcessEvents
| where FileName in~ ('netstat', 'ss', 'lsof')
or ProcessCommandLine contains "netstat"
or ProcessCommandLine contains "ss "
or ProcessCommandLine contains "lsof -i"
| summarize count() by FileName, ProcessCommandLine, DeviceName, AccountName, TimeGenerated
| sort by count_ desc, TimeGenerated desc | Correct | 1 | 7.21s | 0.0325 |
gpt-4.1-finetuned | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0172 |
gpt-4.1-mini | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0084 |
gpt-4.1-nano | ss | DeviceProcessEvents | where ProcessCommandLine contains 'netstat' or ProcessCommandLine contains 'ss' | project ProcessCommandLine, InitiatingProcessCreationTime, DeviceName | order by InitiatingProcessCreationTime desc | take 50 | Incorrect | 4 | 23.63s | 0.0016 |
gpt-4o | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0524 |
gpt-5-high | netstat | let LinuxDevices = DeviceInfo
| where OSPlatform =~ "Linux"
| project DeviceId;
let syslog_cmds = DeviceEvents
| where ActionType has "Syslog"
| join kind=leftsemi (LinuxDevices) on DeviceId
| extend msg = coalesce(tostring(AdditionalFields.SyslogMessage), tostring(AdditionalFields.Message), tostring(AdditionalFields.msg), tostring(AdditionalFields.original_message), tostring(AdditionalFields.raw))
| where isnotempty(msg)
| extend Utility = extract(@"(?i)\b(ss|netstat|lsof|cat)\b", 1, msg)
| where isnotempty(Utility)
| where msg matches regex @"(?i)(\bnetstat\b.*(-t|--tcp|-an)|\bss\b.*(-t|--tcp)|\blsof\b.*(-i\s*(tcp|TCP)|-iTCP)|/proc/net/tcp(6)?)"
| project TimeGenerated, DeviceId, DeviceName, Utility = tolower(Utility), Source = "Syslog", Content = msg;
let proc_cmds = DeviceProcessEvents
| where ActionType == "ProcessCreated"
| join kind=leftsemi (LinuxDevices) on DeviceId
| where FileName in~ ("ss","netstat","lsof","cat") or ProcessCommandLine matches regex @"(?i)\b(ss|netstat|lsof|cat)\b"
| where ProcessCommandLine matches regex @"(?i)(\bnetstat\b.*(-t|--tcp|-an)|\bss\b.*(-t|--tcp)|\blsof\b.*(-i\s*(tcp|TCP)|-iTCP)|/proc/net/tcp(6)?)"
| extend Utility = case(FileName =~ "cat" and ProcessCommandLine contains "/proc/net/tcp", "cat", tolower(FileName))
| project TimeGenerated, DeviceId, DeviceName, Utility, Source = "Process", Content = ProcessCommandLine;
union syslog_cmds, proc_cmds
| summarize FirstSeen = min(TimeGenerated), LastSeen = max(TimeGenerated), Count = count(), SampleContent = arg_max(TimeGenerated, Content) by DeviceId, DeviceName, Utility
| order by LastSeen desc | Correct | 1 | 96.03s | 0.0601 |
gpt-5-mini-high | netstat | union isfuzzy=true DeviceProcessEvents, DeviceEvents
| extend cmd = coalesce(ProcessCommandLine, InitiatingProcessCommandLine)
| where isnotempty(cmd)
| extend cmd_l = tolower(cmd)
// Look for commands commonly used to list TCP connections
| where cmd_l matches regex @"\bnetstat\b" or cmd_l matches regex @"\bss\b" or cmd_l matches regex @"\blsof\b" or cmd_l contains "/proc/net/tcp"
| extend utility = case(
cmd_l matches regex @"\bnetstat\b", "netstat",
cmd_l matches regex @"\bss\b", "ss",
cmd_l matches regex @"\blsof\b", "lsof",
cmd_l contains "/proc/net/tcp", "/proc/net/tcp",
"other")
| project TimeGenerated, DeviceName, DeviceId, AccountName, FileName, ProcessId, utility, cmd
| order by TimeGenerated desc | Correct | 1 | 21.47s | 0.0073 |
gpt-5-mini-low | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0103 |
gpt-5-mini-medium | netstat | // Detect likely Linux invocations of utilities that list TCP connections (ss, netstat, lsof, cat /proc/net/tcp, awk/grep patterns)
let window = 30d;
// Process-based evidence (command lines and file names)
let proc = DeviceProcessEvents
| where TimeGenerated >= ago(window)
| where isnotempty(ProcessCommandLine) or isnotempty(FileName)
| extend cmd = tostring(ProcessCommandLine), fname = tostring(FileName)
| where cmd has_any ("ss","netstat","lsof","/proc/net/tcp","/proc/net/tcp6","cat") or fname in ("ss","netstat","lsof","cat")
| extend utility = case(
cmd matches regex '(^|\\s)ss(\\s|$)|(^|\\s)ss\\s+-', 'ss',
cmd matches regex '(^|\\s)netstat(\\s|$)|(^|\\s)netstat\\s+-', 'netstat',
cmd matches regex '(^|\\s)lsof(\\s|$)|(^|\\s)lsof\\s+-', 'lsof',
(cmd matches regex 'cat\\s+/proc/net/tcp|cat\\s+/proc/net/tcp6|grep\\s+"\\^tcp"\\s+/proc/net/tcp') or (cmd has "awk" and cmd has "/proc/net/tcp"), 'cat/proc',
fname in ("ss","netstat","lsof","cat"), fname,
'other')
| project TimeGenerated, DeviceName, AccountName, ProcessId, FileName, ProcessCommandLine, utility;
// Syslog-style messages and other DeviceEvents that may contain command strings
let evt = DeviceEvents
| where TimeGenerated >= ago(window)
| extend msg = coalesce(tostring(ProcessCommandLine), tostring(AdditionalFields))
| where isnotempty(msg)
| where msg has_any ("ss","netstat","lsof","/proc/net/tcp","cat /proc")
| extend utility = case(
msg matches regex '(^|\\s)ss(\\s|$)|(^|\\s)ss\\s+-', 'ss',
msg matches regex '(^|\\s)netstat(\\s|$)|(^|\\s)netstat\\s+-', 'netstat',
msg matches regex '(^|\\s)lsof(\\s|$)|(^|\\s)lsof\\s+-', 'lsof',
msg matches regex 'cat\\s+/proc/net/tcp|cat\\s+/proc/net/tcp6|grep\\s+"\\^tcp"\\s+/proc/net/tcp|awk', 'cat/proc',
'other')
| project TimeGenerated, DeviceName, AccountName, msg, utility;
// Combine and summarize
union proc, evt
| summarize Count = count(), Examples = make_set(coalesce(ProcessCommandLine, msg), 5), Devices = make_set(DeviceName, 5), FirstSeen = min(TimeGenerated), LastSeen = max(TimeGenerated) by utility
| order by Count desc | Correct | 4 | 99.90s | 0.0221 |
gpt-5-nano-high | ss | let StartTime = ago(1d);
let EndTime = now();
let procEvents = DeviceProcessEvents
| where TimeGenerated between (StartTime .. EndTime)
| extend cmd = tolower(InitiatingProcessCommandLine), fname = tolower(InitiatingProcessFileName)
| extend Utility = iif(fname in ("netstat","ss","lsof"), fname,
iif(cmd contains "netstat", "netstat",
iif(cmd contains "ss", "ss",
iif(cmd contains "lsof", "lsof", "Unknown"))))
| where Utility != "Unknown";
let netEvents = DeviceNetworkEvents
| where TimeGenerated between (StartTime .. EndTime)
| extend cmd = tolower(InitiatingProcessCommandLine), fname = tolower(InitiatingProcessFileName)
| extend Utility = iif(fname in ("netstat","ss","lsof"), fname,
iif(cmd contains "netstat", "netstat",
iif(cmd contains "ss", "ss",
iif(cmd contains "lsof", "lsof", "Unknown"))))
| where Utility != "Unknown";
(procEvents
| union (netEvents)
)
| summarize Count = count() by Utility
| sort by Count desc
| take 1
| project Utility, Count | Incorrect | 3 | 104.95s | 0.0062 |
gpt-5-nano-medium | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0070 |
grok-3-beta | netstat | DeviceProcessEvents
| where ProcessCommandLine contains 'netstat' or ProcessCommandLine contains 'ss' or ProcessCommandLine contains 'lsof'
| where FileName in ('netstat', 'ss', 'lsof')
| project DeviceName, FileName, ProcessCommandLine, ProcessCreationTime, InitiatingProcessFileName, InitiatingProcessCommandLine
| order by ProcessCreationTime desc | Correct | 1 | 10.47s | 0.0259 |
grok-3-mini-beta | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0074 |
o1-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.7670 |
o1-low | netstat | DeviceProcessEvents
| where DeviceId in (DeviceInfo
| where OSPlatform == "Linux"
| distinct DeviceId)
| where ActionType == "ProcessCreated"
| where ProcessCommandLine has_any ("netstat", "ss ", "lsof -i")
| project TimeGenerated, DeviceName, FileName, ProcessCommandLine, InitiatingProcessCommandLine | Correct | 1 | 34.25s | 0.3251 |
o3-high | netstat | // ---- Time window (will be overridden by the playbook) ----
let startTime = ago(24h);
let endTime = now();
//-------------------------------------------------------------
// 1. Identify Linux endpoints
let linux_devices = DeviceInfo
| where TimeGenerated between (startTime .. endTime)
| where OSPlatform startswith "Linux"
| project DeviceId, DeviceName;
// 2. Look for process-creation events that enumerate current TCP connections
DeviceProcessEvents
| where TimeGenerated between (startTime .. endTime)
| where ActionType == "ProcessCreated"
| where DeviceId in (linux_devices)
| extend cmdline = tostring(ProcessCommandLine)
| where FileName in~ ("netstat", "ss", "lsof")
or cmdline matches regex @"\b(netstat|ss|lsof)\b"
or cmdline contains "/proc/net/tcp"
// 3. Normalise the utility name
| extend Utility = case(
FileName =~ "netstat" or cmdline matches regex @"\bnetstat\b", "netstat",
FileName =~ "ss" or cmdline matches regex @"\bss\b", "ss",
FileName =~ "lsof" or (cmdline matches regex @"\blsof\b" and cmdline has "-i"), "lsof",
cmdline contains "/proc/net/tcp", "cat /proc/net/tcp",
"other")
| where Utility != "other"
// 4. Summarise so we can easily answer the question
| summarize FirstSeen = min(TimeGenerated),
LastSeen = max(TimeGenerated),
Executions = count(),
ExampleCmdline = arg_max(TimeGenerated, cmdline)
by DeviceName, Utility
| order by LastSeen desc | Correct | 3 | 104.36s | 0.0589 |
o3-mini-high | netstat | DeviceProcessEvents
| where tolower(InitiatingProcessCommandLine) contains "netstat" or tolower(InitiatingProcessCommandLine) contains " ss "
| project TimeGenerated, DeviceName, InitiatingProcessFileName, InitiatingProcessCommandLine
| distinct TimeGenerated, DeviceName, InitiatingProcessFileName, InitiatingProcessCommandLine | Correct | 3 | 35.37s | 0.0368 |
o3-mini-low | netstat | DeviceProcessEvents
| where InitiatingProcessCommandLine contains "netstat"
| project TimeGenerated, DeviceName, InitiatingProcessFileName, InitiatingProcessCommandLine
| sort by TimeGenerated desc | Correct | 2 | 44.17s | 0.0252 |
o4-mini-high | sh | DeviceInfo
| where OSPlatform == "Linux"
| project DeviceId, DeviceName
| join kind=inner (
DeviceProcessEvents
// look for commands referencing TCP connections
| where ProcessCommandLine contains "tcp"
// common utilities for listing TCP connections
| where ProcessCommandLine contains "netstat" or ProcessCommandLine contains "ss" or ProcessCommandLine contains "lsof"
) on DeviceId
| extend Utility = tostring(split(ProcessCommandLine, ' ')[0])
| summarize Invocations = count() by DeviceName, Utility
| project DeviceName, Utility, Invocations
| sort by Invocations desc | Incorrect | 1 | 45.88s | 0.0114 |
o4-mini-low | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0385 |