Question Analysis
Analyze the system logs for changes that indicate the addition of a port proxy configuration via registry key alteration. Identify which port number was set to listen.
b8223ea9-4be2-44a6-b50a-9657a3d4e72a
Question ID
1/23
Models Correct
1
Valid Answer
4%
Success Rate
Reference Answer
Human-validated correct answer for this question
1337
Model Performance Comparison
See how different AI models performed on this question
Model | Answer | KQL Query | Status | Attempts | Time | Cost |
---|---|---|---|---|---|---|
gemini-2.5-flash-preview-04-17 | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0429 |
gpt-35-turbo | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0118 |
gpt-4-turbo-2024-04-09 | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.2416 |
gpt-4.1 | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0456 |
gpt-4.1-finetuned | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0496 |
gpt-4.1-mini | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0067 |
gpt-4.1-nano | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0014 |
gpt-4o | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0541 |
gpt-5-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.2303 |
gpt-5-mini-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0253 |
gpt-5-mini-low | 8080 | DeviceRegistryEvents
| where isnotempty(RegistryKey) or isnotempty(RegistryValueName) or isnotempty(RegistryValueData)
// Filter to likely PortProxy entries or things mentioning listenport/listenaddress or having an IP:port style name
| where (
tolower(RegistryKey) contains "portproxy"
or tolower(RegistryKey) contains "v4tov4"
or tolower(RegistryKey) contains "v4tov6"
or tolower(RegistryKey) contains "v6tov4"
or tolower(RegistryKey) contains "v6tov6"
or tostring(RegistryValueName) has ":"
or tolower(tostring(RegistryValueName)) has "listenport"
or tolower(tostring(RegistryValueName)) has "listenaddress"
or tostring(RegistryValueData) has ":"
or tolower(tostring(RegistryValueData)) has "listenport"
)
// Focus on creation/modification events where a port proxy would be added or changed
| where ActionType in ("SetValue", "Create", "ValueCreated", "ValueSet", "CreateKey", "Modify") or isnotempty(RegistryValueData) or isnotempty(PreviousRegistryValueData)
| extend valueNameStr = tostring(RegistryValueName), valueDataStr = tostring(RegistryValueData), prevValueDataStr = tostring(PreviousRegistryValueData), keyStr = tostring(RegistryKey)
// Try multiple extraction patterns. Namespace name often contains IP:PORT, so extract port from name first.
| extend port_from_name = extract(@":(\d{1,5})\b", 1, valueNameStr)
| extend port_from_name_suffix = extract(@"(\d{1,5})$", 1, valueNameStr)
// Extract from common textual patterns in value data (e.g., "ListenPort=8080" or ":8080")
| extend port_from_data_listen = extract(@"listenport\s*[:=]\s*(\d{1,5})", 1, tolower(valueDataStr))
| extend port_from_data_colon = extract(@":(\d{1,5})\b", 1, valueDataStr)
| extend port_from_data_anynum = extract(@"\b(\d{1,5})\b", 1, valueDataStr)
// Also check previous value data (in case of a change)
| extend port_from_prev = extract(@":(\d{1,5})\b", 1, prevValueDataStr)
// Coalesce to the first non-empty candidate and convert to int
| extend listeningPortStr = coalesce(port_from_name, port_from_name_suffix, port_from_data_listen, port_from_data_colon, port_from_prev, port_from_data_anynum)
| where isnotempty(listeningPortStr)
| extend listeningPort = toint(listeningPortStr)
| where isnotnull(listeningPort) and listeningPort between (1 .. 65535)
// Provide context and summarize
| project TimeGenerated, DeviceName, DeviceId, RegistryKey = keyStr, RegistryValueName = valueNameStr, RegistryValueData = valueDataStr, PreviousRegistryValueData = prevValueDataStr, listeningPort, ActionType, InitiatingProcessAccountName, InitiatingProcessFileName, InitiatingProcessCommandLine, InitiatingProcessId
| order by TimeGenerated desc
| summarize FirstSeen = min(TimeGenerated), LastSeen = max(TimeGenerated), Count = count() by DeviceName, DeviceId, listeningPort, RegistryKey, RegistryValueName, ActionType, InitiatingProcessFileName, InitiatingProcessAccountName, InitiatingProcessCommandLine
| order by LastSeen desc
| limit 500 | Incorrect | 2 | 47.25s | 0.0110 |
gpt-5-mini-medium | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0190 |
gpt-5-nano-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0089 |
gpt-5-nano-medium | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0078 |
grok-3-beta | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.1042 |
grok-3-mini-beta | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0075 |
o1-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.7736 |
o1-low | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.7408 |
o3-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0965 |
o3-mini-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0438 |
o3-mini-low | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0375 |
o4-mini-high | 1337 | let RegistryChanges = DeviceRegistryEvents
| where RegistryKey has "PortProxy"
| where ActionType in ("RegistryValueSet", "RegistryValueCreate")
| extend ListenPort = coalesce(
toint(extract(@"listenport[=:\s]*(\d+)", 1, RegistryValueData)),
toint(extract(@"(\d{1,5})", 1, RegistryValueName))
)
| where isnotempty(ListenPort)
| project TimeGenerated, DeviceName, Source = "Registry", RegistryKey, RegistryValueName, RegistryValueData, ListenPort;
let NetshCommands = DeviceProcessEvents
| where tolower(ProcessCommandLine) has "portproxy"
| extend ListenPort = toint(extract(@"listenport\s*=\s*(\d+)", 1, ProcessCommandLine))
| where isnotempty(ListenPort)
| project TimeGenerated, DeviceName, Source = "Process", ProcessCommandLine, ListenPort;
RegistryChanges
| union NetshCommands
| sort by TimeGenerated desc
| project TimeGenerated, DeviceName, Source, ListenPort, Details = iff(Source == "Registry", strcat("Key=", RegistryKey, "; ValueName=", RegistryValueName), ProcessCommandLine) | Correct | 1 | 51.05s | 0.0165 |
o4-mini-low | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0440 |