Question Analysis
Windows: Investigate PowerShell process events for instances where a web client fetched and executed an external host-survey tool. What was the name of the script file that was downloaded and run?
5c16ceb4-ba3a-43d7-b848-a13c1f216d95
Question ID
3/16
Models Correct
1
Valid Answer
19%
Success Rate
Reference Answer
Human-validated correct answer for this question
Invoke-Seatbelt.ps1
Model Performance Comparison
See how different AI models performed on this question
| Model | Answer | KQL Query | Status | Attempts | Time | Cost |
|---|---|---|---|---|---|---|
| gemini-2.5-flash-preview-04-17 | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0389 |
| gpt-35-turbo | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0118 |
| gpt-4-turbo-2024-04-09 | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.2490 |
| gpt-4.1 | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0479 |
| gpt-4.1-finetuned | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0463 |
| gpt-4.1-mini | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0056 |
| gpt-4.1-nano | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0017 |
| gpt-4o | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0679 |
| grok-3-beta | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.1108 |
| grok-3-mini-beta | Discovery.bat | DeviceProcessEvents
| where InitiatingProcessFileName =~ 'powershell.exe'
| where ProcessCommandLine contains 'http' or ProcessCommandLine contains '.ps1' // Filter for potential external downloads or script executions
| extend ScriptFile = extract(@'-File\s+([^\s]+)@', 1, ProcessCommandLine) // Extract the script file name from the command line
| where isnotnull(ScriptFile) // Ensure only rows with a detected script file are kept
| summarize arg_max(TimeGenerated, *) by ScriptFile // Summarize to get the most recent event per script file
| project ScriptFile, TimeGenerated, DeviceName, ProcessCommandLine // Project key fields for review | Incorrect | 1 | 15.67s | 0.0016 |
| o1-high | Invoke-Seatbelt.ps1 | DeviceProcessEvents
| where ActionType == "ProcessCreated"
| where FileName =~ "powershell.exe"
| where ProcessCommandLine has_any ("Invoke-WebRequest", "WebClient", "DownloadString", "Invoke-Expression", "IEX")
| extend DownloadedScript = extract(@'([^/]+\.(?:ps1|psm1|ps))', 1, tostring(ProcessCommandLine))
| project TimeGenerated, DeviceName, FileName, ProcessCommandLine, DownloadedScript
| order by TimeGenerated desc | Correct | 1 | 58.24s | 0.4490 |
| o1-low | WinPwn.ps1 | // Investigate PowerShell-based downloads and execution of a script
DeviceProcessEvents
| where ActionType == "ProcessCreated"
| where InitiatingProcessFileName =~ "powershell.exe" or FileName =~ "powershell.exe"
| where ProcessCommandLine has_any ("System.Net.WebClient", "Invoke-WebRequest", "iwr", "wget", "curl", "bitsadmin", "Start-BitsTransfer", "Invoke-WebClient", "iex")
or InitiatingProcessCommandLine has_any ("System.Net.WebClient", "Invoke-WebRequest", "iwr", "wget", "curl", "bitsadmin", "Start-BitsTransfer", "Invoke-WebClient", "iex")
| extend DownloadedScript = extract(@"([\w\-.]+\.ps1)", 1, ProcessCommandLine)
| project TimeGenerated, DeviceName, AccountName, FileName, FolderPath, ProcessCommandLine, DownloadedScript
| order by TimeGenerated desc
| Incorrect | 1 | 72.44s | 0.5097 |
| o3-mini-high | Invoke-Seatbelt.ps1 | DeviceProcessEvents
| where tolower(InitiatingProcessFileName) has "powershell.exe"
| where InitiatingProcessCommandLine contains "http" and InitiatingProcessCommandLine contains ".ps1"
| extend ScriptURL = extract(@"(https?://\S+\.ps1)", 1, InitiatingProcessCommandLine)
| extend ScriptFileName = extract(@"([^/\s]+\.ps1)", 1, ScriptURL)
| where isnotempty(ScriptFileName)
| project TimeGenerated, DeviceName, InitiatingProcessCommandLine, ScriptURL, ScriptFileName
| order by TimeGenerated desc | Correct | 4 | 92.16s | 0.0395 |
| o3-mini-low | WinPwn.ps1 | DeviceProcessEvents
| where FileName =~ "powershell.exe"
| where (ProcessCommandLine contains "http" or InitiatingProcessCommandLine contains "http")
and (ProcessCommandLine contains ".ps1" or InitiatingProcessCommandLine contains ".ps1")
| extend CombinedCmdLine = coalesce(ProcessCommandLine, InitiatingProcessCommandLine)
| extend ScriptName = extract(@"([^/\\]+\.ps1)", 0, CombinedCmdLine)
| where isnotempty(ScriptName)
| project TimeGenerated, DeviceName, ScriptName, ProcessCommandLine, InitiatingProcessCommandLine | Incorrect | 2 | 30.15s | 0.0256 |
| o4-mini-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0472 |
| o4-mini-low | Invoke-Seatbelt.ps1 | DeviceProcessEvents
// (Time filter is applied externally)
| where FileName endswith "powershell.exe" or FileName endswith "pwsh.exe"
| where ProcessCommandLine has_any ("DownloadFile", "DownloadString", "Invoke-WebRequest", "WebClient", "-OutFile")
| extend ScriptFromDownloadFile = extract(@"(?i)DownloadFile\(\s*'https?://[^/]+/([^']+)'", 1, ProcessCommandLine)
| extend ScriptFromDownloadString = extract(@"(?i)DownloadString\(\s*'https?://[^/]+/([^']+\.ps1)'", 1, ProcessCommandLine)
| extend ScriptFromOutFile = extract(@"(?i)-OutFile\s*'([^']+\.ps1)'", 1, ProcessCommandLine)
| extend ScriptFile = coalesce(ScriptFromDownloadFile, ScriptFromDownloadString, ScriptFromOutFile)
| where isnotempty(ScriptFile)
| project TimeGenerated, DeviceName, InitiatingProcessAccountName, ScriptFile, ProcessCommandLine
| sort by TimeGenerated desc | Correct | 5 | 167.81s | 0.0569 |