Question Analysis
Windows: Investigate PowerShell process events for instances where a web client fetched and executed an external host-survey tool. What was the name of the script file that was downloaded and run?
5c16ceb4-ba3a-43d7-b848-a13c1f216d95
Question ID
4/23
Models Correct
1
Valid Answer
17%
Success Rate
Reference Answer
Human-validated correct answer for this question
Invoke-Seatbelt.ps1
Model Performance Comparison
See how different AI models performed on this question
Model | Answer | KQL Query | Status | Attempts | Time | Cost |
---|---|---|---|---|---|---|
gemini-2.5-flash-preview-04-17 | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0389 |
gpt-35-turbo | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0118 |
gpt-4-turbo-2024-04-09 | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.2490 |
gpt-4.1 | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0479 |
gpt-4.1-finetuned | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0463 |
gpt-4.1-mini | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0056 |
gpt-4.1-nano | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0017 |
gpt-4o | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0679 |
gpt-5-high | Invoke-Seatbelt.ps1 | let ps_bins = dynamic(["powershell.exe","pwsh.exe","powershell_ise.exe"]);
let dl_tokens = dynamic(["system.net.webclient","invoke-webrequest","iwr","wget","curl","start-bitstransfer","bitstransfer","downloadfile","downloadstring","invoke-restmethod","irm","new-object net.webclient"]);
let survey_tokens = dynamic(["seatbelt","jaws","winpeas","powerview","powerup","privesc","recon","survey","enum","discovery","inventory","audit","hostrecon","hostenum"]);
// 1) PowerShell saves a .ps1 from the web and later executes it
let saved =
DeviceFileEvents
| where ActionType == 'FileCreated'
| where FileName endswith '.ps1' or tostring(FileOriginUrl) contains '.ps1'
| extend ipfn = tolower(tostring(InitiatingProcessFileName)),
ipcl = tolower(tostring(InitiatingProcessCommandLine)),
fromUrl = iif(isnotempty(FileOriginUrl) and ((tolower(FileOriginUrl) startswith 'http://') or (tolower(FileOriginUrl) startswith 'https://')), FileOriginUrl, '')
| where ipfn in~ (ps_bins) or ipcl contains 'powershell' or ipcl contains 'pwsh'
| where ipcl has_any (dl_tokens) or isnotempty(fromUrl)
| project DeviceId, DeviceName, DownloadTime=TimeGenerated, ScriptFileName=FileName, ScriptPath=FolderPath, SourceUrl=fromUrl, DownloaderPid=InitiatingProcessId, DownloaderCmd=InitiatingProcessCommandLine;
let exec =
DeviceProcessEvents
| where FileName in~ (ps_bins)
| extend lc = tolower(tostring(ProcessCommandLine))
| project DeviceId, ExecTime=TimeGenerated, ExecCmd=ProcessCommandLine, lc, ExecPid=ProcessId;
let saved_and_run =
saved
| join kind=inner (exec) on DeviceId
| where ExecTime between (DownloadTime .. DownloadTime + 1d)
| where lc contains tolower(ScriptFileName) or (isnotempty(ScriptPath) and lc contains tolower(ScriptPath))
| extend MatchType = 'SavedThenExecuted'
| project DeviceName, DeviceId, FirstDownloadTime=DownloadTime, FirstExecTime=ExecTime, ScriptFileName, ScriptPath, SourceUrl, ExecCmd, DownloaderCmd, MatchType;
// 2) In-memory fetch + execute (URL with .ps1 in the PowerShell cmdline + IEX)
let inmem =
DeviceProcessEvents
| where FileName in~ (ps_bins)
| extend ec = tostring(ProcessCommandLine), lc = tolower(tostring(ProcessCommandLine))
| where lc has_any (dl_tokens)
| where lc contains '.ps1'
| extend SourceUrl = tostring(extract('https?://[^\\s\'"<>]+\\.ps1', 0, ec))
| extend ScriptFileName = tostring(extract('([^/?#\\\\]+\\.ps1)', 1, SourceUrl))
| where isnotempty(ScriptFileName)
| where lc has_any (dynamic(['iex','invoke-expression'])) or lc contains '|iex' or lc contains '| iex'
| extend FirstDownloadTime=TimeGenerated, FirstExecTime=TimeGenerated, ScriptPath='', MatchType = 'InMemoryFetchExec'
| project DeviceName, DeviceId, FirstDownloadTime, FirstExecTime, ScriptFileName, ScriptPath, SourceUrl, ExecCmd=ec, DownloaderCmd=ec, MatchType;
union saved_and_run, inmem
| extend IsHostSurvey = iif(tolower(coalesce(ScriptFileName, '')) has_any (survey_tokens) or tolower(coalesce(ExecCmd, '')) has_any (survey_tokens) or tolower(coalesce(SourceUrl, '')) has_any (survey_tokens), true, false)
| where IsHostSurvey
| summarize FirstDownloadTime=min(FirstDownloadTime), FirstExecTime=min(FirstExecTime), SourceUrl=any(SourceUrl), ExecCmd=any(ExecCmd), DownloaderCmd=any(DownloaderCmd), MatchType=any(MatchType) by DeviceName, DeviceId, ScriptFileName
| order by FirstExecTime asc | Correct | 4 | 444.55s | 0.2509 |
gpt-5-mini-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0221 |
gpt-5-mini-low | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0258 |
gpt-5-mini-medium | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0175 |
gpt-5-nano-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0107 |
gpt-5-nano-medium | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0097 |
grok-3-beta | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.1108 |
grok-3-mini-beta | Discovery.bat | DeviceProcessEvents
| where InitiatingProcessFileName =~ 'powershell.exe'
| where ProcessCommandLine contains 'http' or ProcessCommandLine contains '.ps1' // Filter for potential external downloads or script executions
| extend ScriptFile = extract(@'-File\s+([^\s]+)@', 1, ProcessCommandLine) // Extract the script file name from the command line
| where isnotnull(ScriptFile) // Ensure only rows with a detected script file are kept
| summarize arg_max(TimeGenerated, *) by ScriptFile // Summarize to get the most recent event per script file
| project ScriptFile, TimeGenerated, DeviceName, ProcessCommandLine // Project key fields for review | Incorrect | 1 | 15.67s | 0.0016 |
o1-high | Invoke-Seatbelt.ps1 | DeviceProcessEvents
| where ActionType == "ProcessCreated"
| where FileName =~ "powershell.exe"
| where ProcessCommandLine has_any ("Invoke-WebRequest", "WebClient", "DownloadString", "Invoke-Expression", "IEX")
| extend DownloadedScript = extract(@'([^/]+\.(?:ps1|psm1|ps))', 1, tostring(ProcessCommandLine))
| project TimeGenerated, DeviceName, FileName, ProcessCommandLine, DownloadedScript
| order by TimeGenerated desc | Correct | 1 | 58.24s | 0.4490 |
o1-low | WinPwn.ps1 | // Investigate PowerShell-based downloads and execution of a script
DeviceProcessEvents
| where ActionType == "ProcessCreated"
| where InitiatingProcessFileName =~ "powershell.exe" or FileName =~ "powershell.exe"
| where ProcessCommandLine has_any ("System.Net.WebClient", "Invoke-WebRequest", "iwr", "wget", "curl", "bitsadmin", "Start-BitsTransfer", "Invoke-WebClient", "iex")
or InitiatingProcessCommandLine has_any ("System.Net.WebClient", "Invoke-WebRequest", "iwr", "wget", "curl", "bitsadmin", "Start-BitsTransfer", "Invoke-WebClient", "iex")
| extend DownloadedScript = extract(@"([\w\-.]+\.ps1)", 1, ProcessCommandLine)
| project TimeGenerated, DeviceName, AccountName, FileName, FolderPath, ProcessCommandLine, DownloadedScript
| order by TimeGenerated desc
| Incorrect | 1 | 72.44s | 0.5097 |
o3-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.1098 |
o3-mini-high | Invoke-Seatbelt.ps1 | DeviceProcessEvents
| where tolower(InitiatingProcessFileName) has "powershell.exe"
| where InitiatingProcessCommandLine contains "http" and InitiatingProcessCommandLine contains ".ps1"
| extend ScriptURL = extract(@"(https?://\S+\.ps1)", 1, InitiatingProcessCommandLine)
| extend ScriptFileName = extract(@"([^/\s]+\.ps1)", 1, ScriptURL)
| where isnotempty(ScriptFileName)
| project TimeGenerated, DeviceName, InitiatingProcessCommandLine, ScriptURL, ScriptFileName
| order by TimeGenerated desc | Correct | 4 | 92.16s | 0.0395 |
o3-mini-low | WinPwn.ps1 | DeviceProcessEvents
| where FileName =~ "powershell.exe"
| where (ProcessCommandLine contains "http" or InitiatingProcessCommandLine contains "http")
and (ProcessCommandLine contains ".ps1" or InitiatingProcessCommandLine contains ".ps1")
| extend CombinedCmdLine = coalesce(ProcessCommandLine, InitiatingProcessCommandLine)
| extend ScriptName = extract(@"([^/\\]+\.ps1)", 0, CombinedCmdLine)
| where isnotempty(ScriptName)
| project TimeGenerated, DeviceName, ScriptName, ProcessCommandLine, InitiatingProcessCommandLine | Incorrect | 2 | 30.15s | 0.0256 |
o4-mini-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0472 |
o4-mini-low | Invoke-Seatbelt.ps1 | DeviceProcessEvents
// (Time filter is applied externally)
| where FileName endswith "powershell.exe" or FileName endswith "pwsh.exe"
| where ProcessCommandLine has_any ("DownloadFile", "DownloadString", "Invoke-WebRequest", "WebClient", "-OutFile")
| extend ScriptFromDownloadFile = extract(@"(?i)DownloadFile\(\s*'https?://[^/]+/([^']+)'", 1, ProcessCommandLine)
| extend ScriptFromDownloadString = extract(@"(?i)DownloadString\(\s*'https?://[^/]+/([^']+\.ps1)'", 1, ProcessCommandLine)
| extend ScriptFromOutFile = extract(@"(?i)-OutFile\s*'([^']+\.ps1)'", 1, ProcessCommandLine)
| extend ScriptFile = coalesce(ScriptFromDownloadFile, ScriptFromDownloadString, ScriptFromOutFile)
| where isnotempty(ScriptFile)
| project TimeGenerated, DeviceName, InitiatingProcessAccountName, ScriptFile, ProcessCommandLine
| sort by TimeGenerated desc | Correct | 5 | 167.81s | 0.0569 |