Question Analysis
On a Linux system, identify the script that was generated by decoding a base64 data file and then executed. What was the filename of that script?
f45df6be-2e1e-4136-a384-8f18ab3826fb
Question ID
3/16
Models Correct
1
Valid Answer
19%
Success Rate
Reference Answer
Human-validated correct answer for this question
art.sh
Model Performance Comparison
See how different AI models performed on this question
Model | Answer | KQL Query | Status | Attempts | Time | Cost |
---|---|---|---|---|---|---|
gemini-2.5-flash-preview-04-17 | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0538 |
gpt-35-turbo | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0110 |
gpt-4-turbo-2024-04-09 | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.2245 |
gpt-4.1 | /tmp/art.sh | // Step 1: Find commands that decode base64 to an output file
let decoded_scripts = DeviceProcessEvents
| where ProcessCommandLine contains "base64" and ProcessCommandLine has_any ("-d", ">", ">>")
| extend script_file = extract(@"> *([/\w\d_.-]+)", 1, ProcessCommandLine)
| where isnotempty(script_file)
| project DeviceId, script_file, decoded_time=ProcessCreationTime;
// Step 2: Look for executions of that script file on the same device soon after
DeviceProcessEvents
| join kind=inner (decoded_scripts) on DeviceId
| where (FileName == script_file or ProcessCommandLine has script_file)
| where ProcessCreationTime > decoded_time and ProcessCreationTime < decoded_time + 1h
| summarize FirstExecutionTime=min(ProcessCreationTime) by DeviceName, script_file, FileName
| project DeviceName, GeneratedAndExecutedScript=script_file, FileName_Executed=FileName, FirstExecutionTime | Correct | 2 | 15.39s | 0.0261 |
gpt-4.1-finetuned | /tmp/art.sh | // Identify base64 decoding processes with output redirected to a .sh file
DeviceProcessEvents
| where InitiatingProcessCommandLine has "base64 -d"
| where InitiatingProcessCommandLine matches regex "> .+\\.sh"
| project ParentCommandLine = InitiatingProcessCommandLine, ParentProcessName = InitiatingProcessFileName, ParentProcessId = ProcessId, DeviceId, DeviceName, TimeGenerated, AccountName, AccountDomain, AccountSid
| join kind=inner (
DeviceProcessEvents
| where InitiatingProcessCommandLine matches regex " sh "
| project ChildProcessName = InitiatingProcessFileName, ChildCommandLine = InitiatingProcessCommandLine, DeviceId, DeviceName, AccountName, AccountDomain, ProcessId, TimeGenerated
) on DeviceId, DeviceName
| project TimeGenerated, DeviceName, AccountName, AccountDomain, ParentProcessName, ParentCommandLine, ChildProcessName, ChildCommandLine, ScriptName = tostring(split(ChildCommandLine, ' ')[-1])
| sort by TimeGenerated desc
| extend Tactic = "Execution", Technique = "Linux Bash Script Execution", TechniqueID = "T1059.004"
| summarize FirstTimeDecoded = min(TimeGenerated), EventCount = count() by DeviceName, AccountName, AccountDomain, ParentProcessName, ParentCommandLine, ChildProcessName, ChildCommandLine, ScriptName, Tactic, Technique, TechniqueID
| sort by FirstTimeDecoded desc | Correct | 3 | 43.15s | 0.0340 |
gpt-4.1-mini | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0097 |
gpt-4.1-nano | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0019 |
gpt-4o | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0582 |
grok-3-beta | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.1686 |
grok-3-mini-beta | base64 | DeviceInfo
| where OSPlatform == 'Linux'
| project DeviceId
| join kind=inner (DeviceProcessEvents) on DeviceId
| where ProcessCommandLine has 'base64' and (ProcessCommandLine has '-d' or ProcessCommandLine has 'decode')
| project FileName, DeviceId, ProcessCommandLine
| summarize Filename = make_set(FileName) by DeviceId // Summarizes to get unique filenames per device for clarity | Incorrect | 1 | 17.57s | 0.0016 |
o1-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 1.0427 |
o1-low | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.8529 |
o3-mini-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0487 |
o3-mini-low | art.sh | DeviceProcessEvents
| where InitiatingProcessCommandLine has "base64" and (InitiatingProcessCommandLine has "-d" or InitiatingProcessCommandLine has "--decode")
| extend DecodedFile = extract(@">\s*(\S+\.sh)", 1, InitiatingProcessCommandLine)
| where isnotempty(DecodedFile)
| summarize LatestTime = arg_max(TimeGenerated, *) by DecodedFile
| project TimeGenerated = LatestTime, DecodedFile | Correct | 1 | 46.86s | 0.0107 |
o4-mini-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0337 |
o4-mini-low | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0545 |