Question Analysis
On a Linux system, identify the script that was generated by decoding a base64 data file and then executed. What was the filename of that script?
f45df6be-2e1e-4136-a384-8f18ab3826fb
Question ID
5/23
Models Correct
1
Valid Answer
22%
Success Rate
Reference Answer
Human-validated correct answer for this question
art.sh
Model Performance Comparison
See how different AI models performed on this question
Model | Answer | KQL Query | Status | Attempts | Time | Cost |
---|---|---|---|---|---|---|
gemini-2.5-flash-preview-04-17 | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0538 |
gpt-35-turbo | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0110 |
gpt-4-turbo-2024-04-09 | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.2245 |
gpt-4.1 | /tmp/art.sh | // Step 1: Find commands that decode base64 to an output file
let decoded_scripts = DeviceProcessEvents
| where ProcessCommandLine contains "base64" and ProcessCommandLine has_any ("-d", ">", ">>")
| extend script_file = extract(@"> *([/\w\d_.-]+)", 1, ProcessCommandLine)
| where isnotempty(script_file)
| project DeviceId, script_file, decoded_time=ProcessCreationTime;
// Step 2: Look for executions of that script file on the same device soon after
DeviceProcessEvents
| join kind=inner (decoded_scripts) on DeviceId
| where (FileName == script_file or ProcessCommandLine has script_file)
| where ProcessCreationTime > decoded_time and ProcessCreationTime < decoded_time + 1h
| summarize FirstExecutionTime=min(ProcessCreationTime) by DeviceName, script_file, FileName
| project DeviceName, GeneratedAndExecutedScript=script_file, FileName_Executed=FileName, FirstExecutionTime | Correct | 2 | 15.39s | 0.0261 |
gpt-4.1-finetuned | /tmp/art.sh | // Identify base64 decoding processes with output redirected to a .sh file
DeviceProcessEvents
| where InitiatingProcessCommandLine has "base64 -d"
| where InitiatingProcessCommandLine matches regex "> .+\\.sh"
| project ParentCommandLine = InitiatingProcessCommandLine, ParentProcessName = InitiatingProcessFileName, ParentProcessId = ProcessId, DeviceId, DeviceName, TimeGenerated, AccountName, AccountDomain, AccountSid
| join kind=inner (
DeviceProcessEvents
| where InitiatingProcessCommandLine matches regex " sh "
| project ChildProcessName = InitiatingProcessFileName, ChildCommandLine = InitiatingProcessCommandLine, DeviceId, DeviceName, AccountName, AccountDomain, ProcessId, TimeGenerated
) on DeviceId, DeviceName
| project TimeGenerated, DeviceName, AccountName, AccountDomain, ParentProcessName, ParentCommandLine, ChildProcessName, ChildCommandLine, ScriptName = tostring(split(ChildCommandLine, ' ')[-1])
| sort by TimeGenerated desc
| extend Tactic = "Execution", Technique = "Linux Bash Script Execution", TechniqueID = "T1059.004"
| summarize FirstTimeDecoded = min(TimeGenerated), EventCount = count() by DeviceName, AccountName, AccountDomain, ParentProcessName, ParentCommandLine, ChildProcessName, ChildCommandLine, ScriptName, Tactic, Technique, TechniqueID
| sort by FirstTimeDecoded desc | Correct | 3 | 43.15s | 0.0340 |
gpt-4.1-mini | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0097 |
gpt-4.1-nano | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0019 |
gpt-4o | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0582 |
gpt-5-high | art.sh | let linuxDevices = DeviceInfo
| where OSPlatform =~ "Linux"
| distinct DeviceId;
let decodedOutputs = DeviceFileEvents
| where DeviceId in (linuxDevices)
| where ActionType in ("FileCreated", "FileModified")
| where isnotempty(InitiatingProcessCommandLine)
| extend ipc = tolower(InitiatingProcessCommandLine), ipf = tolower(InitiatingProcessFileName)
| where ipc has "base64" and (
ipc has " -d" or ipc has "--decode" or ipc has " enc -base64 -d" or (ipf == "openssl" and ipc has "-d")
)
| extend ScriptPath = iif(isempty(FolderPath), FileName, strcat(FolderPath, iif(FolderPath endswith "/", "", "/"), FileName))
| project DeviceId, DeviceName, DecodeTime=TimeGenerated, ScriptPath, ScriptName=FileName, DecodeProc=InitiatingProcessFileName, DecodeCmd=InitiatingProcessCommandLine;
decodedOutputs
| join kind=inner (
DeviceProcessEvents
| project DeviceId, ExecTime=TimeGenerated, ExecProc=FileName, ExecCmd=ProcessCommandLine
) on DeviceId
| where ExecTime >= DecodeTime and ExecTime <= DecodeTime + 2h
| where ExecCmd has ScriptPath or (ExecCmd has ScriptName and ExecProc in~ ("bash","sh","zsh","dash","sudo"))
| summarize FirstExecTime=min(ExecTime), ExampleExecCmd=arg_min(ExecTime, ExecCmd) by DeviceId, DeviceName, ScriptPath, ScriptName, DecodeTime, DecodeProc, DecodeCmd
| project DeviceName, ScriptFilename=ScriptName, ScriptPath, DecodeTime, FirstExecTime, DecodeProcess=DecodeProc, DecodeCmd, ExampleExecCmd
| order by FirstExecTime asc | Correct | 1 | 138.17s | 0.0692 |
gpt-5-mini-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0281 |
gpt-5-mini-low | T1140.encoded | // Find base64 decoding commands on Linux that write output to a file, then find executions of the decoded script
let decode_patterns = dynamic(["base64 -d","base64 --decode","openssl base64 -d","openssl enc -base64 -d","python -m base64","python -c 'import base64'","perl -MMIME::Base64","ruby -e 'require \"base64\"'","/usr/bin/base64","/bin/base64"]);
// Identify Linux devices
let linuxDevices = DeviceInfo
| where OSPlatform == "Linux"
| project DeviceId, DeviceName;
// Decoder commands: attempts to extract output filename
let decoders = DeviceProcessEvents
| where DeviceId in (linuxDevices | distinct DeviceId)
| where ProcessCommandLine has_any (decode_patterns)
| extend OutFileRedirect = extract(@"(?:>>|>)\s*([^;&|>\s]+)", 1, ProcessCommandLine)
| extend OutFileDashO = extract(@"-o\s+([^\s]+)", 1, ProcessCommandLine)
| extend OutFileDashDash = extract(@"--output=([^\s]+)", 1, ProcessCommandLine)
| extend OutFileOut = extract(@"-out\s+([^\s]+)", 1, ProcessCommandLine)
| extend OutFile = coalesce(OutFileRedirect, OutFileDashO, OutFileDashDash, OutFileOut)
| extend SrcB64 = extract(@"([^\s]+\.(?:b64|base64|b64txt|bin))", 1, ProcessCommandLine)
| extend DecodedFile = OutFile
| project DecodeTime = TimeGenerated, DeviceId, DeviceName, DecodingProcessFile = InitiatingProcessFileName, DecodingProcessCommandLine = ProcessCommandLine, DecodedFile, SrcB64, DecodingProcessId = ProcessId;
// Possible executions
let executions = DeviceProcessEvents
| where DeviceId in (linuxDevices | distinct DeviceId)
| project ExecTime = TimeGenerated, DeviceId, DeviceName, ExecFileName = FileName, ExecProcessCommandLine = ProcessCommandLine, ExecProcessId = ProcessId;
// Join decoders to executions within 1 hour and attempt to match by filename or basename
decoders
| where isnotempty(DecodedFile) or isnotempty(SrcB64)
| join kind=leftouter (
executions
) on DeviceId
| where ExecTime between (DecodeTime .. DecodeTime + 1h)
| extend DecodedBase = iif(isnotempty(DecodedFile), tostring(split(DecodedFile, "/")[-1]), "")
| extend SrcBaseNoExt = iif(isnotempty(SrcB64), replace_regex(SrcB64, "\\.(?:b64|base64|b64txt|bin)$", ""), "")
| where (
(isnotempty(DecodedFile) and (ExecProcessCommandLine contains DecodedFile or ExecProcessCommandLine contains DecodedBase or ExecFileName contains DecodedBase))
or
(isnotempty(SrcB64) and (ExecProcessCommandLine contains SrcBaseNoExt or ExecFileName contains SrcBaseNoExt))
)
| project DecodeTime, DeviceName, DecodingProcessFile, DecodingProcessCommandLine, DecodedFile, DecodedBase, SrcB64, ExecTime, ExecFileName, ExecProcessCommandLine
| order by DecodeTime desc
| take 50 | Incorrect | 4 | 67.74s | 0.0200 |
gpt-5-mini-medium | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0188 |
gpt-5-nano-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0108 |
gpt-5-nano-medium | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0115 |
grok-3-beta | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.1686 |
grok-3-mini-beta | base64 | DeviceInfo
| where OSPlatform == 'Linux'
| project DeviceId
| join kind=inner (DeviceProcessEvents) on DeviceId
| where ProcessCommandLine has 'base64' and (ProcessCommandLine has '-d' or ProcessCommandLine has 'decode')
| project FileName, DeviceId, ProcessCommandLine
| summarize Filename = make_set(FileName) by DeviceId // Summarizes to get unique filenames per device for clarity | Incorrect | 1 | 17.57s | 0.0016 |
o1-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 1.0427 |
o1-low | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.8529 |
o3-high | art.sh | // --------------------------------------------------------------------
// 1. Limit the investigation to Linux endpoints (optional but recommended)
// --------------------------------------------------------------------
let LinuxDevices = DeviceInfo
| where OSPlatform contains "Linux"
| project DeviceId;
// --------------------------------------------------------------------
// 2. Detect base64 decode commands and extract the output file path
// --------------------------------------------------------------------
let Base64Decode = DeviceProcessEvents
| where DeviceId in (LinuxDevices)
| where (FileName =~ "base64" or ProcessCommandLine has "base64") // the utility that is being invoked
| where ProcessCommandLine matches regex @"\s(-d|--decode)\b" // must have the decode switch
| extend OutFilePath = trim("'\"", // try to pull the resulting script path
coalesce(
extract(@"(?:>\s*)(/[^ >\t\n]+)", 1, ProcessCommandLine), // echo AAA | base64 -d > /tmp/x.sh
extract(@"(?:-o\s*)(/[^ >\t\n]+)", 1, ProcessCommandLine) // base64 -d -o /tmp/x.sh
))
| where isnotempty(OutFilePath)
| project DeviceId, DeviceName, DecodeTime = TimeGenerated, OutFilePath,
DecodeCmd = ProcessCommandLine,
ScriptFile = tostring(split(OutFilePath, "/")[-1]);
// --------------------------------------------------------------------
// 3. Look for execution of that very same file afterwards
// --------------------------------------------------------------------
let ScriptExecution = DeviceProcessEvents
| where DeviceId in (LinuxDevices)
| extend ExecPath = strcat(FolderPath, '/', FileName)
| project DeviceId, DeviceName, ExecTime = TimeGenerated, ExecPath,
ExecCmd = ProcessCommandLine, ExecFile = FileName;
// --------------------------------------------------------------------
// 4. Correlate decode -> execution (within 1 h) on the same device
// --------------------------------------------------------------------
Base64Decode
| join kind = inner ScriptExecution on DeviceId
| where ExecTime between (DecodeTime .. DecodeTime + 1h)
and (ExecPath == OutFilePath or ExecCmd has OutFilePath) // executed directly or via interpreter (bash /tmp/x.sh)
| summarize FirstDecode = min(DecodeTime), FirstExec = min(ExecTime)
by DeviceName, ScriptFile, OutFilePath, DecodeCmd
| project DeviceName, ScriptFile, FirstDecode, FirstExec, DecodeCmd
| order by FirstDecode asc | Correct | 2 | 100.86s | 0.0487 |
o3-mini-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0487 |
o3-mini-low | art.sh | DeviceProcessEvents
| where InitiatingProcessCommandLine has "base64" and (InitiatingProcessCommandLine has "-d" or InitiatingProcessCommandLine has "--decode")
| extend DecodedFile = extract(@">\s*(\S+\.sh)", 1, InitiatingProcessCommandLine)
| where isnotempty(DecodedFile)
| summarize LatestTime = arg_max(TimeGenerated, *) by DecodedFile
| project TimeGenerated = LatestTime, DecodedFile | Correct | 1 | 46.86s | 0.0107 |
o4-mini-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0337 |
o4-mini-low | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0545 |