// question
A Linux system shows a 'find' command used to search within .aws directories. Which specific AWS credential filename was the attacker attempting to locate?
Reference KQL answer and model-by-model comparison.
37807632-d3da-442e-8c2e-00f44928ff8f
question id
9/23
models correct
1
valid answer
39%
success rate
// reference answer
Human-validated correct answer for this question.
credentials
// model comparison
| Model | Answer | KQL query | Status | Tries | Time | Cost |
|---|---|---|---|---|---|---|
| gemini-2.5-flash-preview-04-17 | credentials | DeviceProcessEvents
| where ActionType == "ProcessCreated"
| where FileName =~ "find"
| where ProcessCommandLine contains ".aws"
| where ProcessCommandLine contains "-name" or ProcessCommandLine contains "-iname"
| extend searchedFilename = extract(@'-i?name\s+(".*?"|\S+)', 1, ProcessCommandLine)
| project Timestamp, DeviceName, InitiatingProcessAccountName, ProcessCommandLine, searchedFilename | pass | 1 | 11.46s | 0.0048 |
| gpt-35-turbo | no answer | no query generated | fail | 5 | 0.00s | 0.0109 |
| gpt-4-turbo-2024-04-09 | no answer | no query generated | fail | 5 | 0.00s | 0.2321 |
| gpt-4.1 | no answer | no query generated | fail | 5 | 0.00s | 0.0467 |
| gpt-4.1-finetuned | no answer | no query generated | fail | 5 | 0.00s | 0.0685 |
| gpt-4.1-mini | no answer | no query generated | fail | 5 | 0.00s | 0.0080 |
| gpt-4.1-nano | credentials | DeviceProcessEvents | where ProcessCommandLine contains "find" and ProcessCommandLine contains ".aws" | where ProcessCommandLine contains "credentials" or ProcessCommandLine contains "credential" | project TimeGenerated, DeviceName, ProcessId, InitiatingProcessFileName, ProcessCommandLine, InitiatingProcessParentFileName, InitiatingProcessParentId | pass | 1 | 2.04s | 0.0005 |
| gpt-4o | credentials | DeviceProcessEvents | where InitiatingProcessCommandLine contains 'find' and InitiatingProcessCommandLine has '.aws' | where InitiatingProcessCommandLine matches regex @'(?i)(credentials|config|keys)' | project TimeGenerated, DeviceName, InitiatingProcessCommandLine, FolderPath | pass | 3 | 23.69s | 0.0344 |
| gpt-5-high | credentials | let linux_devices = DeviceInfo
| where OSPlatform == "Linux"
| project DeviceId;
DeviceProcessEvents
| where ActionType == "ProcessCreated"
| join kind=inner (linux_devices) on DeviceId
| where tolower(FileName) == "find" or ProcessCommandLine has " find "
| where ProcessCommandLine has ".aws" and ProcessCommandLine has_any ("-name", "-iname")
// Extract the searched pattern after -name / -iname (handles "...", '...', or unquoted)
| extend SearchedName_dq = extract(@'-i?name\s+"([^"]+)"', 1, ProcessCommandLine),
SearchedName_sq = extract(@'-i?name\s+''([^'']+)''', 1, ProcessCommandLine),
SearchedName_nq = extract(@'-i?name\s+([^\s]+)', 1, ProcessCommandLine)
| extend SearchedName = coalesce(SearchedName_dq, SearchedName_sq, SearchedName_nq)
| where isnotempty(SearchedName)
// Normalize: lowercase, trim quotes/spaces, remove wildcard characters
| extend CredentialFilename = replace_string(tolower(trim(" *'\"", SearchedName)), "*", "")
| summarize Count = count(), FirstSeen = min(TimeGenerated), LastSeen = max(TimeGenerated), SampleCommand = any(ProcessCommandLine) by CredentialFilename
| order by Count desc | pass | 2 | 146.50s | 0.0836 |
| gpt-5-mini-high | credentials | DeviceProcessEvents
| where ProcessCommandLine has "find" and ProcessCommandLine has ".aws"
// try to capture the argument after -name or -iname
| extend name_arg = extract("(?i)(?:-name|-iname)\\s+([^ ]+)", 1, ProcessCommandLine)
// try to capture .aws/<file> with forward or back slashes
| extend dotaws_path = extract("(?i)\\.aws/([^ ]+)", 1, ProcessCommandLine)
| extend dotaws_path_bs = extract("(?i)\\.aws\\\\([^ ]+)", 1, ProcessCommandLine)
// capture common credential/config filenames mentioned directly
| extend direct_common = extract("(?i)\\b(credentials(?:\\.json)?|config|access_key|secret_access_key|id_rsa|credentials\\.yml|credentials\\.yaml)\\b", 1, ProcessCommandLine)
// pick the first non-empty candidate and lowercase it
| extend candidate = tolower(coalesce(name_arg, dotaws_path, dotaws_path_bs, direct_common, ""))
| where candidate != ""
| summarize Count = count(), Examples = make_set(ProcessCommandLine, 5) by Candidate = candidate
| sort by Count desc | pass | 3 | 47.84s | 0.0128 |
| gpt-5-mini-low | no answer | no query generated | fail | 5 | 0.00s | 0.0230 |
| gpt-5-mini-medium | credentials | union DeviceProcessEvents, DeviceEvents, DeviceFileEvents | extend CommandLine = tolower(coalesce(ProcessCommandLine, InitiatingProcessCommandLine, "")) | where CommandLine contains "find" and CommandLine contains ".aws" | extend cmd = replace_string(CommandLine, "\\", "/") // take a reasonable window starting at the .aws occurrence | extend after = substring(cmd, indexof(cmd, ".aws"), 300) // get the path segment immediately after .aws/ if present | extend aws_next = tostring(split(after, "/")[1]) // try to parse explicit find arguments (-name, -iname, -path, -wholename) | extend nameToken1 = iff(indexof(cmd, "-name ") >= 0, tostring(split(substring(cmd, indexof(cmd, "-name ") + 6, 200), " ")[0]), "") | extend nameToken2 = iff(indexof(cmd, "-iname ") >= 0, tostring(split(substring(cmd, indexof(cmd, "-iname ") + 7, 200), " ")[0]), "") | extend nameToken3 = iff(indexof(cmd, "-path ") >= 0, tostring(split(substring(cmd, indexof(cmd, "-path ") + 6, 200), " ")[0]), "") | extend nameToken4 = iff(indexof(cmd, "-wholename ") >= 0, tostring(split(substring(cmd, indexof(cmd, "-wholename ") + 11, 200), " ")[0]), "") | extend nameRaw = case(nameToken1 != "", nameToken1, nameToken2 != "", nameToken2, nameToken3 != "", nameToken3, nameToken4 != "", nameToken4, "") | extend RawTarget = iff(nameRaw != "", nameRaw, aws_next) // strip surrounding quotes and wildcard characters | extend TargetFilename = replace_regex(replace_regex(RawTarget, "^[\\\"'\\*]+", ""), "[\\\"'\\*]+$", "") | where isnotempty(TargetFilename) | summarize FirstSeen = min(TimeGenerated), LastSeen = max(TimeGenerated), Count = count(), ExampleCommand = any(CommandLine) by DeviceId, DeviceName, AccountName, TargetFilename | project FirstSeen, LastSeen, Count, DeviceName, DeviceId, AccountName, TargetFilename, ExampleCommand | order by LastSeen desc | pass | 4 | 101.35s | 0.0246 |
| gpt-5-nano-high | no answer | no query generated | fail | 5 | 0.00s | 0.0092 |
| gpt-5-nano-medium | no answer | no query generated | fail | 5 | 0.00s | 0.0084 |
| grok-3-beta | no answer | no query generated | fail | 5 | 0.00s | 0.0848 |
| grok-3-mini-beta | credentials | DeviceInfo
| where OSPlatform == "Linux"
| join kind=inner (
DeviceProcessEvents
| where ProcessCommandLine contains "find" and ProcessCommandLine contains ".aws"
| extend filename = extract(@"-name\s+([^\s]+)", 1, ProcessCommandLine)
| project DeviceId, ProcessCommandLine, filename
) on DeviceId
| project DeviceName, ProcessCommandLine, filename
| summarize UniqueFilenames = make_set(filename) by DeviceName
| project DeviceName, UniqueFilenames | pass | 1 | 13.97s | 0.0016 |
| o1-high | no answer | no query generated | fail | 5 | 0.00s | 0.9261 |
| o1-low | credentials | let LinuxDevices = DeviceInfo
| where OSPlatform startswith "Linux"
| distinct DeviceId;
DeviceProcessEvents
| where DeviceId in (LinuxDevices)
| where ActionType == "ProcessCreated"
| where ProcessCommandLine has "find" and ProcessCommandLine has ".aws"
| project TimeGenerated,
DeviceName,
InitiatingProcessAccountName,
ProcessCommandLine,
AWSFilename = extract("-name\\s+['\"]([^'\"]+)", 1, ProcessCommandLine)
| where isnotempty(AWSFilename)
| pass | 3 | 77.32s | 0.4870 |
| o3-high | no answer | no query generated | fail | 5 | 0.00s | 0.0934 |
| o3-mini-high | no answer | no query generated | fail | 5 | 0.00s | 0.0453 |
| o3-mini-low | credentials | DeviceProcessEvents
| where InitiatingProcessCommandLine has "find" and InitiatingProcessCommandLine has ".aws"
| extend credential_filename = extract("-name\\s+['\\\"]?([^'\\\"\\s]+)", 1, InitiatingProcessCommandLine)
| where isnotempty(credential_filename)
| project TimeGenerated, DeviceName, InitiatingProcessCommandLine, credential_filename
| sort by TimeGenerated desc | pass | 3 | 28.92s | 0.0221 |
| o4-mini-high | no answer | no query generated | fail | 5 | 0.00s | 0.0342 |
| o4-mini-low | no answer | no query generated | fail | 5 | 0.00s | 0.0297 |