Question Analysis
On a Linux host, identify the process invocation that altered a user’s login shell. What was the full command used?
c7ac59cb-13cc-4622-81dc-6d2fee9bfac7
Question ID
16/23
Models Correct
1
Valid Answer
70%
Success Rate
Reference Answer
Human-validated correct answer for this question
chsh -s /bin/sh art
Model Performance Comparison
See how different AI models performed on this question
Model | Answer | KQL Query | Status | Attempts | Time | Cost |
---|---|---|---|---|---|---|
gemini-2.5-flash-preview-04-17 | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0373 |
gpt-35-turbo | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0110 |
gpt-4-turbo-2024-04-09 | chsh -s /bin/sh art | DeviceProcessEvents
| where InitiatingProcessFileName contains 'chsh' or ProcessCommandLine contains 'chsh'
| project TimeGenerated, DeviceName, InitiatingProcessAccountName, ProcessCommandLine, InitiatingProcessCommandLine
| distinct TimeGenerated, DeviceName, InitiatingProcessAccountName, ProcessCommandLine, InitiatingProcessCommandLine | Correct | 4 | 36.27s | 0.2200 |
gpt-4.1 | chsh -s /bin/sh art | DeviceProcessEvents
| where FileName in~ ("chsh", "usermod", "sed", "tee", "echo")
or (ProcessCommandLine has "/etc/passwd" and (ProcessCommandLine has ">" or ProcessCommandLine has "tee" or ProcessCommandLine has "sed" or ProcessCommandLine has "usermod" or ProcessCommandLine has "chsh" or ProcessCommandLine has "echo"))
| where (ProcessCommandLine has "-s" or ProcessCommandLine has "shell" or ProcessCommandLine has "/etc/passwd")
| project TimeGenerated, DeviceName, AccountName, InitiatingProcessAccountName, FileName, ProcessCommandLine
| order by TimeGenerated desc | Correct | 3 | 14.45s | 0.0332 |
gpt-4.1-finetuned | The process invocation that altered a user's login shell is: chsh -s /bin/sh art. The full command used was 'chsh -s /bin/sh art'. Additional invocations are present such as useradd with the --shell option, usermod for lock/unlock/expire date, and a combined bash command performing multiple modifications. | DeviceProcessEvents
| where ProcessCommandLine contains "chsh" or ProcessCommandLine contains "usermod"
| summarize by AccountName, DeviceName, ProcessCommandLine
| project AccountName, DeviceName, ProcessCommandLine, Explanation = strcat("Account ", AccountName, " had shell modified via: ", ProcessCommandLine, " on device: ", DeviceName)
| limit 100 | Correct | 2 | 23.28s | 0.0186 |
gpt-4.1-mini | chsh -s /bin/sh art | DeviceProcessEvents
| where DeviceName contains "linux" or InitiatingProcessCommandLine contains "chsh" or InitiatingProcessCommandLine contains "usermod"
| where ProcessCommandLine contains "/etc/passwd" or ProcessCommandLine contains "chsh" or ProcessCommandLine contains "usermod"
| project TimeGenerated, DeviceName, InitiatingProcessAccountName, ProcessCommandLine
| sort by TimeGenerated desc | Correct | 2 | 27.80s | 0.0061 |
gpt-4.1-nano | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0013 |
gpt-4o | bash -c '[ "$(uname)" = "FreeBSD" ] && pw useradd art -g wheel -s /bin/csh || useradd -s /bin/bash art; cat /etc/passwd |grep ^art; chsh -s /bin/sh art; cat /etc/passwd |grep ^art' | DeviceProcessEvents
| where isnotempty(ProcessCommandLine) and ProcessCommandLine contains "chsh"
| project TimeGenerated, DeviceName, AccountName, AccountDomain, ProcessCommandLine | Correct | 3 | 29.41s | 0.0536 |
gpt-5-high | chsh -s /bin/sh art | let LinuxDevices = DeviceInfo
| where OSPlatform =~ "Linux"
| project DeviceId, DeviceName;
let ShellChangeProcs = DeviceProcessEvents
| where FileName in~ ("usermod","chsh") or ProcessCommandLine has_any ("usermod -s","usermod --shell","chsh -s","chsh --shell")
| where ProcessCommandLine has_any (" -s "," --shell ")
| project TimeGenerated, DeviceId, Actor=AccountName, Process=FileName, FullCommand=ProcessCommandLine, Parent=InitiatingProcessParentFileName, ParentCommand=InitiatingProcessCommandLine;
let PasswdWrites = DeviceFileEvents
| where tolower(FolderPath) endswith "/etc/passwd"
| where ActionType in~ ("FileCreated","FileModified","FileRenamed","FileWritten","FileContentModified")
| project DeviceId, FileEventTime=TimeGenerated, FileAction=ActionType;
ShellChangeProcs
| join kind=inner LinuxDevices on DeviceId
| join kind=leftouter PasswdWrites on DeviceId
| where isnull(FileEventTime) or FileEventTime between (TimeGenerated - 10m .. TimeGenerated + 10m)
| summarize FileEventTime=min(FileEventTime), FileAction=any(FileAction) by TimeGenerated, DeviceName, Actor, Process, FullCommand, Parent
| order by TimeGenerated desc | Correct | 3 | 309.93s | 0.2009 |
gpt-5-mini-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0236 |
gpt-5-mini-low | chsh -s /bin/sh art | // Identify Linux devices
let linuxDevices = DeviceInfo
| where OSPlatform contains "Linux" or OSDistribution contains "Linux" or OSPlatform has "linux"
| distinct DeviceId, DeviceName;
// File events that touch typical files used to change shells
let fileMods = DeviceFileEvents
| where DeviceId in (linuxDevices | distinct DeviceId)
| where TimeGenerated >= ago(30d) // time range can be overwritten by external timerange
| where ActionType has_any ("FileModified", "FileCreated", "FileWrite", "FileSaved", "FileTruncated", "FileUpdated", "FileDeleted", "FileRenamed")
| where (tolower(FileName) in ("passwd","shadow","shells") ) or (FolderPath contains "/etc/")
| extend CandidateCommand = coalesce(InitiatingProcessCommandLine, "")
| project TimeGenerated, DeviceId, DeviceName, SourceTable = "DeviceFileEvents", ActionType, FileName, FolderPath, InitiatingProcessFileName, InitiatingProcessId, CandidateCommand, InitiatingProcessAccountName;
// Process events that likely change shells
let processMods = DeviceProcessEvents
| where DeviceId in (linuxDevices | distinct DeviceId)
| where TimeGenerated >= ago(30d)
| extend pc = coalesce(ProcessCommandLine, ""), ipcc = coalesce(InitiatingProcessCommandLine, "")
| where pc has_any ("chsh", "usermod", "-s", "/etc/passwd", "/etc/shadow", "sed -i", "awk -F", "chpasswd", "printf ", ">> /etc/passwd", "> /etc/passwd")
| extend CandidateCommand = iif(isnotempty(ipcc), ipcc, pc)
| project TimeGenerated=ProcessCreationTime, DeviceId, DeviceName, SourceTable = "DeviceProcessEvents", ProcessId, CandidateCommand, ProcessCommandLine=pc, InitiatingProcessFileName, InitiatingProcessId, InitiatingProcessAccountName;
// Generic events table fallback
let genericEvents = DeviceEvents
| where DeviceId in (linuxDevices | distinct DeviceId)
| where TimeGenerated >= ago(30d)
| extend ce = coalesce(ProcessCommandLine, InitiatingProcessCommandLine, "")
| where ce has_any ("chsh", "usermod", "/etc/passwd", "/etc/shadow", "/etc/shells", "sed -i", "awk -F", "chpasswd", "printf ")
| project TimeGenerated, DeviceId, DeviceName, SourceTable = "DeviceEvents", ActionType, CandidateCommand = ce, InitiatingProcessFileName, InitiatingProcessId, InitiatingProcessAccountName;
// Union and prioritize explicit initiator command line
(fileMods
| union processMods
| union genericEvents)
| extend FullCommand = trim('\"''', CandidateCommand)
| where isnotempty(FullCommand)
| where FullCommand has_any ("chsh", "usermod", "/etc/passwd", "/etc/shadow", "/etc/shells", "sed -i", "awk -F", "chpasswd", "printf", ">> /etc/passwd", "> /etc/passwd")
| summarize arg_max(TimeGenerated, *) by tostring(DeviceId), tostring(FullCommand)
| project TimeGenerated, DeviceName, DeviceId, InitiatingProcessFileName, InitiatingProcessId, InitiatingProcessAccountName, FullCommand, SourceTable
| order by TimeGenerated desc
| take 50
| Correct | 2 | 22.28s | 0.0079 |
gpt-5-mini-medium | chsh -s /bin/sh art | let passwd_file_events =
DeviceFileEvents
| where FolderPath has "/etc" and (FileName == "passwd" or FileName contains "passwd")
| where tolower(ActionType) has_any ("write","modify","modified","create","created","rename","delete")
| project FileEventTime=TimeGenerated, DeviceName, DeviceId, ReportId, FileName, FolderPath, ActionType, InitiatingProcessId, InitiatingProcessFileName, InitiatingProcessCommandLine, InitiatingProcessAccountName;
let processes_with_shell_change_indicators =
DeviceProcessEvents
| where tolower(ProcessCommandLine) has_any ("chsh","chsh -s","usermod -s","vipw")
| project ProcessTime=TimeGenerated, DeviceName, DeviceId, ProcessId, ProcessCommandLine, AccountName = AccountName, ProcessFileName = FileName;
passwd_file_events
| join kind=leftouter (
DeviceProcessEvents
| project ProcessTime=TimeGenerated, DeviceId, ProcessId, ProcessCommandLine, AccountName = AccountName, ProcessFileName = FileName
) on DeviceId, $left.InitiatingProcessId == $right.ProcessId
| extend FullCommand = coalesce(InitiatingProcessCommandLine, ProcessCommandLine)
| project EventTime = FileEventTime, DeviceName, DeviceId, Account = InitiatingProcessAccountName, InitiatingProcessFileName, InitiatingProcessId, ProcessId, FullCommand, FilePath = strcat(FolderPath, "/", FileName), ActionType
| union (
processes_with_shell_change_indicators
| extend EventTime = ProcessTime, FullCommand = ProcessCommandLine, FilePath = "", ActionType = "ProcessIndicatorMatched"
| project EventTime, DeviceName, DeviceId, Account = AccountName, InitiatingProcessFileName = ProcessFileName, InitiatingProcessId = ProcessId, ProcessId, FullCommand, FilePath, ActionType
)
| sort by EventTime desc
| summarize arg_max(EventTime, *) by tostring(DeviceId), tostring(FullCommand), tostring(InitiatingProcessId)
| project EventTime, DeviceName, DeviceId, Account, InitiatingProcessFileName, InitiatingProcessId, ProcessId, FullCommand, FilePath, ActionType
| limit 100 | Correct | 1 | 46.24s | 0.0096 |
gpt-5-nano-high | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0102 |
gpt-5-nano-medium | usermod --expiredate 99999 art | let linuxDevices = DeviceInfo
| where OSPlatform contains "Linux"
| project DeviceId;
DeviceProcessEvents
| where DeviceId in (linuxDevices)
| where InitiatingProcessFileName in ("chsh","usermod") or InitiatingProcessCommandLine contains " chsh" or InitiatingProcessCommandLine contains "usermod"
| extend TargetUser = tostring(extract(@"(?:^|\s)([A-Za-z_][A-Za-z0-9_\-]+)$", 1, InitiatingProcessCommandLine))
| extend TargetShell = tostring(extract(@"-s\s+([^\s]+)", 1, InitiatingProcessCommandLine))
| where isnotempty(TargetUser) or isnotempty(TargetShell)
| sort by TimeGenerated desc
| summarize arg_max(TimeGenerated, InitiatingProcessCommandLine, InitiatingProcessFileName, InitiatingProcessId, TargetUser, TargetShell) by DeviceName
| project TimeGenerated, DeviceName, InitiatingProcessFileName, InitiatingProcessCommandLine, TargetUser, TargetShell | Incorrect | 3 | 97.68s | 0.0062 |
grok-3-beta | chsh -s /bin/sh art | let shellChangeCommands = dynamic(['chsh', 'usermod']);
DeviceFileEvents
| where FolderPath contains '/etc/passwd' and ActionType == 'FileModified'
| project TimeGenerated, DeviceName, InitiatingProcessFileName, InitiatingProcessCommandLine, ActionType, FileName, FolderPath
| union (
DeviceProcessEvents
| where ProcessCommandLine has_any (shellChangeCommands)
| project TimeGenerated, DeviceName, InitiatingProcessFileName, InitiatingProcessCommandLine, ActionType='ProcessExecuted', FileName='', FolderPath=''
)
| order by TimeGenerated desc | Correct | 1 | 10.83s | 0.0346 |
grok-3-mini-beta | chsh -s /bin/sh art | let linuxDevices = DeviceInfo | where OSPlatform == 'Linux' | distinct DeviceId;
DeviceProcessEvents
| where DeviceId in (linuxDevices)
| where ProcessCommandLine contains "usermod -s" or ProcessCommandLine contains "chsh"
| project Timestamp, DeviceName, AccountName, ProcessCommandLine
| order by Timestamp desc
| limit 100 // Limit results for practicality; adjust as needed | Correct | 1 | 17.10s | 0.0021 |
o1-high | chsh -s /bin/sh art | DeviceProcessEvents
| where DeviceId in (DeviceInfo | where OSPlatform == "Linux" | distinct DeviceId)
| where ProcessCommandLine has_any ("chsh","usermod -s")
| project TimeGenerated, DeviceName, AccountDomain, AccountName, ProcessCommandLine, InitiatingProcessCommandLine | Correct | 1 | 40.01s | 0.3123 |
o1-low | chsh -s /bin/sh art | DeviceProcessEvents
| where DeviceId in (DeviceInfo
| where OSPlatform =~ "Linux"
| distinct DeviceId)
| where FileName in~ ("chsh","usermod") or ProcessCommandLine has_any ("chsh", "usermod -s", "usermod --shell")
| project TimeGenerated, DeviceName, AccountName, ProcessCommandLine
| order by TimeGenerated desc | Correct | 1 | 40.33s | 0.3277 |
o3-high | chsh -s /bin/sh art | // --- Linux login-shell change detection -----------------------------------
// Time filter is injected automatically by the hunting portal
// but can be added manually if needed, e.g. | where TimeGenerated between (datetime(2023-09-14) .. datetime(2023-09-21))
// 1. Track any edit to /etc/passwd (the file that stores the user shell)
let PasswdEdits =
DeviceFileEvents
| where ActionType in ("FileModified","FileCreated","FileWritten","FileDeleted","Modified")
| where FolderPath == "/etc" and FileName == "passwd"
| project DeviceId, ReportId, PasswdEditTime = TimeGenerated;
// 2. Find processes that are normally used to change a login shell
let ShellChangers =
DeviceProcessEvents
| where ActionType == "ProcessCreated"
// Defender for Endpoint marks Linux paths with a leading '/'; good enough indication of OS
| where FolderPath startswith "/" // keep only Linux processes
| where FileName in ("chsh","usermod")
| where ProcessCommandLine has_any (" -s ", " --shell") // command must include the shell parameter
| extend PotentialShellChange = 1;
// 3. Merge both data-sets to obtain a consolidated view
ShellChangers
| join kind=leftouter PasswdEdits on DeviceId, ReportId
| project TimeGenerated, DeviceName, AccountName, FileName, ProcessCommandLine,
InitiatingProcessAccountName, InitiatingProcessCommandLine,
HasEditedPasswd = iff(isnotempty(PasswdEditTime), true, false)
| order by TimeGenerated desc | Correct | 1 | 53.20s | 0.0339 |
o3-mini-high | chsh -s /bin/sh art | DeviceProcessEvents
| where ProcessCommandLine has "chsh" or ProcessCommandLine has "usermod -s"
| project TimeGenerated, DeviceName, ProcessCommandLine
| sort by TimeGenerated desc | Correct | 1 | 17.94s | 0.0138 |
o3-mini-low | chsh -s /bin/sh art | DeviceProcessEvents
| where isnotempty(InitiatingProcessCommandLine)
| where InitiatingProcessCommandLine has_any ("chsh", "usermod") and InitiatingProcessCommandLine has "-s"
| project TimeGenerated, DeviceName, InitiatingProcessCommandLine
| order by TimeGenerated desc | Correct | 2 | 92.02s | 0.0309 |
o4-mini-high | chsh -s /bin/sh art | DeviceProcessEvents
// limit to typical shell‐change commands
| where ProcessCommandLine has "chsh -s" or ProcessCommandLine has "usermod -s"
| project TimeGenerated, DeviceName, AccountName = InitiatingProcessAccountName, ChangedByProcess = FileName, FullCommandLine = ProcessCommandLine
| sort by TimeGenerated desc | Correct | 2 | 51.81s | 0.0229 |
o4-mini-low | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0366 |