Question Analysis
On a Linux host, identify the process invocation that altered a user’s login shell. What was the full command used?
c7ac59cb-13cc-4622-81dc-6d2fee9bfac7
Question ID
12/16
Models Correct
1
Valid Answer
75%
Success Rate
Reference Answer
Human-validated correct answer for this question
chsh -s /bin/sh art
Model Performance Comparison
See how different AI models performed on this question
| Model | Answer | KQL Query | Status | Attempts | Time | Cost |
|---|---|---|---|---|---|---|
| gemini-2.5-flash-preview-04-17 | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0373 |
| gpt-35-turbo | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0110 |
| gpt-4-turbo-2024-04-09 | chsh -s /bin/sh art | DeviceProcessEvents
| where InitiatingProcessFileName contains 'chsh' or ProcessCommandLine contains 'chsh'
| project TimeGenerated, DeviceName, InitiatingProcessAccountName, ProcessCommandLine, InitiatingProcessCommandLine
| distinct TimeGenerated, DeviceName, InitiatingProcessAccountName, ProcessCommandLine, InitiatingProcessCommandLine | Correct | 4 | 36.27s | 0.2200 |
| gpt-4.1 | chsh -s /bin/sh art | DeviceProcessEvents
| where FileName in~ ("chsh", "usermod", "sed", "tee", "echo")
or (ProcessCommandLine has "/etc/passwd" and (ProcessCommandLine has ">" or ProcessCommandLine has "tee" or ProcessCommandLine has "sed" or ProcessCommandLine has "usermod" or ProcessCommandLine has "chsh" or ProcessCommandLine has "echo"))
| where (ProcessCommandLine has "-s" or ProcessCommandLine has "shell" or ProcessCommandLine has "/etc/passwd")
| project TimeGenerated, DeviceName, AccountName, InitiatingProcessAccountName, FileName, ProcessCommandLine
| order by TimeGenerated desc | Correct | 3 | 14.45s | 0.0332 |
| gpt-4.1-finetuned | The process invocation that altered a user's login shell is: chsh -s /bin/sh art. The full command used was 'chsh -s /bin/sh art'. Additional invocations are present such as useradd with the --shell option, usermod for lock/unlock/expire date, and a combined bash command performing multiple modifications. | DeviceProcessEvents
| where ProcessCommandLine contains "chsh" or ProcessCommandLine contains "usermod"
| summarize by AccountName, DeviceName, ProcessCommandLine
| project AccountName, DeviceName, ProcessCommandLine, Explanation = strcat("Account ", AccountName, " had shell modified via: ", ProcessCommandLine, " on device: ", DeviceName)
| limit 100 | Correct | 2 | 23.28s | 0.0186 |
| gpt-4.1-mini | chsh -s /bin/sh art | DeviceProcessEvents
| where DeviceName contains "linux" or InitiatingProcessCommandLine contains "chsh" or InitiatingProcessCommandLine contains "usermod"
| where ProcessCommandLine contains "/etc/passwd" or ProcessCommandLine contains "chsh" or ProcessCommandLine contains "usermod"
| project TimeGenerated, DeviceName, InitiatingProcessAccountName, ProcessCommandLine
| sort by TimeGenerated desc | Correct | 2 | 27.80s | 0.0061 |
| gpt-4.1-nano | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0013 |
| gpt-4o | bash -c '[ "$(uname)" = "FreeBSD" ] && pw useradd art -g wheel -s /bin/csh || useradd -s /bin/bash art; cat /etc/passwd |grep ^art; chsh -s /bin/sh art; cat /etc/passwd |grep ^art' | DeviceProcessEvents
| where isnotempty(ProcessCommandLine) and ProcessCommandLine contains "chsh"
| project TimeGenerated, DeviceName, AccountName, AccountDomain, ProcessCommandLine | Correct | 3 | 29.41s | 0.0536 |
| grok-3-beta | chsh -s /bin/sh art | let shellChangeCommands = dynamic(['chsh', 'usermod']);
DeviceFileEvents
| where FolderPath contains '/etc/passwd' and ActionType == 'FileModified'
| project TimeGenerated, DeviceName, InitiatingProcessFileName, InitiatingProcessCommandLine, ActionType, FileName, FolderPath
| union (
DeviceProcessEvents
| where ProcessCommandLine has_any (shellChangeCommands)
| project TimeGenerated, DeviceName, InitiatingProcessFileName, InitiatingProcessCommandLine, ActionType='ProcessExecuted', FileName='', FolderPath=''
)
| order by TimeGenerated desc | Correct | 1 | 10.83s | 0.0346 |
| grok-3-mini-beta | chsh -s /bin/sh art | let linuxDevices = DeviceInfo | where OSPlatform == 'Linux' | distinct DeviceId;
DeviceProcessEvents
| where DeviceId in (linuxDevices)
| where ProcessCommandLine contains "usermod -s" or ProcessCommandLine contains "chsh"
| project Timestamp, DeviceName, AccountName, ProcessCommandLine
| order by Timestamp desc
| limit 100 // Limit results for practicality; adjust as needed | Correct | 1 | 17.10s | 0.0021 |
| o1-high | chsh -s /bin/sh art | DeviceProcessEvents
| where DeviceId in (DeviceInfo | where OSPlatform == "Linux" | distinct DeviceId)
| where ProcessCommandLine has_any ("chsh","usermod -s")
| project TimeGenerated, DeviceName, AccountDomain, AccountName, ProcessCommandLine, InitiatingProcessCommandLine | Correct | 1 | 40.01s | 0.3123 |
| o1-low | chsh -s /bin/sh art | DeviceProcessEvents
| where DeviceId in (DeviceInfo
| where OSPlatform =~ "Linux"
| distinct DeviceId)
| where FileName in~ ("chsh","usermod") or ProcessCommandLine has_any ("chsh", "usermod -s", "usermod --shell")
| project TimeGenerated, DeviceName, AccountName, ProcessCommandLine
| order by TimeGenerated desc | Correct | 1 | 40.33s | 0.3277 |
| o3-mini-high | chsh -s /bin/sh art | DeviceProcessEvents
| where ProcessCommandLine has "chsh" or ProcessCommandLine has "usermod -s"
| project TimeGenerated, DeviceName, ProcessCommandLine
| sort by TimeGenerated desc | Correct | 1 | 17.94s | 0.0138 |
| o3-mini-low | chsh -s /bin/sh art | DeviceProcessEvents
| where isnotempty(InitiatingProcessCommandLine)
| where InitiatingProcessCommandLine has_any ("chsh", "usermod") and InitiatingProcessCommandLine has "-s"
| project TimeGenerated, DeviceName, InitiatingProcessCommandLine
| order by TimeGenerated desc | Correct | 2 | 92.02s | 0.0309 |
| o4-mini-high | chsh -s /bin/sh art | DeviceProcessEvents
// limit to typical shell‐change commands
| where ProcessCommandLine has "chsh -s" or ProcessCommandLine has "usermod -s"
| project TimeGenerated, DeviceName, AccountName = InitiatingProcessAccountName, ChangedByProcess = FileName, FullCommandLine = ProcessCommandLine
| sort by TimeGenerated desc | Correct | 2 | 51.81s | 0.0229 |
| o4-mini-low | No answer provided | No query generated | Incorrect | 5 | 0.00s | 0.0366 |