search

sonic-net / sonic-buildimage

Scripts which perform an installable binary image build for SONiC
Other
746 stars 1.44k forks source link

[TACACS+] | Unexpected output to command "sh" by remote SSH client #14404

Open nazariig opened 1 year ago

nazariig commented 1 year ago

Description

TACACS returned unexpected result:

Expected: 'authorize failed by TACACS+ with given arguments, not executing'  
Result:   '/usr/bin/dash not authorized by TACACS+ with given arguments, not executing'

Plugin failed to connect to TACACS server:

Mar 20 12:30:30.094668 r-leopard-58 ERR bash: TACACS+: Failed to determine tty, passing UNK
Mar 20 12:30:31.095774 r-leopard-58 ERR bash: TACACS+: Failed to connecting to 10.213.103.5:49 to request authorization for /usr/bin/dash: Operation now in progress
Mar 20 12:30:31.096335 r-leopard-58 ERR bash: TACACS+: Failed to connect to TACACS server(s)

TACACS+ plugin:

https://github.com/sonic-net/sonic-buildimage/blob/202205/src/tacacs/bash_tacplus/bash_tacplus.c#L422

/*
 * Tacacs authorization.
 */
int on_shell_execve (char *user, int shell_level, char *cmd, char **argv)

...

    if (tacacs_ctrl & AUTHORIZATION_FLAG_TACACS) {
        output_debug("start TACACS+ authorization for command %s with given arguments\n", cmd);
        int ret = authorization_with_host_and_tty(user_namd, cmd, argv, argc);
        switch (ret) {
            case 0:
            break;
            case -2:
                // -2 means no servers, so not authorized
                fprintf(stdout, "%s not authorized by TACACS+ with given arguments, not executing\n", cmd);
            break;
            default:
                fprintf(stdout, "%s authorize failed by TACACS+ with given arguments, not executing\n", cmd);
            break;
        }

        if ((tacacs_ctrl & AUTHORIZATION_FLAG_LOCAL) == 0) {
            // when local authorization disabled, tacacs authorization failed will block user from run current command
            output_debug("local authorization disabled, TACACS+ authorization result: %d\n", ret);
            return ret;
        }
    }

TACACS config:

sonic-mgmt\tests\tacacs\tac_plus.conf.j2

user = {{ tacacs_authorization_user }} {
    login = des {{ tacacs_authorization_user_passwd }}
    member = netuser
    cmd = /usr/bin/cat {
        deny .*
    }
    # disable following command for UT test_bypass_authorization
    cmd = /usr/local/bin/config {
        deny   tacacs
        permit .*
    }
    cmd = /usr/bin/python {
        deny .*
    }
    cmd = /usr/bin/find {
        deny   -exec
        permit .*
    }
    cmd = /lib/arch-linux-abi/ld-linux-arch.so {
        deny .*
    }
    cmd = /usr/bin/dash {
        deny .*
    }
}

Testcase:

def test_bypass_authorization(
        duthosts, enum_rand_one_per_hwsku_hostname,
        setup_authorization_tacacs, check_tacacs, remote_user_client):
    duthost = duthosts[enum_rand_one_per_hwsku_hostname]

...

    exit_code, stdout, stderr = ssh_run_command(remote_user_client, '"sh"')
    pytest_assert(exit_code == 1)
    check_ssh_output(stdout, 'authorize failed by TACACS+ with given arguments, not executing')

DASH:

root@r-leopard-58:/home/admin# which sh
/usr/bin/sh

root@r-leopard-58:/home/admin# stat /usr/bin/sh
  File: /usr/bin/sh -> dash
  Size: 4               Blocks: 0          IO Block: 1024   symbolic link
Device: 1ah/26d Inode: 8078        Links: 1
Access: (0777/lrwxrwxrwx)  Uid: (    0/    root)   Gid: (    0/    root)
Access: 2023-03-23 07:28:14.000000000 +0000
Modify: 2023-03-23 07:28:14.000000000 +0000
Change: 2023-03-23 07:28:14.000000000 +0000
 Birth: -

root@r-leopard-58:/home/admin# which dash
/usr/bin/dash

Log:

=================================== FAILURES =================================== 
___________________ test_bypass_authorization[r-leopard-58] ____________________ 
duthosts = [<MultiAsicSonicHost r-leopard-58>] 
enum_rand_one_per_hwsku_hostname = 'r-leopard-58' 
tacacs_creds = {'local_user': 'test_louser', 'local_user_passwd': '123456', 'r-leopard-58': {'ansible_become_pass': 'YourPaSsWoRd', '...ble_ssh_user': 'user', 'bgp_slb_passive_range': '10.255.0.0/25', ...}, 'tacacs_authorization_user': 'test_auuser', ...} 
check_tacacs = None 
remote_user_client = <paramiko.client.SSHClient object at 0x7f055aa275d0> 
    def test_bypass_authorization(duthosts, enum_rand_one_per_hwsku_hostname, tacacs_creds, check_tacacs, remote_user_client): 
        duthost = duthosts[enum_rand_one_per_hwsku_hostname] 
        duthost.shell("sudo config aaa authorization tacacs+") 
        """ 
            Verify user can't run script with sh/python with following command. 
                python ./testscript.py 
            NOTE: TACACS UT using tac_plus as server side, there is a bug that tac_plus can't handle an authorization message contains more than 10 attributes. 
                  Because every command parameter will convert to a TACACS attribute, please don't using more than 5 command parameters in test case. 
        """ 
        exit_code, stdout, stderr = ssh_run_command(remote_user_client, 'echo "" >> ./testscript.py') 
        pytest_assert(exit_code == 0) 
        exit_code, stdout, stderr = ssh_run_command(remote_user_client, "python ./testscript.py") 
        pytest_assert(exit_code == 1) 
        check_ssh_output(stdout, 'authorize failed by TACACS+ with given arguments, not executing') 
        # Verify user can't run 'find' command with '-exec' parameter. 
        exit_code, stdout, stderr = ssh_run_command(remote_user_client, "find . -exec") 
        pytest_assert(exit_code == 1) 
        exp_outputs = ['not authorized by TACACS+ with given arguments, not executing', 
                       'authorize failed by TACACS+ with given arguments, not executing'] 
        check_ssh_output_any_of(stdout, exp_outputs) 
        # Verify user can run 'find' command without '-exec' parameter. 
        exit_code, stdout, stderr = ssh_run_command(remote_user_client, "find . /bin/sh") 
        pytest_assert(exit_code == 0) 
        check_ssh_output(stdout, '/bin/sh') 
        """ 
            Verify user can't run command with loader: 
                /lib/x86_64-linux-gnu/ld-linux-x86-64.so.2 sh 
        """ 
        ld_path = get_ld_path(duthost) 
        if not ld_path: 
            exit_code, stdout, stderr = ssh_run_command(remote_user_client, ld_path + " sh") 
            pytest_assert(exit_code == 1) 
            check_ssh_output(stdout, 'authorize failed by TACACS+ with given arguments, not executing') 
        """ 
            Verify user can't run command with prefix/quoting: 
                \sh 
                "sh" 
                echo $(sh -c ls) 
        """ 
        exit_code, stdout, stderr = ssh_run_command(remote_user_client, "\\sh") 
        pytest_assert(exit_code == 1) 
        check_ssh_output(stdout, 'authorize failed by TACACS+ with given arguments, not executing') 
        exit_code, stdout, stderr = ssh_run_command(remote_user_client, '"sh"') 
        pytest_assert(exit_code == 1) 
>       check_ssh_output(stdout, 'authorize failed by TACACS+ with given arguments, not executing') 
check_tacacs = None 
duthost    = <MultiAsicSonicHost r-leopard-58> 
duthosts   = [<MultiAsicSonicHost r-leopard-58>] 
enum_rand_one_per_hwsku_hostname = 'r-leopard-58' 
exit_code  = 1 
exp_outputs = ['not authorized by TACACS+ with given arguments, not executing', 'authorize failed by TACACS+ with given arguments, not executing'] 
ld_path    = u'/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2' 
remote_user_client = <paramiko.client.SSHClient object at 0x7f055aa275d0> 
stderr     = [] 
stdout     = ['/usr/bin/dash not authorized by TACACS+ with given arguments, not executing 
'] 
tacacs_creds = {'local_user': 'test_louser', 'local_user_passwd': '123456', 'r-leopard-58': {'ansible_become_pass': 'YourPaSsWoRd', '...ble_ssh_user': 'user', 'bgp_slb_passive_range': '10.255.0.0/25', ...}, 'tacacs_authorization_user': 'test_auuser', ...} 
tacacs/test_authorization.py:326: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
res = ['/usr/bin/dash not authorized by TACACS+ with given arguments, not executing 
'] 
exp_val = 'authorize failed by TACACS+ with given arguments, not executing' 
    def check_ssh_output(res, exp_val): 
        content_exist = False 
        for l in res: 
            if exp_val in l: 
                content_exist = True 
                break 
>       pytest_assert(content_exist) 
E       Failed: <Failed instance> 
content_exist = False 
exp_val    = 'authorize failed by TACACS+ with given arguments, not executing' 
l          = '/usr/bin/dash not authorized by TACACS+ with given arguments, not executing 
' 
res        = ['/usr/bin/dash not authorized by TACACS+ with given arguments, not executing 
'] 
tacacs/test_authorization.py:47: Failed 
- generated xml file: /root/mars/workspace/sonic-mgmt/tests/junit_7266824_0.7.1.1.32.1.3.2.1.1.1.xml - 
=========================== short test summary info ============================ 
FAILED tacacs/test_authorization.py::test_bypass_authorization[r-leopard-58] 
===================== 1 failed, 7 passed in 207.73 seconds =====================

Steps to reproduce the issue:

  1. Run script 
    
#!/bin/bash

SCRIPT=$0 FULL_PATH=$(realpath ${SCRIPT}) SCRIPT_PATH=$(dirname ${FULL_PATH}) BASE_PATH=$(dirname ${SCRIPT_PATH}) LOG_PATH="logs"

export ANSIBLE_CONFIG=${BASE_PATH}/ansible export ANSIBLE_LIBRARY=${BASE_PATH}/ansible/library/ export ANSIBLE_CONNECTION_PLUGINS=${BASE_PATH}/ansible/plugins/connection export ANSIBLE_CLICONF_PLUGINS=${BASE_PATH}/ansible/cliconf_plugins export ANSIBLE_TERMINAL_PLUGINS=${BASE_PATH}/ansible/terminal_plugins

Kill pytest and ansible-playbook process

pkill --signal 9 pytest pkill --signal 9 ansible-playbook

Kill ssh initiated by ansible, try to match full command begins with 'ssh' and contains path '/.ansible'

pkill --signal 9 -f "^ssh.*/.ansible"

rm -fr ${BASE_PATH}/tests/_cache

#

Test

#

TACACS --------------------------------------------------------------------------------------------------------------

HOST="r-leopard-58" TOPO="t1-lag"

py.test tacacs/test_authorization.py \ --inventory="../ansible/inventory,../ansible/veos" --host-pattern ${HOST} --module-path ../ansible/library/ \ --testbed ${HOST}-${TOPO} --testbed_file ../ansible/testbed.csv --allow_recover \ --assert plain --log-cli-level info --show-capture=stdout -ra --showlocals -v \ --clean-alluredir --alluredir=/tmp/allure-results --allure_server_addr="10.215.11.120" \ --skip_sanity --dynamic_update_skip_reason


**NOTE:** the reproduction probability is very low

#### Describe the results you received:

/usr/bin/dash not authorized by TACACS+ with given arguments, not executing


#### Describe the results you expected:

authorize failed by TACACS+ with given arguments, not executing



#### Output of `show version`:
* N/A

#### Output of `show techsupport`:
* N/A

#### Additional information you deem important (e.g. issue happens only occasionally):
<!--
     Also attach debug file produced by `sudo generate_dump`
-->
* N/A
liuh-80 commented 11 months ago

@nazariig , please check if the TACACS server correctly configured on 10.213.103.5:

Mar 20 12:30:31.095774 r-leopard-58 ERR bash: TACACS+: Failed to connecting to 10.213.103.5:49 to request authorization for /usr/bin/dash: Operation now in progress

the TACACS server been installed by following code: https://github.com/sonic-net/sonic-mgmt/blob/master/spytest/utilities/services.py

RUN apt update && apt install tacacs+ -y RUN service tacacs_plus start

Please check if the code running correctly on your test environment.