search

pry0cc / axiom

The dynamic infrastructure framework for everybody! Distribute the workload of many different scanning tools with ease, including nmap, ffuf, masscan, nuclei, meg and many more!
MIT License
4.06k stars 645 forks source link

axiom-exec not working #603

Closed chevyphillip closed 2 years ago

chevyphillip commented 2 years ago

Instances have been selected. 

> axiom-exec whoami
/Users/chevy/.axiom/interact/axiom-exec: line 357: -tL: command not found
cat: /Users/chevy/.axiom/tmp/exec/axiom-exec+1661976916/hosts_preflight: No such file or directory
cp: cannot stat '/Users/chevy/.axiom/tmp/exec/axiom-exec+1661976916/hosts_preflight': No such file or directory
/Users/chevy/.axiom/interact/axiom-exec: line 368: --silent: command not found
tail: cannot open '/Users/chevy/.axiom/tmp/exec/axiom-exec+1661976916/logs/*' for reading: No such file or directory
tail: no files remaining
/Users/chevy/.axiom/interact/axiom-exec: line 425: --silent: command not found
/Users/chevy/.axiom/interact/axiom-exec: line 430: -tL: command not found
0xtavian commented 2 years ago

@chevonmdphillip can you run with —debug and post the output?

chevyphillip commented 2 years ago

Sure!

> axiom-exec id --debug

###########################################################################################################
# clean_up
#
clean_up() {
kill -0 $remotetailPID 2>  /dev/null && kill -9 $remotetailPID  &> /dev/null
kill -0 $tailPID 2>  /dev/null && kill -9 $tailPID  &> /dev/null
kill -0 $downloaderPID 2>  /dev/null && kill -9 $downloaderPID  &> /dev/null
echo -e "${Blue}Killing remote processes in a backgroud job${Color_Off}"
$interlace_cmd_nobar -c "$ssh_command _target_ 'tmux kill-session -t $uid'"  >/dev/null 2>&1
$interlace_cmd_nobar -c "$ssh_exit_command _target_ " >/dev/null 2>&1
mv "$tmp" "$logs">/dev/null 2>&1
if [[ $keep_logs != true ]]; then
 rm -r $logs >/dev/null 2>&1
fi
stty sane
tput init
exit
}

###########################################################################################################
# Display Help Menu
#
if [[ "$*" == "--help" ]] || [[ "$*" == "-h" ]] || [[ "$*" == "" ]]; then
 usage
 exit
fi
+ [[ id --debug == \-\-\h\e\l\p ]]
+ [[ id --debug == \-\h ]]
+ [[ id --debug == '' ]]

###########################################################################################################
# SSH Cache Flag
#
if [[ "$cache" == "false" ]]; then
 generate_sshconfig
fi
+ [[ false == \f\a\l\s\e ]]
+ generate_sshconfig
ls -l "$AXIOM_PATH/accounts/" | grep "json" | grep -v 'total ' | awk '{ print $9 }' | sed 's/\.json//g'
++ ls -l /Users/chevy/.axiom/accounts/
++ grep json
++ grep -v 'total '
++ awk '{ print $9 }'
++ sed 's/\.json//g'
+ accounts='do.example
main-axiom
main'
ls -lh ~/.axiom/axiom.json | awk '{ print $11 }' | tr '/' '\n' | grep json | sed 's/\.json//g'
++ ls -lh /Users/chevy/.axiom/axiom.json
++ awk '{ print $11 }'
++ tr / '\n'
++ grep json
++ sed 's/\.json//g'
+ current=main-axiom
+ sshnew=/Users/chevy/.axiom/.sshconfig.new30109
instances
++ instances
++ doctl compute droplet list -o json
+ droplets='[
  {
    "id": 314838202,
    "name": "stone16",
    "memory": 1024,
    "vcpus": 1,
    "disk": 25,
    "region": {
      "slug": "tor1",
      "name": "Toronto 1",
      "sizes": [
        "s-1vcpu-1gb",
        "s-1vcpu-1gb-amd",
        "s-1vcpu-1gb-intel",
        "s-1vcpu-2gb",
        "s-1vcpu-2gb-amd",
        "s-1vcpu-2gb-intel",
        "s-2vcpu-2gb",
        "s-2vcpu-2gb-amd",
        "s-2vcpu-2gb-intel",
        "s-2vcpu-4gb",
        "s-2vcpu-4gb-amd",
        "s-2vcpu-4gb-intel",
        "c-2",
        "c2-2vcpu-4gb",
        "s-4vcpu-8gb",
        "s-4vcpu-8gb-amd",
        "s-4vcpu-8gb-intel",
        "g-2vcpu-8gb",
        "gd-2vcpu-8gb",
        "m-2vcpu-16gb",
        "c-4",
        "c2-4vcpu-8gb",
        "s-8vcpu-16gb",
        "m3-2vcpu-16gb",
        "s-8vcpu-16gb-amd",
        "s-8vcpu-16gb-intel",
        "g-4vcpu-16gb",
        "so-2vcpu-16gb",
        "m6-2vcpu-16gb",
        "gd-4vcpu-16gb",
        "so1_5-2vcpu-16gb",
        "m-4vcpu-32gb",
        "c-8",
        "c2-8vcpu-16gb",
        "m3-4vcpu-32gb",
        "g-8vcpu-32gb",
        "so-4vcpu-32gb",
        "m6-4vcpu-32gb",
        "gd-8vcpu-32gb",
        "so1_5-4vcpu-32gb",
        "m-8vcpu-64gb",
        "c-16",
        "c2-16vcpu-32gb",
        "m3-8vcpu-64gb",
        "g-16vcpu-64gb",
        "so-8vcpu-64gb",
        "m6-8vcpu-64gb",
        "gd-16vcpu-64gb",
        "so1_5-8vcpu-64gb",
        "m-16vcpu-128gb",
        "c-32",
        "c2-32vcpu-64gb",
        "m3-16vcpu-128gb",
        "c-48",
        "m-24vcpu-192gb",
        "g-32vcpu-128gb",
        "so-16vcpu-128gb",
        "m6-16vcpu-128gb",
        "gd-32vcpu-128gb",
        "c2-48vcpu-96gb",
        "m3-24vcpu-192gb",
        "g-40vcpu-160gb",
        "so1_5-16vcpu-128gb",
        "m-32vcpu-256gb",
        "gd-40vcpu-160gb",
        "so-24vcpu-192gb",
        "m6-24vcpu-192gb",
        "m3-32vcpu-256gb",
        "so1_5-24vcpu-192gb",
        "so-32vcpu-256gb",
        "m6-32vcpu-256gb",
        "so1_5-32vcpu-256gb"
      ],
      "available": true,
      "features": [
        "backups",
        "ipv6",
        "metadata",
        "install_agent",
        "storage",
        "image_transfer"
      ]
    },
    "image": {
      "id": 115944010,
      "name": "axiom-default-1661951687",
      "type": "snapshot",
      "distribution": "Ubuntu",
      "regions": [
        "tor1"
      ],
      "min_disk_size": 25,
      "size_gigabytes": 12.61,
      "created_at": "2022-08-31T13:56:16Z",
      "status": "available"
    },
    "size": {
      "slug": "s-1vcpu-1gb",
      "memory": 1024,
      "vcpus": 1,
      "disk": 25,
      "price_monthly": 6,
      "price_hourly": 0.00893,
      "regions": [
        "ams3",
        "blr1",
        "fra1",
        "lon1",
        "nyc1",
        "nyc3",
        "sfo3",
        "sgp1",
        "tor1"
      ],
      "available": true,
      "transfer": 1,
      "description": "Basic"
    },
    "size_slug": "s-1vcpu-1gb",
    "features": [
      "droplet_agent",
      "ipv6",
      "private_networking"
    ],
    "status": "active",
    "networks": {
      "v4": [
        {
          "ip_address": "147.182.158.169",
          "netmask": "255.255.240.0",
          "gateway": "147.182.144.1",
          "type": "public"
        },
        {
          "ip_address": "10.118.0.2",
          "netmask": "255.255.240.0",
          "gateway": "10.118.0.1",
          "type": "private"
        }
      ],
      "v6": [
        {
          "ip_address": "2604:a880:cad:d0::e84:c001",
          "netmask": 64,
          "gateway": "2604:a880:cad:d0::1",
          "type": "public"
        }
      ]
    },
    "created_at": "2022-08-31T20:31:11Z",
    "volume_ids": [],
    "vpc_uuid": "949c2fe9-e680-4643-933c-12fe80b3dbc6"
  }
]'
+ echo -n ''
+ echo -e '\tServerAliveInterval 60\n'
cat "$AXIOM_PATH/axiom.json" | jq -r '.sshkey'
++ cat /Users/chevy/.axiom/axiom.json
++ jq -r .sshkey
+ sshkey=axiom_rsa
+ echo -e 'IdentityFile /Users/chevy/.ssh/axiom_rsa'
cat "$AXIOM_PATH/axiom.json" | jq -r '.generate_sshconfig'
++ cat /Users/chevy/.axiom/axiom.json
++ jq -r .generate_sshconfig
+ generate_sshconfig=null
+ [[ null == \p\r\i\v\a\t\e ]]
+ [[ null == \c\a\c\h\e ]]
echo "$droplets" | jq -r '.[].name'
++ jq -r '.[].name'
++ echo '[
  {
    "id": 314838202,
    "name": "stone16",
    "memory": 1024,
    "vcpus": 1,
    "disk": 25,
    "region": {
      "slug": "tor1",
      "name": "Toronto 1",
      "sizes": [
        "s-1vcpu-1gb",
        "s-1vcpu-1gb-amd",
        "s-1vcpu-1gb-intel",
        "s-1vcpu-2gb",
        "s-1vcpu-2gb-amd",
        "s-1vcpu-2gb-intel",
        "s-2vcpu-2gb",
        "s-2vcpu-2gb-amd",
        "s-2vcpu-2gb-intel",
        "s-2vcpu-4gb",
        "s-2vcpu-4gb-amd",
        "s-2vcpu-4gb-intel",
        "c-2",
        "c2-2vcpu-4gb",
        "s-4vcpu-8gb",
        "s-4vcpu-8gb-amd",
        "s-4vcpu-8gb-intel",
        "g-2vcpu-8gb",
        "gd-2vcpu-8gb",
        "m-2vcpu-16gb",
        "c-4",
        "c2-4vcpu-8gb",
        "s-8vcpu-16gb",
        "m3-2vcpu-16gb",
        "s-8vcpu-16gb-amd",
        "s-8vcpu-16gb-intel",
        "g-4vcpu-16gb",
        "so-2vcpu-16gb",
        "m6-2vcpu-16gb",
        "gd-4vcpu-16gb",
        "so1_5-2vcpu-16gb",
        "m-4vcpu-32gb",
        "c-8",
        "c2-8vcpu-16gb",
        "m3-4vcpu-32gb",
        "g-8vcpu-32gb",
        "so-4vcpu-32gb",
        "m6-4vcpu-32gb",
        "gd-8vcpu-32gb",
        "so1_5-4vcpu-32gb",
        "m-8vcpu-64gb",
        "c-16",
        "c2-16vcpu-32gb",
        "m3-8vcpu-64gb",
        "g-16vcpu-64gb",
        "so-8vcpu-64gb",
        "m6-8vcpu-64gb",
        "gd-16vcpu-64gb",
        "so1_5-8vcpu-64gb",
        "m-16vcpu-128gb",
        "c-32",
        "c2-32vcpu-64gb",
        "m3-16vcpu-128gb",
        "c-48",
        "m-24vcpu-192gb",
        "g-32vcpu-128gb",
        "so-16vcpu-128gb",
        "m6-16vcpu-128gb",
        "gd-32vcpu-128gb",
        "c2-48vcpu-96gb",
        "m3-24vcpu-192gb",
        "g-40vcpu-160gb",
        "so1_5-16vcpu-128gb",
        "m-32vcpu-256gb",
        "gd-40vcpu-160gb",
        "so-24vcpu-192gb",
        "m6-24vcpu-192gb",
        "m3-32vcpu-256gb",
        "so1_5-24vcpu-192gb",
        "so-32vcpu-256gb",
        "m6-32vcpu-256gb",
        "so1_5-32vcpu-256gb"
      ],
      "available": true,
      "features": [
        "backups",
        "ipv6",
        "metadata",
        "install_agent",
        "storage",
        "image_transfer"
      ]
    },
    "image": {
      "id": 115944010,
      "name": "axiom-default-1661951687",
      "type": "snapshot",
      "distribution": "Ubuntu",
      "regions": [
        "tor1"
      ],
      "min_disk_size": 25,
      "size_gigabytes": 12.61,
      "created_at": "2022-08-31T13:56:16Z",
      "status": "available"
    },
    "size": {
      "slug": "s-1vcpu-1gb",
      "memory": 1024,
      "vcpus": 1,
      "disk": 25,
      "price_monthly": 6,
      "price_hourly": 0.00893,
      "regions": [
        "ams3",
        "blr1",
        "fra1",
        "lon1",
        "nyc1",
        "nyc3",
        "sfo3",
        "sgp1",
        "tor1"
      ],
      "available": true,
      "transfer": 1,
      "description": "Basic"
    },
    "size_slug": "s-1vcpu-1gb",
    "features": [
      "droplet_agent",
      "ipv6",
      "private_networking"
    ],
    "status": "active",
    "networks": {
      "v4": [
        {
          "ip_address": "147.182.158.169",
          "netmask": "255.255.240.0",
          "gateway": "147.182.144.1",
          "type": "public"
        },
        {
          "ip_address": "10.118.0.2",
          "netmask": "255.255.240.0",
          "gateway": "10.118.0.1",
          "type": "private"
        }
      ],
      "v6": [
        {
          "ip_address": "2604:a880:cad:d0::e84:c001",
          "netmask": 64,
          "gateway": "2604:a880:cad:d0::1",
          "type": "public"
        }
      ]
    },
    "created_at": "2022-08-31T20:31:11Z",
    "volume_ids": [],
    "vpc_uuid": "949c2fe9-e680-4643-933c-12fe80b3dbc6"
  }
]'
+ for name in '$(echo "$droplets" | jq -r '\''.[].name'\'')'
echo "$droplets" | jq -r ".[] | select(.name==\"$name\") | .networks.v4[] | select(.type==\"public\") | .ip_address"
++ echo '[
  {
    "id": 314838202,
    "name": "stone16",
    "memory": 1024,
    "vcpus": 1,
    "disk": 25,
    "region": {
      "slug": "tor1",
      "name": "Toronto 1",
      "sizes": [
        "s-1vcpu-1gb",
        "s-1vcpu-1gb-amd",
        "s-1vcpu-1gb-intel",
        "s-1vcpu-2gb",
        "s-1vcpu-2gb-amd",
        "s-1vcpu-2gb-intel",
        "s-2vcpu-2gb",
++ jq -r '.[] | select(.name=="stone16") | .networks.v4[] | select(.type=="public") | .ip_address'
        "s-2vcpu-2gb-amd",
        "s-2vcpu-2gb-intel",
        "s-2vcpu-4gb",
        "s-2vcpu-4gb-amd",
        "s-2vcpu-4gb-intel",
        "c-2",
        "c2-2vcpu-4gb",
        "s-4vcpu-8gb",
        "s-4vcpu-8gb-amd",
        "s-4vcpu-8gb-intel",
        "g-2vcpu-8gb",
        "gd-2vcpu-8gb",
        "m-2vcpu-16gb",
        "c-4",
        "c2-4vcpu-8gb",
        "s-8vcpu-16gb",
        "m3-2vcpu-16gb",
        "s-8vcpu-16gb-amd",
        "s-8vcpu-16gb-intel",
        "g-4vcpu-16gb",
        "so-2vcpu-16gb",
        "m6-2vcpu-16gb",
        "gd-4vcpu-16gb",
        "so1_5-2vcpu-16gb",
        "m-4vcpu-32gb",
        "c-8",
        "c2-8vcpu-16gb",
        "m3-4vcpu-32gb",
        "g-8vcpu-32gb",
        "so-4vcpu-32gb",
        "m6-4vcpu-32gb",
        "gd-8vcpu-32gb",
        "so1_5-4vcpu-32gb",
        "m-8vcpu-64gb",
        "c-16",
        "c2-16vcpu-32gb",
        "m3-8vcpu-64gb",
        "g-16vcpu-64gb",
        "so-8vcpu-64gb",
        "m6-8vcpu-64gb",
        "gd-16vcpu-64gb",
        "so1_5-8vcpu-64gb",
        "m-16vcpu-128gb",
        "c-32",
        "c2-32vcpu-64gb",
        "m3-16vcpu-128gb",
        "c-48",
        "m-24vcpu-192gb",
        "g-32vcpu-128gb",
        "so-16vcpu-128gb",
        "m6-16vcpu-128gb",
        "gd-32vcpu-128gb",
        "c2-48vcpu-96gb",
        "m3-24vcpu-192gb",
        "g-40vcpu-160gb",
        "so1_5-16vcpu-128gb",
        "m-32vcpu-256gb",
        "gd-40vcpu-160gb",
        "so-24vcpu-192gb",
        "m6-24vcpu-192gb",
        "m3-32vcpu-256gb",
        "so1_5-24vcpu-192gb",
        "so-32vcpu-256gb",
        "m6-32vcpu-256gb",
        "so1_5-32vcpu-256gb"
      ],
      "available": true,
      "features": [
        "backups",
        "ipv6",
        "metadata",
        "install_agent",
        "storage",
        "image_transfer"
      ]
    },
    "image": {
      "id": 115944010,
      "name": "axiom-default-1661951687",
      "type": "snapshot",
      "distribution": "Ubuntu",
      "regions": [
        "tor1"
      ],
      "min_disk_size": 25,
      "size_gigabytes": 12.61,
      "created_at": "2022-08-31T13:56:16Z",
      "status": "available"
    },
    "size": {
      "slug": "s-1vcpu-1gb",
      "memory": 1024,
      "vcpus": 1,
      "disk": 25,
      "price_monthly": 6,
      "price_hourly": 0.00893,
      "regions": [
        "ams3",
        "blr1",
        "fra1",
        "lon1",
        "nyc1",
        "nyc3",
        "sfo3",
        "sgp1",
        "tor1"
      ],
      "available": true,
      "transfer": 1,
      "description": "Basic"
    },
    "size_slug": "s-1vcpu-1gb",
    "features": [
      "droplet_agent",
      "ipv6",
      "private_networking"
    ],
    "status": "active",
    "networks": {
      "v4": [
        {
          "ip_address": "147.182.158.169",
          "netmask": "255.255.240.0",
          "gateway": "147.182.144.1",
          "type": "public"
        },
        {
          "ip_address": "10.118.0.2",
          "netmask": "255.255.240.0",
          "gateway": "10.118.0.1",
          "type": "private"
        }
      ],
      "v6": [
        {
          "ip_address": "2604:a880:cad:d0::e84:c001",
          "netmask": 64,
          "gateway": "2604:a880:cad:d0::1",
          "type": "public"
        }
      ]
    },
    "created_at": "2022-08-31T20:31:11Z",
    "volume_ids": [],
    "vpc_uuid": "949c2fe9-e680-4643-933c-12fe80b3dbc6"
  }
]'
+ ip=147.182.158.169
+ echo -e 'Host stone16\n\tHostName 147.182.158.169\n\tUser op\n\tPort 2266\n'
+ mv /Users/chevy/.axiom/.sshconfig.new30109 /Users/chevy/.axiom/.sshconfig
+ '[' '' '!=' null ']'
+ gen_app_sshconfig
+ echo ''
appliance_list
++ appliance_list
++ appliances
++ jq -r '.[].name'
++ curl -s /heartbeats/
++ jq

###########################################################################################################
# Store $args in $commands
#
commands="$args"
+ commands=' id'

###########################################################################################################
# If --tmux is in the command, connect to instance and spawn a new tmux session
#
if [[ $use_tmux == true ]] ;then
 if [[ -z ${tmux_session_name:+x} ]]; then
  tmux_session_name=$uid
 fi
 commands="tmux new-session -d -s $tmux_session_name \""$commands"\""
fi
+ [[ false == true ]]

###########################################################################################################
#  Create temporary directories and set tmp path to be used for logs
#
completed="$tmp/status/completed/"
+ completed=/Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/status/completed/
inprogress="$tmp/status/inprogress/"
+ inprogress=/Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/status/inprogress/
mkdir -p "$tmp/input"
+ mkdir -p /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/input
mkdir -p "$tmp/split"
+ mkdir -p /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/split
mkdir -p "$tmp/output"
+ mkdir -p /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/output
mkdir -p "$tmp/logs"
+ mkdir -p /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/logs
mkdir -p "$completed"
+ mkdir -p /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/status/completed/
mkdir -p "$inprogress"
+ mkdir -p /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/status/inprogress/
mkdir -p "$tmp/status"
+ mkdir -p /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/status

###########################################################################################################
#  cp the selected.conf to different file names ( one for Interlace, one for selected.conf)
#  Make a copy of the current SSH config and use it for axiom-scan
#
cat "$AXIOM_PATH/selected.conf" >> "$tmp/hosts"
+ cat /Users/chevy/.axiom/selected.conf
cp "$tmp/hosts" "$tmp/selected.conf"
+ cp /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/hosts /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/selected.conf
cp "$sshconfig" "$tmp/sshconfig"
+ cp /Users/chevy/.axiom/.sshconfig /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/sshconfig
sshconfig="$tmp/sshconfig"
+ sshconfig=/Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/sshconfig

###########################################################################################################
#  Create temporary SSH sockets to use with axiom-scan. An advantage of SSH multiplexing is that the overhead
#  of creating new TCP connections and negotiating the secure connection is eliminated. This allow us to do
#  subsequent SSH exec operations ( like downloading results etc ) with no additional overhead.
#
mkdir -p "$tmp/sockets"
+ mkdir -p /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/sockets
socket_tmp=$(echo "$tmp/sockets")
echo "$tmp/sockets"
++ echo /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/sockets
+ socket_tmp=/Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/sockets
cat <<EOT >> $(echo $sshconfig)
Host *
    ControlMaster auto
    ControlPath $socket_tmp/%r@%h-%p
    ControlPersist 600
EOT
+ cat
echo $sshconfig
++ echo /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/sshconfig

###########################################################################################################
#  if --fleet isnt provided or has null value, default to selected.conf
#
if [[ -z ${fleet:+x} ]]; then
total_instances="$(wc -l "$tmp/hosts" | awk '{ print $1 }')"
instances=$(cat "$tmp/hosts")
else

###########################################################################################################
#  if --fleet value is non-null and the value is a file, use selected from file
#
 if [[ -f "$fleet" ]] ; then
  instances=$(cat "$fleet")
  echo "$instances" | tr ' ' '\n' > "$tmp/hosts"
  total_instances="$(wc -l "$tmp/hosts" | awk '{ print $1 }')"

###########################################################################################################
#  else it must the fleet name
#
else
  instances=$(query_instances_cache "$fleet*")
  echo "$instances" | tr ' ' '\n' > "$tmp/hosts"
  total_instances="$(wc -l "$tmp/hosts" | awk '{ print $1 }')"
 fi
fi
+ [[ -z '' ]]
wc -l "$tmp/hosts" | awk '{ print $1 }'
++ wc -l /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/hosts
++ awk '{ print $1 }'
+ total_instances=1
cat "$tmp/hosts"
++ cat /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/hosts
+ instances=stone16

###########################################################################################################
#  if --instance is provided, add it to total instances
#
if [[ $instance != false ]]; then
 name=$(query_instances_cache "$instance")
 echo "$name" | tr ' ' '\n' > "$tmp/hosts"
 total_instances="$(wc -l "$tmp/hosts" | awk '{ print $1 }')"
fi
+ [[ false != false ]]

###########################################################################################################
#  prepare the default SSH and interlace command and execute the user provided command in parallel
#
scan_dir="/tmp/exec/$uid"
+ scan_dir=/tmp/exec/axiom-exec+1661978323
ssh_command="ssh -F $sshconfig -o StrictHostKeyChecking=no -o PasswordAuthentication=no"
+ ssh_command='ssh -F /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/sshconfig -o StrictHostKeyChecking=no -o PasswordAuthentication=no'
interlace_cmd="$(which interlace) -tL $tmp/hosts  -threads $total_instances"
which interlace
++ which interlace
+ interlace_cmd=' -tL /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/hosts  -threads 1'
interlace_cmd_silent="$(which interlace) -tL $tmp/hosts --silent -threads $total_instances"
which interlace
++ which interlace
+ interlace_cmd_silent=' -tL /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/hosts --silent -threads 1'
interlace_cmd_nobar="$(which interlace) --silent --no-bar -tL $tmp/hosts -threads $total_instances"
which interlace
++ which interlace
+ interlace_cmd_nobar=' --silent --no-bar -tL /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/hosts -threads 1'
ssh_exit_command="ssh -F $sshconfig -O exit -o StrictHostKeyChecking=no"
+ ssh_exit_command='ssh -F /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/sshconfig -O exit -o StrictHostKeyChecking=no'

###########################################################################################################
#  prevents Interlace hangups from hijacking your terminal
#
stty -echoctl
+ stty -echoctl
trap clean_up SIGINT SIGTERM
+ trap clean_up SIGINT SIGTERM

###########################################################################################################
# preflight check
#
if [[ $pre_flight == true ]]; then
ssh_command_preflight="ssh -F $sshconfig -o StrictHostKeyChecking=no -o PasswordAuthentication=no -o ConnectTimeout=$preflight_timeout"
$interlace_cmd_silent -c "$ssh_command_preflight _target_ 'echo _target_' >> $tmp/hosts_preflight"
cat "$tmp/hosts_preflight" | sort -u  > "$tmp/hosts"
cp "$tmp/hosts_preflight" "$tmp/selected.conf"
fi
+ [[ true == true ]]
+ ssh_command_preflight='ssh -F /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/sshconfig -o StrictHostKeyChecking=no -o PasswordAuthentication=no -o ConnectTimeout=5'
+ -tL /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/hosts --silent -threads 1 -c 'ssh -F /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/sshconfig -o StrictHostKeyChecking=no -o PasswordAuthentication=no -o ConnectTimeout=5 _target_ '\''echo _target_'\'' >> /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/hosts_preflight'
/Users/chevy/.axiom/interact/axiom-exec: line 357: -tL: command not found
+ cat /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/hosts_preflight
+ sort -u
cat: /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/hosts_preflight: No such file or directory
+ cp /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/hosts_preflight /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/selected.conf
cp: cannot stat '/Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/hosts_preflight': No such file or directory

###########################################################################################################
#  store bash command in a file and upload it to remote instances with axiom-scp
#
echo "$commands" > "$tmp/command"
+ echo ' id'
$interlace_cmd_nobar -c "$ssh_command _target_ 'mkdir -p $scan_dir'" >/dev/null 2>&1
+ --silent --no-bar -tL /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/hosts -threads 1 -c 'ssh -F /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/sshconfig -o StrictHostKeyChecking=no -o PasswordAuthentication=no _target_ '\''mkdir -p /tmp/exec/axiom-exec+1661978323'\'''

$interlace_cmd_nobar -c "axiom-scp $tmp/command _target_:$scan_dir/command --cache -F=$sshconfig >/dev/null 2>&1; touch $tmp/logs/_target_"
+ --silent --no-bar -tL /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/hosts -threads 1 -c 'axiom-scp /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/command _target_:/tmp/exec/axiom-exec+1661978323/command --cache -F=/Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/sshconfig >/dev/null 2>&1; touch /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/logs/_target_'
/Users/chevy/.axiom/interact/axiom-exec: line 368: --silent: command not found

###########################################################################################################
#  this function is spanwed in the background and periodically probes all instances to see if their part of the exec has completed.
#  when the remote exec process has finished, it creates a file named $(hostname) in the remote exec working directory. During the
#  exec if axiom see's the $(hostname) file, break the loop and exit
#
function downloader () {
while true; do
sleep 2
$interlace_cmd_nobar -c "axiom-scp _target_:$scan_dir/_target_ $tmp/status/inprogress/_target_ --cache -F=$sshconfig >/dev/null 2>&1"
ls $tmp/status/inprogress/ | anew -q $tmp/status/completed/hosts
cat $tmp/status/completed/hosts | sort -u | wc -l | tee $tmp/status/downloader_instances  >/dev/null 2>&1
cat $tmp/status/completed/hosts | sort -u | tee $tmp/status/downloader_hosts  >/dev/null 2>&1
if [[ "$(cat $tmp/status/downloader_instances)" -eq "0"  ]]; then
 sleep 2
 downloader
fi
downloader_cmd="$(which interlace) --no-bar --silent -tL $tmp/status/downloader_hosts -threads $(cat $tmp/status/downloader_instances)"
if [[ "$commands" =~ "_target_" ]]; then
$downloader_cmd -c "axiom-scp _target_:$scan_dir/output/ $tmp/output/ --cache -F=$sshconfig >/dev/null 2>&1"
else
$downloader_cmd -c "axiom-scp _target_:$scan_dir/output $tmp/output/_target_.$ext --cache -F=$sshconfig >/dev/null 2>&1"
fi
mv $tmp/status/completed/hosts $tmp/status/completed/hosts.tmp
cat $tmp/status/completed/hosts.tmp | sort -u >> $tmp/status/completed/hosts
 if cmp -s $tmp/status/completed/hosts $tmp/hosts ; then
  kill -9 $(cat $tmp/status/remotetailPID)  >> /dev/null 2>&1
  wait $(cat $tmp/status/remotetailPID)  >> /dev/null 2>&1
 break >> /dev/null 2>&1
 else
  downloader
 fi
done
}

###########################################################################################################
#  interactive mode flag
#
if [[ "$quick_execution" == "false" ]]; then

###########################################################################################################
#  dont tail if quiet is true
#
if [[ "$quiet" == "false" ]]; then
    tail -q -f $tmp/logs/* &
    tailPID=$!
fi

###########################################################################################################
#  disable progress bar, reduce verbosity, only terminal output of the command is returned to terminal
#
touch $tmp/status/completed/hosts
touch $tmp/status/completed/status
sleep 3
downloader &
downloaderPID=$!
$interlace_cmd_nobar -c "$ssh_command _target_ 'cd $scan_dir && touch stderr.log stdout.log && tail -f stderr.log & tail -f stdout.log' >> $tmp/logs/_target_ 2>&1 " &
remotetailPID=$!
echo $remotetailPID > $tmp/status/remotetailPID

if [[ "$nobar" == "false" ]]; then
 $interlace_cmd -c "$ssh_command _target_ 'tmux new -d -s $uid && tmux send-keys -t $uid \"bash -i $scan_dir/command  > >(tee -a $scan_dir/stdout.log) 2> >(tee -a $scan_dir/stderr.log >&2) ; touch $scan_dir/_target_\" ENTER ' \"&& tmux send-keys -t $uid exit ENTER\""
 wait $remotetailPID  >> /dev/null 2>&1
else
 $interlace_cmd_silent -c "$ssh_command _target_ 'tmux new -d -s $uid && tmux send-keys -t $uid \"bash -i $scan_dir/command  > >(tee -a $scan_dir/stdout.log) 2> >(tee -a $scan_dir/stderr.log >&2) ; touch $scan_dir/_target_\" ENTER ' \"&& tmux send-keys -t $uid exit ENTER\""
 wait $remotetailPID  >> /dev/null 2>&1
fi

###########################################################################################################
#  interactive mode logic
#
else
if [[ "$nobar" == "false" ]]; then
 $interlace_cmd -c "$ssh_command _target_ '$commands'"
 pid=$!
else
 $interlace_cmd_nobar -c "$ssh_command _target_  '$commands'"
 pid=$!
fi
fi
+ [[ false == \f\a\l\s\e ]]
+ [[ false == \f\a\l\s\e ]]
+ tailPID=55785
+ touch /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/status/completed/hosts
+ tail -q -f '/Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/logs/*'
tail: cannot open '/Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/logs/*' for reading: No such file or directory
tail: no files remaining
+ touch /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/status/completed/status
+ sleep 3
+ downloaderPID=55791
+ downloader
+ true
+ sleep 2
+ remotetailPID=55792
+ echo 55792
+ [[ false == \f\a\l\s\e ]]
+ --silent --no-bar -tL /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/hosts -threads 1 -c 'ssh -F /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/sshconfig -o StrictHostKeyChecking=no -o PasswordAuthentication=no _target_ '\''cd /tmp/exec/axiom-exec+1661978323 && touch stderr.log stdout.log && tail -f stderr.log & tail -f stdout.log'\'' >> /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/logs/_target_ 2>&1 '
+ -tL /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/hosts -threads 1 -c 'ssh -F /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/sshconfig -o StrictHostKeyChecking=no -o PasswordAuthentication=no _target_ '\''tmux new -d -s axiom-exec+1661978323 && tmux send-keys -t axiom-exec+1661978323 "bash -i /tmp/exec/axiom-exec+1661978323/command  > >(tee -a /tmp/exec/axiom-exec+1661978323/stdout.log) 2> >(tee -a /tmp/exec/axiom-exec+1661978323/stderr.log >&2) ; touch /tmp/exec/axiom-exec+1661978323/_target_" ENTER '\'' "&& tmux send-keys -t axiom-exec+1661978323 exit ENTER"'
/Users/chevy/.axiom/interact/axiom-exec: line 425: --silent: command not found
/Users/chevy/.axiom/interact/axiom-exec: line 430: -tL: command not found
+ wait 55792

###########################################################################################################
#  keep logs flag aka --logs
#
mv "$tmp" "$logs">/dev/null 2>&1
+ mv /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323 /Users/chevy/.axiom/logs/exec/axiom-exec+1661978323
if [[ $keep_logs != true ]]; then
 rm -r $logs >/dev/null 2>&1
fi
+ [[ false != true ]]
+ rm -r /Users/chevy/.axiom/logs/exec/axiom-exec+1661978323

###########################################################################################################
#  kill tmux sessions with any orphaned proceses
#
$interlace_cmd_nobar -c "$ssh_command _target_ 'tmux kill-session -t $uid'"  >/dev/null 2>&1
+ --silent --no-bar -tL /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/hosts -threads 1 -c 'ssh -F /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/sshconfig -o StrictHostKeyChecking=no -o PasswordAuthentication=no _target_ '\''tmux kill-session -t axiom-exec+1661978323'\'''
$interlace_cmd_nobar -c "$ssh_exit_command _target_ " >/dev/null 2>&1
+ --silent --no-bar -tL /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/hosts -threads 1 -c 'ssh -F /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/sshconfig -O exit -o StrictHostKeyChecking=no _target_ '

###########################################################################################################
# house keeping
#
rm -r $socket_tmp >/dev/null 2>&1
+ rm -r /Users/chevy/.axiom/tmp/exec/axiom-exec+1661978323/sockets
kill -0 $remotetailPID 2>  /dev/null && kill -9 $remotetailPID  &> /dev/null
+ kill -0 55792
kill -0 $tailPID 2>  /dev/null && kill -9 $tailPID  &> /dev/null
+ kill -0 55785
kill -0 $downloaderPID 2>  /dev/null && kill -9 $downloaderPID  &> /dev/null
+ kill -0 55791
+ kill -9 55791
wait $tailPID 2>/dev/null
+ wait 55785
wait $downloaderPID 2>/dev/null
+ wait 55791
stty sane
+ stty sane
tput init
+ tput init
0xtavian commented 2 years ago

Thanks! Do you have Interlace installed and is it in your path? It should have automatically installed during axiom-configure. What OS are you using as a controller and how did you install axiom? @chevonmdphillip

chevyphillip commented 2 years ago

@0xtavian awesome! Thank you that worked. I needed Interlace and anew installed. Thanks!