worker service not starting on cdh6.3

[13/Jul/2020 00:12:41 +0000] 117396 MainThread redactor INFO Started launcher: /opt/cloudera/cm-agent/service/csd/csd.sh start_worker [13/Jul/2020 00:12:41 +0000] 117396 MainThread redactor ERROR Redaction rules file doesn't exist, not redacting logs. file: redaction-rules.json, directory: /run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER [13/Jul/2020 00:12:41 +0000] 117396 MainThread redactor INFO Re-exec watcher: /opt/cloudera/cm-agent/bin/cm proc_watcher 117423 Mon Jul 13 00:12:42 EDT 2020

locate_java_home
locate_java_home_no_verify
JAVA11_HOME_CANDIDATES=('/usr/java/jdk-11' '/usr/lib/jvm/jdk-11' '/usr/lib/jvm/java-11-oracle')
local JAVA11_HOME_CANDIDATES
OPENJAVA11_HOME_CANDIDATES=('/usr/lib/jvm/jdk-11' '/usr/lib64/jvm/jdk-11')
local OPENJAVA11_HOME_CANDIDATES
JAVA8_HOME_CANDIDATES=('/usr/java/jdk1.8' '/usr/java/jre1.8' '/usr/lib/jvm/j2sdk1.8-oracle' '/usr/lib/jvm/j2sdk1.8-oracle/jre' '/usr/lib/jvm/java-8-oracle')
local JAVA8_HOME_CANDIDATES
OPENJAVA8_HOME_CANDIDATES=('/usr/lib/jvm/java-1.8.0-openjdk' '/usr/lib/jvm/java-8-openjdk' '/usr/lib64/jvm/java-1.8.0-openjdk' '/usr/lib64/jvm/java-8-openjdk')
local OPENJAVA8_HOME_CANDIDATES
MISCJAVA_HOME_CANDIDATES=('/Library/Java/Home' '/usr/java/default' '/usr/lib/jvm/default-java' '/usr/lib/jvm/java-openjdk' '/usr/lib/jvm/jre-openjdk')
local MISCJAVA_HOME_CANDIDATES
case ${BIGTOP_JAVA_MAJOR} in
JAVA_HOME_CANDIDATES=(${JAVA8_HOME_CANDIDATES[@]} ${MISCJAVA_HOME_CANDIDATES[@]} ${OPENJAVA8_HOME_CANDIDATES[@]} ${JAVA11_HOME_CANDIDATES[@]} ${OPENJAVA11_HOME_CANDIDATES[@]})
'[' -z /usr/lib/jvm/jre-openjdk ']'
verify_java_home
'[' -z /usr/lib/jvm/jre-openjdk ']'
echo JAVA_HOME=/usr/lib/jvm/jre-openjdk
'[' -n '' ']'
source_parcel_environment
'[' '!' -z /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/meta/airflow_env.sh ']'
OLD_IFS='
'
IFS=:
SCRIPT_ARRAY=($SCM_DEFINES_SCRIPTS)
DIRNAME_ARRAY=($PARCEL_DIRNAMES)
IFS='
'
COUNT=1 ++ seq 1 1
for i in 'seq 1 $COUNT'
SCRIPT=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/meta/airflow_env.sh
PARCEL_DIRNAME=AIRFLOW-1.10.3-python2.7.15_b2
. /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/meta/airflow_env.sh ++ AIRFLOW_DIRNAME=AIRFLOW-1.10.3-python2.7.15_b2 ++ export AIRFLOW_DIR=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2 ++ AIRFLOW_DIR=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2 ++ export PATH=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin ++ PATH=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin ++ export PYTHONHOME=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2 ++ PYTHONHOME=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2 ++ export PYTHONPATH=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7 ++ PYTHONPATH=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7 ++ '[' -z '' ']' ++ export AIRFLOW_PYTHON=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/bin/python2.7 ++ AIRFLOW_PYTHON=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/bin/python2.7
echo 'Using /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER as conf dir'
echo 'Using scripts/control.sh as process script'
replace_conf_dir
echo CONF_DIR=/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER
echo CMF_CONF_DIR=
EXCLUDE_CMF_FILES=('cloudera-config.sh' 'hue.sh' 'impala.sh' 'sqoop.sh' 'supervisor.conf' 'config.zip' 'proc.json' '.log' '.keytab' 'jceks') ++ printf '! -name %s ' cloudera-config.sh hue.sh impala.sh sqoop.sh supervisor.conf config.zip proc.json '.log' '.keytab' 'jceks'
find /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER -type f '!' -path '/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/logs/' '!' -name cloudera-config.sh '!' -name hue.sh '!' -name impala.sh '!' -name sqoop.sh '!' -name supervisor.conf '!' -name config.zip '!' -name proc.json '!' -name '.log' '!' -name '.keytab' '!' -name 'jceks' -exec perl -pi -e 's#{{CMF_CONF_DIR}}#/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER#g' '{}' ';'
make_scripts_executable
find /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER -regex '.*.(py|sh)$' -exec chmod u+x '{}' ';'
RUN_DIR=/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER
'[' '' == true ']'
chmod u+x /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/scripts/control.sh
export COMMON_SCRIPT=/opt/cloudera/cm-agent/service/common/cloudera-config.sh
COMMON_SCRIPT=/opt/cloudera/cm-agent/service/common/cloudera-config.sh
exec /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/scripts/control.sh start_worker
date Mon Jul 13 00:12:42 EDT 2020
CMD=start_worker
OPTS=
log '*** AIRFLOW_DIR: /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2' ++ date '+%Y-%m-%d %H:%M:%S'
timestamp='2020-07-13 00:12:42'
echo '2020-07-13 00:12:42: *** AIRFLOW_DIR: /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2'
log '*** AIRFLOW_HOME: ' ++ date '+%Y-%m-%d %H:%M:%S'
timestamp='2020-07-13 00:12:42'
echo '2020-07-13 00:12:42: *** AIRFLOW_HOME: '
log '*** AIRFLOW_CONFIG: ' ++ date '+%Y-%m-%d %H:%M:%S'
timestamp='2020-07-13 00:12:42'
echo '2020-07-13 00:12:42: *** AIRFLOW_CONFIG: '
log '*** PYTHONHOME: /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2' ++ date '+%Y-%m-%d %H:%M:%S'
timestamp='2020-07-13 00:12:42'
echo '2020-07-13 00:12:42: *** PYTHONHOME: /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2'
log '*** PYTHONPATH: /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7' ++ date '+%Y-%m-%d %H:%M:%S'
timestamp='2020-07-13 00:12:42'
echo '2020-07-13 00:12:42: *** PYTHONPATH: /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7'
case $CMD in
update_daemon_config /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER
local DIR=/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER
export AIRFLOW_CONFIG=/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg
AIRFLOW_CONFIG=/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg
log '** AIRFLOW_CONFIG: /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg' ++ date '+%Y-%m-%d %H:%M:%S'
timestamp='2020-07-13 00:12:42'
echo '2020-07-13 00:12:42: ** AIRFLOW_CONFIG: /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg'
deploy_client_config /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER
log 'Converting airflow.properties to airflow.cfg ...' ++ date '+%Y-%m-%d %H:%M:%S'
timestamp='2020-07-13 00:12:42'
echo '2020-07-13 00:12:42: Converting airflow.properties to airflow.cfg ...'
local DIR=/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER
local SECTION
local KV
local KEY
local VAL
read -r LINE ++ echo AIRFLOW_HOME=/var/lib/airflow ++ awk -F: '{print $1}'
SECTION=AIRFLOW_HOME=/var/lib/airflow ++ sed -e 's|AIRFLOW_HOME=/var/lib/airflow:||' ++ echo AIRFLOW_HOME=/var/lib/airflow
KV=AIRFLOW_HOME=/var/lib/airflow ++ echo AIRFLOW_HOME=/var/lib/airflow ++ awk -F= '{print $1}'
KEY=AIRFLOW_HOME ++ awk -F= '{print $2}' ++ echo AIRFLOW_HOME=/var/lib/airflow
VAL=/var/lib/airflow
'[' AIRFLOW_HOME=/var/lib/airflow '!=' AIRFLOW_HOME=/var/lib/airflow ']'
'[' -n AIRFLOW_HOME=/var/lib/airflow ']'
eval AIRFLOW_HOME=/var/lib/airflow ++ AIRFLOW_HOME=/var/lib/airflow
read -r LINE ++ awk -F: '{print $1}' ++ echo CELERY_BROKER=RabbitMQ
SECTION=CELERY_BROKER=RabbitMQ ++ sed -e 's|CELERY_BROKER=RabbitMQ:||' ++ echo CELERY_BROKER=RabbitMQ
KV=CELERY_BROKER=RabbitMQ ++ echo CELERY_BROKER=RabbitMQ ++ awk -F= '{print $1}'
KEY=CELERY_BROKER ++ echo CELERY_BROKER=RabbitMQ ++ awk -F= '{print $2}'
VAL=RabbitMQ
'[' CELERY_BROKER=RabbitMQ '!=' CELERY_BROKER=RabbitMQ ']'
'[' -n CELERY_BROKER=RabbitMQ ']'
eval CELERY_BROKER=RabbitMQ ++ CELERY_BROKER=RabbitMQ
read -r LINE ++ echo CELERY_BROKER_HOST= ++ awk -F: '{print $1}'
SECTION=CELERY_BROKER_HOST= ++ echo CELERY_BROKER_HOST= ++ sed -e 's|CELERY_BROKER_HOST=:||'
KV=CELERY_BROKER_HOST= ++ awk -F= '{print $1}' ++ echo CELERY_BROKER_HOST=
KEY=CELERY_BROKER_HOST ++ echo CELERY_BROKER_HOST= ++ awk -F= '{print $2}'
VAL=
'[' CELERY_BROKER_HOST= '!=' CELERY_BROKER_HOST= ']'
'[' -n CELERY_BROKER_HOST= ']'
eval CELERY_BROKER_HOST= ++ CELERY_BROKER_HOST=
read -r LINE ++ awk -F: '{print $1}' ++ echo CELERY_BROKER_PASS=
SECTION=CELERY_BROKER_PASS= ++ echo CELERY_BROKER_PASS= ++ sed -e 's|CELERY_BROKER_PASS=:||'
KV=CELERY_BROKER_PASS= ++ echo CELERY_BROKER_PASS= ++ awk -F= '{print $1}'
KEY=CELERY_BROKER_PASS ++ echo CELERY_BROKER_PASS= ++ awk -F= '{print $2}'
VAL=
'[' CELERY_BROKER_PASS= '!=' CELERY_BROKER_PASS= ']'
'[' -n CELERY_BROKER_PASS= ']'
eval CELERY_BROKER_PASS= ++ CELERY_BROKER_PASS=
read -r LINE ++ awk -F: '{print $1}' ++ echo CELERY_BROKER_PORT=
SECTION=CELERY_BROKER_PORT= ++ echo CELERY_BROKER_PORT= ++ sed -e 's|CELERY_BROKER_PORT=:||'
KV=CELERY_BROKER_PORT= ++ awk -F= '{print $1}' ++ echo CELERY_BROKER_PORT=
KEY=CELERY_BROKER_PORT ++ echo CELERY_BROKER_PORT= ++ awk -F= '{print $2}'
VAL=
'[' CELERY_BROKER_PORT= '!=' CELERY_BROKER_PORT= ']'
'[' -n CELERY_BROKER_PORT= ']'
eval CELERY_BROKER_PORT= ++ CELERY_BROKER_PORT=
read -r LINE ++ echo CELERY_BROKER_USER= ++ awk -F: '{print $1}'
SECTION=CELERY_BROKER_USER= ++ echo CELERY_BROKER_USER= ++ sed -e 's|CELERY_BROKER_USER=:||'
KV=CELERY_BROKER_USER= ++ echo CELERY_BROKER_USER= ++ awk -F= '{print $1}'
KEY=CELERY_BROKER_USER ++ echo CELERY_BROKER_USER= ++ awk -F= '{print $2}'
VAL=
'[' CELERY_BROKER_USER= '!=' CELERY_BROKER_USER= ']'
'[' -n CELERY_BROKER_USER= ']'
eval CELERY_BROKER_USER= ++ CELERY_BROKER_USER=
read -r LINE ++ echo DB_HOST=usadc-vsedgdh03 ++ awk -F: '{print $1}'
SECTION=DB_HOST=usadc-vsedgdh03 ++ echo DB_HOST=usadc-vsedgdh03 ++ sed -e 's|DB_HOST=usadc-vsedgdh03:||'
KV=DB_HOST=usadc-vsedgdh03 ++ echo DB_HOST=usadc-vsedgdh03 ++ awk -F= '{print $1}'
KEY=DB_HOST ++ echo DB_HOST=usadc-vsedgdh03 ++ awk -F= '{print $2}'
VAL=usadc-vsedgdh03
'[' DB_HOST=usadc-vsedgdh03 '!=' DB_HOST=usadc-vsedgdh03 ']'
'[' -n DB_HOST=usadc-vsedgdh03 ']'
eval DB_HOST=usadc-vsedgdh03 ++ DB_HOST=usadc-vsedgdh03
read -r LINE ++ echo DB_NAME=airflow ++ awk -F: '{print $1}'
SECTION=DB_NAME=airflow ++ echo DB_NAME=airflow ++ sed -e 's|DB_NAME=airflow:||'
KV=DB_NAME=airflow ++ echo DB_NAME=airflow ++ awk -F= '{print $1}'
KEY=DB_NAME ++ awk -F= '{print $2}' ++ echo DB_NAME=airflow
VAL=airflow
'[' DB_NAME=airflow '!=' DB_NAME=airflow ']'
'[' -n DB_NAME=airflow ']'
eval DB_NAME=airflow ++ DB_NAME=airflow
read -r LINE ++ echo DB_PASS=p@ssw0rd123 ++ awk -F: '{print $1}'
SECTION=DB_PASS=p@ssw0rd123 ++ sed -e 's|DB_PASS=p@ssw0rd123:||' ++ echo DB_PASS=p@ssw0rd123
KV=DB_PASS=p@ssw0rd123 ++ echo DB_PASS=p@ssw0rd123 ++ awk -F= '{print $1}'
KEY=DB_PASS ++ echo DB_PASS=p@ssw0rd123 ++ awk -F= '{print $2}'
VAL=p@ssw0rd123
'[' DB_PASS=p@ssw0rd123 '!=' DB_PASS=p@ssw0rd123 ']'
'[' -n DB_PASS=p@ssw0rd123 ']'
eval DB_PASS=p@ssw0rd123 ++ DB_PASS=p@ssw0rd123
read -r LINE ++ echo DB_PORT=3306 ++ awk -F: '{print $1}'
SECTION=DB_PORT=3306 ++ echo DB_PORT=3306 ++ sed -e 's|DB_PORT=3306:||'
KV=DB_PORT=3306 ++ awk -F= '{print $1}' ++ echo DB_PORT=3306
KEY=DB_PORT ++ echo DB_PORT=3306 ++ awk -F= '{print $2}'
VAL=3306
'[' DB_PORT=3306 '!=' DB_PORT=3306 ']'
'[' -n DB_PORT=3306 ']'
eval DB_PORT=3306 ++ DB_PORT=3306
read -r LINE ++ echo DB_TYPE=MySQL ++ awk -F: '{print $1}'
SECTION=DB_TYPE=MySQL ++ echo DB_TYPE=MySQL ++ sed -e 's|DB_TYPE=MySQL:||'
KV=DB_TYPE=MySQL ++ echo DB_TYPE=MySQL ++ awk -F= '{print $1}'
KEY=DB_TYPE ++ echo DB_TYPE=MySQL ++ awk -F= '{print $2}'
VAL=MySQL
'[' DB_TYPE=MySQL '!=' DB_TYPE=MySQL ']'
'[' -n DB_TYPE=MySQL ']'
eval DB_TYPE=MySQL ++ DB_TYPE=MySQL
read -r LINE ++ echo DB_USER=airflow ++ awk -F: '{print $1}'
SECTION=DB_USER=airflow ++ echo DB_USER=airflow ++ sed -e 's|DB_USER=airflow:||'
KV=DB_USER=airflow ++ echo DB_USER=airflow ++ awk -F= '{print $1}'
KEY=DB_USER ++ echo DB_USER=airflow ++ awk -F= '{print $2}'
VAL=airflow
'[' DB_USER=airflow '!=' DB_USER=airflow ']'
'[' -n DB_USER=airflow ']'
eval DB_USER=airflow ++ DB_USER=airflow
read -r LINE ++ echo api:auth_backend=airflow.api.auth.backend.default ++ awk -F: '{print $1}'
SECTION=api ++ echo api:auth_backend=airflow.api.auth.backend.default ++ sed -e 's|api:||'
KV=auth_backend=airflow.api.auth.backend.default ++ awk -F= '{print $1}' ++ echo auth_backend=airflow.api.auth.backend.default
KEY=auth_backend ++ awk -F= '{print $2}' ++ echo auth_backend=airflow.api.auth.backend.default
VAL=airflow.api.auth.backend.default
'[' api '!=' auth_backend=airflow.api.auth.backend.default ']'
'[' airflow.api.auth.backend.default == true ']'
'[' airflow.api.auth.backend.default == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg api auth_backend airflow.api.auth.backend.default
read -r LINE ++ echo celery:default_queue=default ++ awk -F: '{print $1}'
SECTION=celery ++ echo celery:default_queue=default ++ sed -e 's|celery:||'
KV=default_queue=default ++ awk -F= '{print $1}' ++ echo default_queue=default
KEY=default_queue ++ echo default_queue=default ++ awk -F= '{print $2}'
VAL=default
'[' celery '!=' default_queue=default ']'
'[' default == true ']'
'[' default == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg celery default_queue default
read -r LINE ++ echo celery:worker_concurrency=16 ++ awk -F: '{print $1}'
SECTION=celery ++ sed -e 's|celery:||' ++ echo celery:worker_concurrency=16
KV=worker_concurrency=16 ++ echo worker_concurrency=16 ++ awk -F= '{print $1}'
KEY=worker_concurrency ++ echo worker_concurrency=16 ++ awk -F= '{print $2}'
VAL=16
'[' celery '!=' worker_concurrency=16 ']'
'[' 16 == true ']'
'[' 16 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg celery worker_concurrency 16
read -r LINE ++ echo celery:worker_log_server_port=8793 ++ awk -F: '{print $1}'
SECTION=celery ++ echo celery:worker_log_server_port=8793 ++ sed -e 's|celery:||'
KV=worker_log_server_port=8793 ++ awk -F= '{print $1}' ++ echo worker_log_server_port=8793
KEY=worker_log_server_port ++ awk -F= '{print $2}' ++ echo worker_log_server_port=8793
VAL=8793
'[' celery '!=' worker_log_server_port=8793 ']'
'[' 8793 == true ']'
'[' 8793 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg celery worker_log_server_port 8793
read -r LINE ++ echo cli:api_client=airflow.api.client.local_client ++ awk -F: '{print $1}'
SECTION=cli ++ echo cli:api_client=airflow.api.client.local_client ++ sed -e 's|cli:||'
KV=api_client=airflow.api.client.local_client ++ awk -F= '{print $1}' ++ echo api_client=airflow.api.client.local_client
KEY=api_client ++ echo api_client=airflow.api.client.local_client ++ awk -F= '{print $2}'
VAL=airflow.api.client.local_client
'[' cli '!=' api_client=airflow.api.client.local_client ']'
'[' airflow.api.client.local_client == true ']'
'[' airflow.api.client.local_client == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg cli api_client airflow.api.client.local_client
read -r LINE ++ echo cli:endpoint_url=http://localhost:8080 ++ awk -F: '{print $1}'
SECTION=cli ++ echo cli:endpoint_url=http://localhost:8080 ++ sed -e 's|cli:||'
KV=endpoint_url=http://localhost:8080 ++ awk -F= '{print $1}' ++ echo endpoint_url=http://localhost:8080
KEY=endpoint_url ++ echo endpoint_url=http://localhost:8080 ++ awk -F= '{print $2}'
VAL=http://localhost:8080
'[' cli '!=' endpoint_url=http://localhost:8080 ']'
'[' http://localhost:8080 == true ']'
'[' http://localhost:8080 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg cli endpoint_url http://localhost:8080
read -r LINE ++ echo core:base_log_folder=/var/log/airflow ++ awk -F: '{print $1}'
SECTION=core ++ echo core:base_log_folder=/var/log/airflow ++ sed -e 's|core:||'
KV=base_log_folder=/var/log/airflow ++ awk -F= '{print $1}' ++ echo base_log_folder=/var/log/airflow
KEY=base_log_folder ++ echo base_log_folder=/var/log/airflow ++ awk -F= '{print $2}'
VAL=/var/log/airflow
'[' core '!=' base_log_folder=/var/log/airflow ']'
'[' /var/log/airflow == true ']'
'[' /var/log/airflow == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core base_log_folder /var/log/airflow
read -r LINE ++ echo core:dag_concurrency=4 ++ awk -F: '{print $1}'
SECTION=core ++ sed -e 's|core:||' ++ echo core:dag_concurrency=4
KV=dag_concurrency=4 ++ awk -F= '{print $1}' ++ echo dag_concurrency=4
KEY=dag_concurrency ++ awk -F= '{print $2}' ++ echo dag_concurrency=4
VAL=4
'[' core '!=' dag_concurrency=4 ']'
'[' 4 == true ']'
'[' 4 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core dag_concurrency 4
read -r LINE ++ awk -F: '{print $1}' ++ echo core:dagbag_import_timeout=30
SECTION=core ++ sed -e 's|core:||' ++ echo core:dagbag_import_timeout=30
KV=dagbag_import_timeout=30 ++ echo dagbag_import_timeout=30 ++ awk -F= '{print $1}'
KEY=dagbag_import_timeout ++ echo dagbag_import_timeout=30 ++ awk -F= '{print $2}'
VAL=30
'[' core '!=' dagbag_import_timeout=30 ']'
'[' 30 == true ']'
'[' 30 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core dagbag_import_timeout 30
read -r LINE ++ echo core:dags_are_paused_at_creation=true ++ awk -F: '{print $1}'
SECTION=core ++ sed -e 's|core:||' ++ echo core:dags_are_paused_at_creation=true
KV=dags_are_paused_at_creation=true ++ awk -F= '{print $1}' ++ echo dags_are_paused_at_creation=true
KEY=dags_are_paused_at_creation ++ echo dags_are_paused_at_creation=true ++ awk -F= '{print $2}'
VAL=true
'[' core '!=' dags_are_paused_at_creation=true ']'
'[' true == true ']'
VAL=True
'[' True == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core dags_are_paused_at_creation True
read -r LINE ++ echo core:dags_folder=/var/lib/airflow/dags ++ awk -F: '{print $1}'
SECTION=core ++ echo core:dags_folder=/var/lib/airflow/dags ++ sed -e 's|core:||'
KV=dags_folder=/var/lib/airflow/dags ++ echo dags_folder=/var/lib/airflow/dags ++ awk -F= '{print $1}'
KEY=dags_folder ++ echo dags_folder=/var/lib/airflow/dags ++ awk -F= '{print $2}'
VAL=/var/lib/airflow/dags
'[' core '!=' dags_folder=/var/lib/airflow/dags ']'
'[' /var/lib/airflow/dags == true ']'
'[' /var/lib/airflow/dags == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core dags_folder /var/lib/airflow/dags
read -r LINE ++ echo core:default_impersonation= ++ awk -F: '{print $1}'
SECTION=core ++ echo core:default_impersonation= ++ sed -e 's|core:||'
KV=default_impersonation= ++ echo default_impersonation= ++ awk -F= '{print $1}'
KEY=default_impersonation ++ echo default_impersonation= ++ awk -F= '{print $2}'
VAL=
'[' core '!=' default_impersonation= ']'
'[' '' == true ']'
'[' '' == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core default_impersonation ''
read -r LINE ++ awk -F: '{print $1}' ++ echo core:donot_pickle=false
SECTION=core ++ sed -e 's|core:||' ++ echo core:donot_pickle=false
KV=donot_pickle=false ++ echo donot_pickle=false ++ awk -F= '{print $1}'
KEY=donot_pickle ++ awk -F= '{print $2}' ++ echo donot_pickle=false
VAL=false
'[' core '!=' donot_pickle=false ']'
'[' false == true ']'
'[' false == false ']'
VAL=False
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core donot_pickle False
read -r LINE ++ awk -F: '{print $1}' ++ echo core:executor=LocalExecutor
SECTION=core ++ sed -e 's|core:||' ++ echo core:executor=LocalExecutor
KV=executor=LocalExecutor ++ echo executor=LocalExecutor ++ awk -F= '{print $1}'
KEY=executor ++ echo executor=LocalExecutor ++ awk -F= '{print $2}'
VAL=LocalExecutor
'[' core '!=' executor=LocalExecutor ']'
'[' LocalExecutor == true ']'
'[' LocalExecutor == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core executor LocalExecutor
read -r LINE ++ echo core:fernet_key= ++ awk -F: '{print $1}'
SECTION=core ++ sed -e 's|core:||' ++ echo core:fernet_key=
KV=fernet_key= ++ awk -F= '{print $1}' ++ echo fernet_key=
KEY=fernet_key ++ echo fernet_key= ++ awk -F= '{print $2}'
VAL=
'[' core '!=' fernet_key= ']'
'[' '' == true ']'
'[' '' == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core fernet_key ''
read -r LINE ++ awk -F: '{print $1}' ++ echo core:load_examples=false
SECTION=core ++ echo core:load_examples=false ++ sed -e 's|core:||'
KV=load_examples=false ++ awk -F= '{print $1}' ++ echo load_examples=false
KEY=load_examples ++ echo load_examples=false ++ awk -F= '{print $2}'
VAL=false
'[' core '!=' load_examples=false ']'
'[' false == true ']'
'[' false == false ']'
VAL=False
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core load_examples False
read -r LINE ++ awk -F: '{print $1}' ++ echo core:max_active_runs_per_dag=16
SECTION=core ++ sed -e 's|core:||' ++ echo core:max_active_runs_per_dag=16
KV=max_active_runs_per_dag=16 ++ echo max_active_runs_per_dag=16 ++ awk -F= '{print $1}'
KEY=max_active_runs_per_dag ++ awk -F= '{print $2}' ++ echo max_active_runs_per_dag=16
VAL=16
'[' core '!=' max_active_runs_per_dag=16 ']'
'[' 16 == true ']'
'[' 16 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core max_active_runs_per_dag 16
read -r LINE ++ awk -F: '{print $1}' ++ echo core:non_pooled_task_slot_count=128
SECTION=core ++ sed -e 's|core:||' ++ echo core:non_pooled_task_slot_count=128
KV=non_pooled_task_slot_count=128 ++ awk -F= '{print $1}' ++ echo non_pooled_task_slot_count=128
KEY=non_pooled_task_slot_count ++ awk -F= '{print $2}' ++ echo non_pooled_task_slot_count=128
VAL=128
'[' core '!=' non_pooled_task_slot_count=128 ']'
'[' 128 == true ']'
'[' 128 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core non_pooled_task_slot_count 128
read -r LINE ++ awk -F: '{print $1}' ++ echo core:parallelism=8
SECTION=core ++ echo core:parallelism=8 ++ sed -e 's|core:||'
KV=parallelism=8 ++ echo parallelism=8 ++ awk -F= '{print $1}'
KEY=parallelism ++ echo parallelism=8 ++ awk -F= '{print $2}'
VAL=8
'[' core '!=' parallelism=8 ']'
'[' 8 == true ']'
'[' 8 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core parallelism 8
read -r LINE ++ awk -F: '{print $1}' ++ echo core:plugins_folder=/var/lib/airflow/plugins
SECTION=core ++ echo core:plugins_folder=/var/lib/airflow/plugins ++ sed -e 's|core:||'
KV=plugins_folder=/var/lib/airflow/plugins ++ echo plugins_folder=/var/lib/airflow/plugins ++ awk -F= '{print $1}'
KEY=plugins_folder ++ echo plugins_folder=/var/lib/airflow/plugins ++ awk -F= '{print $2}'
VAL=/var/lib/airflow/plugins
'[' core '!=' plugins_folder=/var/lib/airflow/plugins ']'
'[' /var/lib/airflow/plugins == true ']'
'[' /var/lib/airflow/plugins == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core plugins_folder /var/lib/airflow/plugins
read -r LINE ++ awk -F: '{print $1}' ++ echo core:sql_alchemy_pool_enabled=true
SECTION=core ++ sed -e 's|core:||' ++ echo core:sql_alchemy_pool_enabled=true
KV=sql_alchemy_pool_enabled=true ++ awk -F= '{print $1}' ++ echo sql_alchemy_pool_enabled=true
KEY=sql_alchemy_pool_enabled ++ awk -F= '{print $2}' ++ echo sql_alchemy_pool_enabled=true
VAL=true
'[' core '!=' sql_alchemy_pool_enabled=true ']'
'[' true == true ']'
VAL=True
'[' True == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core sql_alchemy_pool_enabled True
read -r LINE ++ echo core:sql_alchemy_pool_recycle=1800 ++ awk -F: '{print $1}'
SECTION=core ++ echo core:sql_alchemy_pool_recycle=1800 ++ sed -e 's|core:||'
KV=sql_alchemy_pool_recycle=1800 ++ awk -F= '{print $1}' ++ echo sql_alchemy_pool_recycle=1800
KEY=sql_alchemy_pool_recycle ++ awk -F= '{print $2}' ++ echo sql_alchemy_pool_recycle=1800
VAL=1800
'[' core '!=' sql_alchemy_pool_recycle=1800 ']'
'[' 1800 == true ']'
'[' 1800 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core sql_alchemy_pool_recycle 1800
read -r LINE ++ echo core:sql_alchemy_pool_size=5 ++ awk -F: '{print $1}'
SECTION=core ++ echo core:sql_alchemy_pool_size=5 ++ sed -e 's|core:||'
KV=sql_alchemy_pool_size=5 ++ echo sql_alchemy_pool_size=5 ++ awk -F= '{print $1}'
KEY=sql_alchemy_pool_size ++ echo sql_alchemy_pool_size=5 ++ awk -F= '{print $2}'
VAL=5
'[' core '!=' sql_alchemy_pool_size=5 ']'
'[' 5 == true ']'
'[' 5 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core sql_alchemy_pool_size 5
read -r LINE ++ echo core:sql_alchemy_reconnect_timeout=300 ++ awk -F: '{print $1}'
SECTION=core ++ echo core:sql_alchemy_reconnect_timeout=300 ++ sed -e 's|core:||'
KV=sql_alchemy_reconnect_timeout=300 ++ echo sql_alchemy_reconnect_timeout=300 ++ awk -F= '{print $1}'
KEY=sql_alchemy_reconnect_timeout ++ awk -F= '{print $2}' ++ echo sql_alchemy_reconnect_timeout=300
VAL=300
'[' core '!=' sql_alchemy_reconnect_timeout=300 ']'
'[' 300 == true ']'
'[' 300 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core sql_alchemy_reconnect_timeout 300
read -r LINE ++ echo core:task_runner=BashTaskRunner ++ awk -F: '{print $1}'
SECTION=core ++ echo core:task_runner=BashTaskRunner ++ sed -e 's|core:||'
KV=task_runner=BashTaskRunner ++ echo task_runner=BashTaskRunner ++ awk -F= '{print $1}'
KEY=task_runner ++ echo task_runner=BashTaskRunner ++ awk -F= '{print $2}'
VAL=BashTaskRunner
'[' core '!=' task_runner=BashTaskRunner ']'
'[' BashTaskRunner == true ']'
'[' BashTaskRunner == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core task_runner BashTaskRunner
read -r LINE ++ echo hive:default_hive_mapred_queue= ++ awk -F: '{print $1}'
SECTION=hive ++ echo hive:default_hive_mapred_queue= ++ sed -e 's|hive:||'
KV=default_hive_mapred_queue= ++ echo default_hive_mapred_queue= ++ awk -F= '{print $1}'
KEY=default_hive_mapred_queue ++ echo default_hive_mapred_queue= ++ awk -F= '{print $2}'
VAL=
'[' hive '!=' default_hive_mapred_queue= ']'
'[' '' == true ']'
'[' '' == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg hive default_hive_mapred_queue ''
read -r LINE ++ echo log_dir=/var/log/airflow ++ awk -F: '{print $1}'
SECTION=log_dir=/var/log/airflow ++ echo log_dir=/var/log/airflow ++ sed -e 's|log_dir=/var/log/airflow:||'
KV=log_dir=/var/log/airflow ++ echo log_dir=/var/log/airflow ++ awk -F= '{print $1}'
KEY=log_dir ++ echo log_dir=/var/log/airflow ++ awk -F= '{print $2}'
VAL=/var/log/airflow
'[' log_dir=/var/log/airflow '!=' log_dir=/var/log/airflow ']'
'[' -n log_dir=/var/log/airflow ']'
eval log_dir=/var/log/airflow ++ log_dir=/var/log/airflow
read -r LINE ++ echo operators:default_cpus=1 ++ awk -F: '{print $1}'
SECTION=operators ++ echo operators:default_cpus=1 ++ sed -e 's|operators:||'
KV=default_cpus=1 ++ echo default_cpus=1 ++ awk -F= '{print $1}'
KEY=default_cpus ++ echo default_cpus=1 ++ awk -F= '{print $2}'
VAL=1
'[' operators '!=' default_cpus=1 ']'
'[' 1 == true ']'
'[' 1 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg operators default_cpus 1
read -r LINE ++ echo operators:default_disk=512 ++ awk -F: '{print $1}'
SECTION=operators ++ echo operators:default_disk=512 ++ sed -e 's|operators:||'
KV=default_disk=512 ++ echo default_disk=512 ++ awk -F= '{print $1}'
KEY=default_disk ++ echo default_disk=512 ++ awk -F= '{print $2}'
VAL=512
'[' operators '!=' default_disk=512 ']'
'[' 512 == true ']'
'[' 512 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg operators default_disk 512
read -r LINE ++ echo operators:default_gpus=0 ++ awk -F: '{print $1}'
SECTION=operators ++ echo operators:default_gpus=0 ++ sed -e 's|operators:||'
KV=default_gpus=0 ++ echo default_gpus=0 ++ awk -F= '{print $1}'
KEY=default_gpus ++ echo default_gpus=0 ++ awk -F= '{print $2}'
VAL=0
'[' operators '!=' default_gpus=0 ']'
'[' 0 == true ']'
'[' 0 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg operators default_gpus 0
read -r LINE ++ echo operators:default_owner=Airflow ++ awk -F: '{print $1}'
SECTION=operators ++ echo operators:default_owner=Airflow ++ sed -e 's|operators:||'
KV=default_owner=Airflow ++ echo default_owner=Airflow ++ awk -F= '{print $1}'
KEY=default_owner ++ echo default_owner=Airflow ++ awk -F= '{print $2}'
VAL=Airflow
'[' operators '!=' default_owner=Airflow ']'
'[' Airflow == true ']'
'[' Airflow == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg operators default_owner Airflow
read -r LINE ++ echo operators:default_ram=512 ++ awk -F: '{print $1}'
SECTION=operators ++ echo operators:default_ram=512 ++ sed -e 's|operators:||'
KV=default_ram=512 ++ echo default_ram=512 ++ awk -F= '{print $1}'
KEY=default_ram ++ echo default_ram=512 ++ awk -F= '{print $2}'
VAL=512
'[' operators '!=' default_ram=512 ']'
'[' 512 == true ']'
'[' 512 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg operators default_ram 512
read -r LINE ++ echo smtp:smtp_host=localhost ++ awk -F: '{print $1}'
SECTION=smtp ++ echo smtp:smtp_host=localhost ++ sed -e 's|smtp:||'
KV=smtp_host=localhost ++ awk -F= '{print $1}' ++ echo smtp_host=localhost
KEY=smtp_host ++ awk -F= '{print $2}' ++ echo smtp_host=localhost
VAL=localhost
'[' smtp '!=' smtp_host=localhost ']'
'[' localhost == true ']'
'[' localhost == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg smtp smtp_host localhost
read -r LINE ++ awk -F: '{print $1}' ++ echo smtp:smtp_mail_from=airflow@localhost
SECTION=smtp ++ sed -e 's|smtp:||' ++ echo smtp:smtp_mail_from=airflow@localhost
KV=smtp_mail_from=airflow@localhost ++ awk -F= '{print $1}' ++ echo smtp_mail_from=airflow@localhost
KEY=smtp_mail_from ++ awk -F= '{print $2}' ++ echo smtp_mail_from=airflow@localhost
VAL=airflow@localhost
'[' smtp '!=' smtp_mail_from=airflow@localhost ']'
'[' airflow@localhost == true ']'
'[' airflow@localhost == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg smtp smtp_mail_from airflow@localhost
read -r LINE ++ echo smtp:smtp_password= ++ awk -F: '{print $1}'
SECTION=smtp ++ echo smtp:smtp_password= ++ sed -e 's|smtp:||'
KV=smtp_password= ++ echo smtp_password= ++ awk -F= '{print $1}'
KEY=smtp_password ++ echo smtp_password= ++ awk -F= '{print $2}'
VAL=
'[' smtp '!=' smtp_password= ']'
'[' '' == true ']'
'[' '' == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg smtp smtp_password ''
read -r LINE ++ echo smtp:smtp_port=25 ++ awk -F: '{print $1}'
SECTION=smtp ++ echo smtp:smtp_port=25 ++ sed -e 's|smtp:||'
KV=smtp_port=25 ++ echo smtp_port=25 ++ awk -F= '{print $1}'
KEY=smtp_port ++ echo smtp_port=25 ++ awk -F= '{print $2}'
VAL=25
'[' smtp '!=' smtp_port=25 ']'
'[' 25 == true ']'
'[' 25 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg smtp smtp_port 25
read -r LINE ++ echo smtp:smtp_ssl=false ++ awk -F: '{print $1}'
SECTION=smtp ++ echo smtp:smtp_ssl=false ++ sed -e 's|smtp:||'
KV=smtp_ssl=false ++ echo smtp_ssl=false ++ awk -F= '{print $1}'
KEY=smtp_ssl ++ echo smtp_ssl=false ++ awk -F= '{print $2}'
VAL=false
'[' smtp '!=' smtp_ssl=false ']'
'[' false == true ']'
'[' false == false ']'
VAL=False
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg smtp smtp_ssl False
read -r LINE ++ echo smtp:smtp_starttls=true ++ awk -F: '{print $1}'
SECTION=smtp ++ echo smtp:smtp_starttls=true ++ sed -e 's|smtp:||'
KV=smtp_starttls=true ++ echo smtp_starttls=true ++ awk -F= '{print $1}'
KEY=smtp_starttls ++ echo smtp_starttls=true ++ awk -F= '{print $2}'
VAL=true
'[' smtp '!=' smtp_starttls=true ']'
'[' true == true ']'
VAL=True
'[' True == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg smtp smtp_starttls True
read -r LINE ++ echo smtp:smtp_user= ++ awk -F: '{print $1}'
SECTION=smtp ++ echo smtp:smtp_user= ++ sed -e 's|smtp:||'
KV=smtp_user= ++ echo smtp_user= ++ awk -F= '{print $1}'
KEY=smtp_user ++ awk -F= '{print $2}' ++ echo smtp_user=
VAL=
'[' smtp '!=' smtp_user= ']'
'[' '' == true ']'
'[' '' == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg smtp smtp_user ''
read -r LINE
'[' MySQL == SQLite3 ']'
'[' MySQL == MySQL ']'
VAL=mysql://airflow:p@ssw0rd123@usadc-vsedgdh03:3306/airflow
'[' -n mysql://airflow:p@ssw0rd123@usadc-vsedgdh03:3306/airflow ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core sql_alchemy_conn mysql://airflow:p@ssw0rd123@usadc-vsedgdh03:3306/airflow
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg celery result_backend db+mysql://airflow:p@ssw0rd123@usadc-vsedgdh03:3306/airflow
'[' RabbitMQ == RabbitMQ ']'
VAL=amqp://:@:/
'[' -n amqp://:@:/ ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg celery broker_url amqp://:@:/
echo -e '\nexport AIRFLOW_HOME=/var/lib/airflow'
echo -e '\nexport AIRFLOW_CONFIG=/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg'
chgrp airflow /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow-env.sh
log 'Starting Airflow worker...' ++ date '+%Y-%m-%d %H:%M:%S'
timestamp='2020-07-13 00:12:45'
echo '2020-07-13 00:12:45: Starting Airflow worker...'
su -s /bin/bash - airflow -c 'CONF_DIR=/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER exec /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/bin/airflow-cm.sh worker ' /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/airflow/configuration.py:214: FutureWarning: The task_runner setting in [core] has the old default value of u'BashTaskRunner'. This value has been changed to u'StandardTaskRunner' in the running config, but please update your config before Apache Airflow 2.0. FutureWarning, Traceback (most recent call last): File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/bin/.airflow", line 32, in args.func(args) File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/airflow/utils/cli.py", line 74, in wrapper return f(*args, kwargs) File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/airflow/bin/cli.py", line 1090, in worker worker.run(options) File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/celery/bin/worker.py", line 257, in run kwargs) File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/celery/worker/worker.py", line 101, in init self.setup_instance(self.prepare_args(kwargs)) File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/celery/worker/worker.py", line 122, in setup_instance self._conninfo = self.app.connection_for_read() File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/celery/app/base.py", line 743, in connection_for_read return self._connection(url or self.conf.broker_read_url, kwargs) File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/celery/app/base.py", line 819, in _connection 'broker_connection_timeout', connect_timeout File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/kombu/connection.py", line 189, in init url_params = parse_url(hostname) File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/kombu/utils/url.py", line 47, in parse_url scheme, host, port, user, password, path, query = _parse_url(url) File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/kombu/utils/url.py", line 81, in url_to_parts parts.port, File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/future/backports/urllib/parse.py", line 162, in port port = int(port, 10) File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/future/types/newint.py", line 76, in new return super(newint, cls).new(cls, val) ValueError: invalid literal for long() with base 10: '' [13/Jul/2020 00:12:48 +0000] 120062 MainThread redactor INFO Started launcher: /opt/cloudera/cm-agent/service/csd/csd.sh start_worker [13/Jul/2020 00:12:48 +0000] 120062 MainThread redactor ERROR Redaction rules file doesn't exist, not redacting logs. file: redaction-rules.json, directory: /run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER [13/Jul/2020 00:12:48 +0000] 120062 MainThread redactor INFO Re-exec watcher: /opt/cloudera/cm-agent/bin/cm proc_watcher 120079 Mon Jul 13 00:12:48 EDT 2020
locate_java_home
locate_java_home_no_verify
JAVA11_HOME_CANDIDATES=('/usr/java/jdk-11' '/usr/lib/jvm/jdk-11' '/usr/lib/jvm/java-11-oracle')
local JAVA11_HOME_CANDIDATES
OPENJAVA11_HOME_CANDIDATES=('/usr/lib/jvm/jdk-11' '/usr/lib64/jvm/jdk-11')
local OPENJAVA11_HOME_CANDIDATES
JAVA8_HOME_CANDIDATES=('/usr/java/jdk1.8' '/usr/java/jre1.8' '/usr/lib/jvm/j2sdk1.8-oracle' '/usr/lib/jvm/j2sdk1.8-oracle/jre' '/usr/lib/jvm/java-8-oracle')
local JAVA8_HOME_CANDIDATES
OPENJAVA8_HOME_CANDIDATES=('/usr/lib/jvm/java-1.8.0-openjdk' '/usr/lib/jvm/java-8-openjdk' '/usr/lib64/jvm/java-1.8.0-openjdk' '/usr/lib64/jvm/java-8-openjdk')
local OPENJAVA8_HOME_CANDIDATES
MISCJAVA_HOME_CANDIDATES=('/Library/Java/Home' '/usr/java/default' '/usr/lib/jvm/default-java' '/usr/lib/jvm/java-openjdk' '/usr/lib/jvm/jre-openjdk')
local MISCJAVA_HOME_CANDIDATES
case ${BIGTOP_JAVA_MAJOR} in
JAVA_HOME_CANDIDATES=(${JAVA8_HOME_CANDIDATES[@]} ${MISCJAVA_HOME_CANDIDATES[@]} ${OPENJAVA8_HOME_CANDIDATES[@]} ${JAVA11_HOME_CANDIDATES[@]} ${OPENJAVA11_HOME_CANDIDATES[@]})
'[' -z /usr/lib/jvm/jre-openjdk ']'
verify_java_home
'[' -z /usr/lib/jvm/jre-openjdk ']'
echo JAVA_HOME=/usr/lib/jvm/jre-openjdk
'[' -n '' ']'
source_parcel_environment
'[' '!' -z /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/meta/airflow_env.sh ']'
OLD_IFS='
'
IFS=:
SCRIPT_ARRAY=($SCM_DEFINES_SCRIPTS)
DIRNAME_ARRAY=($PARCEL_DIRNAMES)
IFS='
'
COUNT=1 ++ seq 1 1
for i in 'seq 1 $COUNT'
SCRIPT=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/meta/airflow_env.sh
PARCEL_DIRNAME=AIRFLOW-1.10.3-python2.7.15_b2
. /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/meta/airflow_env.sh ++ AIRFLOW_DIRNAME=AIRFLOW-1.10.3-python2.7.15_b2 ++ export AIRFLOW_DIR=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2 ++ AIRFLOW_DIR=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2 ++ export PATH=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin ++ PATH=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin ++ export PYTHONHOME=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2 ++ PYTHONHOME=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2 ++ export PYTHONPATH=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7 ++ PYTHONPATH=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7 ++ '[' -z '' ']' ++ export AIRFLOW_PYTHON=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/bin/python2.7 ++ AIRFLOW_PYTHON=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/bin/python2.7
echo 'Using /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER as conf dir'
echo 'Using scripts/control.sh as process script'
replace_conf_dir
echo CONF_DIR=/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER
echo CMF_CONF_DIR=
EXCLUDE_CMF_FILES=('cloudera-config.sh' 'hue.sh' 'impala.sh' 'sqoop.sh' 'supervisor.conf' 'config.zip' 'proc.json' '.log' '.keytab' 'jceks') ++ printf '! -name %s ' cloudera-config.sh hue.sh impala.sh sqoop.sh supervisor.conf config.zip proc.json '.log' '.keytab' 'jceks'
find /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER -type f '!' -path '/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/logs/' '!' -name cloudera-config.sh '!' -name hue.sh '!' -name impala.sh '!' -name sqoop.sh '!' -name supervisor.conf '!' -name config.zip '!' -name proc.json '!' -name '.log' '!' -name '.keytab' '!' -name 'jceks' -exec perl -pi -e 's#{{CMF_CONF_DIR}}#/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER#g' '{}' ';'
make_scripts_executable
find /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER -regex '.*.(py|sh)$' -exec chmod u+x '{}' ';'
RUN_DIR=/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER
'[' '' == true ']'
chmod u+x /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/scripts/control.sh
export COMMON_SCRIPT=/opt/cloudera/cm-agent/service/common/cloudera-config.sh
COMMON_SCRIPT=/opt/cloudera/cm-agent/service/common/cloudera-config.sh
exec /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/scripts/control.sh start_worker
date Mon Jul 13 00:12:49 EDT 2020
CMD=start_worker
OPTS=
log '*** AIRFLOW_DIR: /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2' ++ date '+%Y-%m-%d %H:%M:%S'
timestamp='2020-07-13 00:12:49'
echo '2020-07-13 00:12:49: *** AIRFLOW_DIR: /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2'
log '*** AIRFLOW_HOME: ' ++ date '+%Y-%m-%d %H:%M:%S'
timestamp='2020-07-13 00:12:49'
echo '2020-07-13 00:12:49: *** AIRFLOW_HOME: '
log '*** AIRFLOW_CONFIG: ' ++ date '+%Y-%m-%d %H:%M:%S'
timestamp='2020-07-13 00:12:49'
echo '2020-07-13 00:12:49: *** AIRFLOW_CONFIG: '
log '*** PYTHONHOME: /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2' ++ date '+%Y-%m-%d %H:%M:%S'
timestamp='2020-07-13 00:12:49'
echo '2020-07-13 00:12:49: *** PYTHONHOME: /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2'
log '*** PYTHONPATH: /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7' ++ date '+%Y-%m-%d %H:%M:%S'
timestamp='2020-07-13 00:12:49'
echo '2020-07-13 00:12:49: *** PYTHONPATH: /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7'
case $CMD in
update_daemon_config /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER
local DIR=/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER
export AIRFLOW_CONFIG=/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg
AIRFLOW_CONFIG=/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg
log '** AIRFLOW_CONFIG: /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg' ++ date '+%Y-%m-%d %H:%M:%S'
timestamp='2020-07-13 00:12:49'
echo '2020-07-13 00:12:49: ** AIRFLOW_CONFIG: /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg'
deploy_client_config /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER
log 'Converting airflow.properties to airflow.cfg ...' ++ date '+%Y-%m-%d %H:%M:%S'
timestamp='2020-07-13 00:12:49'
echo '2020-07-13 00:12:49: Converting airflow.properties to airflow.cfg ...'
local DIR=/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER
local SECTION
local KV
local KEY
local VAL
read -r LINE ++ awk -F: '{print $1}' ++ echo AIRFLOW_HOME=/var/lib/airflow
SECTION=AIRFLOW_HOME=/var/lib/airflow ++ echo AIRFLOW_HOME=/var/lib/airflow ++ sed -e 's|AIRFLOW_HOME=/var/lib/airflow:||'
KV=AIRFLOW_HOME=/var/lib/airflow ++ echo AIRFLOW_HOME=/var/lib/airflow ++ awk -F= '{print $1}'
KEY=AIRFLOW_HOME ++ echo AIRFLOW_HOME=/var/lib/airflow ++ awk -F= '{print $2}'
VAL=/var/lib/airflow
'[' AIRFLOW_HOME=/var/lib/airflow '!=' AIRFLOW_HOME=/var/lib/airflow ']'
'[' -n AIRFLOW_HOME=/var/lib/airflow ']'
eval AIRFLOW_HOME=/var/lib/airflow ++ AIRFLOW_HOME=/var/lib/airflow
read -r LINE ++ echo CELERY_BROKER=RabbitMQ ++ awk -F: '{print $1}'
SECTION=CELERY_BROKER=RabbitMQ ++ echo CELERY_BROKER=RabbitMQ ++ sed -e 's|CELERY_BROKER=RabbitMQ:||'
KV=CELERY_BROKER=RabbitMQ ++ awk -F= '{print $1}' ++ echo CELERY_BROKER=RabbitMQ
KEY=CELERY_BROKER ++ echo CELERY_BROKER=RabbitMQ ++ awk -F= '{print $2}'
VAL=RabbitMQ
'[' CELERY_BROKER=RabbitMQ '!=' CELERY_BROKER=RabbitMQ ']'
'[' -n CELERY_BROKER=RabbitMQ ']'
eval CELERY_BROKER=RabbitMQ ++ CELERY_BROKER=RabbitMQ
read -r LINE ++ echo CELERY_BROKER_HOST= ++ awk -F: '{print $1}'
SECTION=CELERY_BROKER_HOST= ++ echo CELERY_BROKER_HOST= ++ sed -e 's|CELERY_BROKER_HOST=:||'
KV=CELERY_BROKER_HOST= ++ echo CELERY_BROKER_HOST= ++ awk -F= '{print $1}'
KEY=CELERY_BROKER_HOST ++ echo CELERY_BROKER_HOST= ++ awk -F= '{print $2}'
VAL=
'[' CELERY_BROKER_HOST= '!=' CELERY_BROKER_HOST= ']'
'[' -n CELERY_BROKER_HOST= ']'
eval CELERY_BROKER_HOST= ++ CELERY_BROKER_HOST=
read -r LINE ++ awk -F: '{print $1}' ++ echo CELERY_BROKER_PASS=
SECTION=CELERY_BROKER_PASS= ++ sed -e 's|CELERY_BROKER_PASS=:||' ++ echo CELERY_BROKER_PASS=
KV=CELERY_BROKER_PASS= ++ echo CELERY_BROKER_PASS= ++ awk -F= '{print $1}'
KEY=CELERY_BROKER_PASS ++ echo CELERY_BROKER_PASS= ++ awk -F= '{print $2}'
VAL=
'[' CELERY_BROKER_PASS= '!=' CELERY_BROKER_PASS= ']'
'[' -n CELERY_BROKER_PASS= ']'
eval CELERY_BROKER_PASS= ++ CELERY_BROKER_PASS=
read -r LINE ++ echo CELERY_BROKER_PORT= ++ awk -F: '{print $1}'
SECTION=CELERY_BROKER_PORT= ++ echo CELERY_BROKER_PORT= ++ sed -e 's|CELERY_BROKER_PORT=:||'
KV=CELERY_BROKER_PORT= ++ awk -F= '{print $1}' ++ echo CELERY_BROKER_PORT=
KEY=CELERY_BROKER_PORT ++ awk -F= '{print $2}' ++ echo CELERY_BROKER_PORT=
VAL=
'[' CELERY_BROKER_PORT= '!=' CELERY_BROKER_PORT= ']'
'[' -n CELERY_BROKER_PORT= ']'
eval CELERY_BROKER_PORT= ++ CELERY_BROKER_PORT=
read -r LINE ++ awk -F: '{print $1}' ++ echo CELERY_BROKER_USER=
SECTION=CELERY_BROKER_USER= ++ echo CELERY_BROKER_USER= ++ sed -e 's|CELERY_BROKER_USER=:||'
KV=CELERY_BROKER_USER= ++ echo CELERY_BROKER_USER= ++ awk -F= '{print $1}'
KEY=CELERY_BROKER_USER ++ echo CELERY_BROKER_USER= ++ awk -F= '{print $2}'
VAL=
'[' CELERY_BROKER_USER= '!=' CELERY_BROKER_USER= ']'
'[' -n CELERY_BROKER_USER= ']'
eval CELERY_BROKER_USER= ++ CELERY_BROKER_USER=
read -r LINE ++ echo DB_HOST=usadc-vsedgdh03 ++ awk -F: '{print $1}'
SECTION=DB_HOST=usadc-vsedgdh03 ++ echo DB_HOST=usadc-vsedgdh03 ++ sed -e 's|DB_HOST=usadc-vsedgdh03:||'
KV=DB_HOST=usadc-vsedgdh03 ++ echo DB_HOST=usadc-vsedgdh03 ++ awk -F= '{print $1}'
KEY=DB_HOST ++ echo DB_HOST=usadc-vsedgdh03 ++ awk -F= '{print $2}'
VAL=usadc-vsedgdh03
'[' DB_HOST=usadc-vsedgdh03 '!=' DB_HOST=usadc-vsedgdh03 ']'
'[' -n DB_HOST=usadc-vsedgdh03 ']'
eval DB_HOST=usadc-vsedgdh03 ++ DB_HOST=usadc-vsedgdh03
read -r LINE ++ echo DB_NAME=airflow ++ awk -F: '{print $1}'
SECTION=DB_NAME=airflow ++ echo DB_NAME=airflow ++ sed -e 's|DB_NAME=airflow:||'
KV=DB_NAME=airflow ++ echo DB_NAME=airflow ++ awk -F= '{print $1}'
KEY=DB_NAME ++ echo DB_NAME=airflow ++ awk -F= '{print $2}'
VAL=airflow
'[' DB_NAME=airflow '!=' DB_NAME=airflow ']'
'[' -n DB_NAME=airflow ']'
eval DB_NAME=airflow ++ DB_NAME=airflow
read -r LINE ++ echo DB_PASS=p@ssw0rd123 ++ awk -F: '{print $1}'
SECTION=DB_PASS=p@ssw0rd123 ++ echo DB_PASS=p@ssw0rd123 ++ sed -e 's|DB_PASS=p@ssw0rd123:||'
KV=DB_PASS=p@ssw0rd123 ++ echo DB_PASS=p@ssw0rd123 ++ awk -F= '{print $1}'
KEY=DB_PASS ++ echo DB_PASS=p@ssw0rd123 ++ awk -F= '{print $2}'
VAL=p@ssw0rd123
'[' DB_PASS=p@ssw0rd123 '!=' DB_PASS=p@ssw0rd123 ']'
'[' -n DB_PASS=p@ssw0rd123 ']'
eval DB_PASS=p@ssw0rd123 ++ DB_PASS=p@ssw0rd123
read -r LINE ++ echo DB_PORT=3306 ++ awk -F: '{print $1}'
SECTION=DB_PORT=3306 ++ echo DB_PORT=3306 ++ sed -e 's|DB_PORT=3306:||'
KV=DB_PORT=3306 ++ awk -F= '{print $1}' ++ echo DB_PORT=3306
KEY=DB_PORT ++ echo DB_PORT=3306 ++ awk -F= '{print $2}'
VAL=3306
'[' DB_PORT=3306 '!=' DB_PORT=3306 ']'
'[' -n DB_PORT=3306 ']'
eval DB_PORT=3306 ++ DB_PORT=3306
read -r LINE ++ awk -F: '{print $1}' ++ echo DB_TYPE=MySQL
SECTION=DB_TYPE=MySQL ++ sed -e 's|DB_TYPE=MySQL:||' ++ echo DB_TYPE=MySQL
KV=DB_TYPE=MySQL ++ echo DB_TYPE=MySQL ++ awk -F= '{print $1}'
KEY=DB_TYPE ++ echo DB_TYPE=MySQL ++ awk -F= '{print $2}'
VAL=MySQL
'[' DB_TYPE=MySQL '!=' DB_TYPE=MySQL ']'
'[' -n DB_TYPE=MySQL ']'
eval DB_TYPE=MySQL ++ DB_TYPE=MySQL
read -r LINE ++ awk -F: '{print $1}' ++ echo DB_USER=airflow
SECTION=DB_USER=airflow ++ sed -e 's|DB_USER=airflow:||' ++ echo DB_USER=airflow
KV=DB_USER=airflow ++ echo DB_USER=airflow ++ awk -F= '{print $1}'
KEY=DB_USER ++ echo DB_USER=airflow ++ awk -F= '{print $2}'
VAL=airflow
'[' DB_USER=airflow '!=' DB_USER=airflow ']'
'[' -n DB_USER=airflow ']'
eval DB_USER=airflow ++ DB_USER=airflow
read -r LINE ++ echo api:auth_backend=airflow.api.auth.backend.default ++ awk -F: '{print $1}'
SECTION=api ++ echo api:auth_backend=airflow.api.auth.backend.default ++ sed -e 's|api:||'
KV=auth_backend=airflow.api.auth.backend.default ++ echo auth_backend=airflow.api.auth.backend.default ++ awk -F= '{print $1}'
KEY=auth_backend ++ echo auth_backend=airflow.api.auth.backend.default ++ awk -F= '{print $2}'
VAL=airflow.api.auth.backend.default
'[' api '!=' auth_backend=airflow.api.auth.backend.default ']'
'[' airflow.api.auth.backend.default == true ']'
'[' airflow.api.auth.backend.default == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg api auth_backend airflow.api.auth.backend.default
read -r LINE ++ echo celery:default_queue=default ++ awk -F: '{print $1}'
SECTION=celery ++ echo celery:default_queue=default ++ sed -e 's|celery:||'
KV=default_queue=default ++ echo default_queue=default ++ awk -F= '{print $1}'
KEY=default_queue ++ echo default_queue=default ++ awk -F= '{print $2}'
VAL=default
'[' celery '!=' default_queue=default ']'
'[' default == true ']'
'[' default == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg celery default_queue default
read -r LINE ++ echo celery:worker_concurrency=16 ++ awk -F: '{print $1}'
SECTION=celery ++ echo celery:worker_concurrency=16 ++ sed -e 's|celery:||'
KV=worker_concurrency=16 ++ echo worker_concurrency=16 ++ awk -F= '{print $1}'
KEY=worker_concurrency ++ echo worker_concurrency=16 ++ awk -F= '{print $2}'
VAL=16
'[' celery '!=' worker_concurrency=16 ']'
'[' 16 == true ']'
'[' 16 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg celery worker_concurrency 16
read -r LINE ++ awk -F: '{print $1}' ++ echo celery:worker_log_server_port=8793
SECTION=celery ++ echo celery:worker_log_server_port=8793 ++ sed -e 's|celery:||'
KV=worker_log_server_port=8793 ++ echo worker_log_server_port=8793 ++ awk -F= '{print $1}'
KEY=worker_log_server_port ++ echo worker_log_server_port=8793 ++ awk -F= '{print $2}'
VAL=8793
'[' celery '!=' worker_log_server_port=8793 ']'
'[' 8793 == true ']'
'[' 8793 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg celery worker_log_server_port 8793
read -r LINE ++ echo cli:api_client=airflow.api.client.local_client ++ awk -F: '{print $1}'
SECTION=cli ++ echo cli:api_client=airflow.api.client.local_client ++ sed -e 's|cli:||'
KV=api_client=airflow.api.client.local_client ++ echo api_client=airflow.api.client.local_client ++ awk -F= '{print $1}'
KEY=api_client ++ echo api_client=airflow.api.client.local_client ++ awk -F= '{print $2}'
VAL=airflow.api.client.local_client
'[' cli '!=' api_client=airflow.api.client.local_client ']'
'[' airflow.api.client.local_client == true ']'
'[' airflow.api.client.local_client == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg cli api_client airflow.api.client.local_client
read -r LINE ++ echo cli:endpoint_url=http://localhost:8080 ++ awk -F: '{print $1}'
SECTION=cli ++ echo cli:endpoint_url=http://localhost:8080 ++ sed -e 's|cli:||'
KV=endpoint_url=http://localhost:8080 ++ echo endpoint_url=http://localhost:8080 ++ awk -F= '{print $1}'
KEY=endpoint_url ++ awk -F= '{print $2}' ++ echo endpoint_url=http://localhost:8080
VAL=http://localhost:8080
'[' cli '!=' endpoint_url=http://localhost:8080 ']'
'[' http://localhost:8080 == true ']'
'[' http://localhost:8080 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg cli endpoint_url http://localhost:8080
read -r LINE ++ awk -F: '{print $1}' ++ echo core:base_log_folder=/var/log/airflow
SECTION=core ++ echo core:base_log_folder=/var/log/airflow ++ sed -e 's|core:||'
KV=base_log_folder=/var/log/airflow ++ awk -F= '{print $1}' ++ echo base_log_folder=/var/log/airflow
KEY=base_log_folder ++ echo base_log_folder=/var/log/airflow ++ awk -F= '{print $2}'
VAL=/var/log/airflow
'[' core '!=' base_log_folder=/var/log/airflow ']'
'[' /var/log/airflow == true ']'
'[' /var/log/airflow == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core base_log_folder /var/log/airflow
read -r LINE ++ echo core:dag_concurrency=4 ++ awk -F: '{print $1}'
SECTION=core ++ echo core:dag_concurrency=4 ++ sed -e 's|core:||'
KV=dag_concurrency=4 ++ echo dag_concurrency=4 ++ awk -F= '{print $1}'
KEY=dag_concurrency ++ echo dag_concurrency=4 ++ awk -F= '{print $2}'
VAL=4
'[' core '!=' dag_concurrency=4 ']'
'[' 4 == true ']'
'[' 4 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core dag_concurrency 4
read -r LINE ++ echo core:dagbag_import_timeout=30 ++ awk -F: '{print $1}'
SECTION=core ++ sed -e 's|core:||' ++ echo core:dagbag_import_timeout=30
KV=dagbag_import_timeout=30 ++ echo dagbag_import_timeout=30 ++ awk -F= '{print $1}'
KEY=dagbag_import_timeout ++ echo dagbag_import_timeout=30 ++ awk -F= '{print $2}'
VAL=30
'[' core '!=' dagbag_import_timeout=30 ']'
'[' 30 == true ']'
'[' 30 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core dagbag_import_timeout 30
read -r LINE ++ echo core:dags_are_paused_at_creation=true ++ awk -F: '{print $1}'
SECTION=core ++ echo core:dags_are_paused_at_creation=true ++ sed -e 's|core:||'
KV=dags_are_paused_at_creation=true ++ echo dags_are_paused_at_creation=true ++ awk -F= '{print $1}'
KEY=dags_are_paused_at_creation ++ echo dags_are_paused_at_creation=true ++ awk -F= '{print $2}'
VAL=true
'[' core '!=' dags_are_paused_at_creation=true ']'
'[' true == true ']'
VAL=True
'[' True == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core dags_are_paused_at_creation True
read -r LINE ++ awk -F: '{print $1}' ++ echo core:dags_folder=/var/lib/airflow/dags
SECTION=core ++ echo core:dags_folder=/var/lib/airflow/dags ++ sed -e 's|core:||'
KV=dags_folder=/var/lib/airflow/dags ++ awk -F= '{print $1}' ++ echo dags_folder=/var/lib/airflow/dags
KEY=dags_folder ++ echo dags_folder=/var/lib/airflow/dags ++ awk -F= '{print $2}'
VAL=/var/lib/airflow/dags
'[' core '!=' dags_folder=/var/lib/airflow/dags ']'
'[' /var/lib/airflow/dags == true ']'
'[' /var/lib/airflow/dags == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core dags_folder /var/lib/airflow/dags
read -r LINE ++ echo core:default_impersonation= ++ awk -F: '{print $1}'
SECTION=core ++ echo core:default_impersonation= ++ sed -e 's|core:||'
KV=default_impersonation= ++ echo default_impersonation= ++ awk -F= '{print $1}'
KEY=default_impersonation ++ echo default_impersonation= ++ awk -F= '{print $2}'
VAL=
'[' core '!=' default_impersonation= ']'
'[' '' == true ']'
'[' '' == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core default_impersonation ''
read -r LINE ++ echo core:donot_pickle=false ++ awk -F: '{print $1}'
SECTION=core ++ sed -e 's|core:||' ++ echo core:donot_pickle=false
KV=donot_pickle=false ++ echo donot_pickle=false ++ awk -F= '{print $1}'
KEY=donot_pickle ++ echo donot_pickle=false ++ awk -F= '{print $2}'
VAL=false
'[' core '!=' donot_pickle=false ']'
'[' false == true ']'
'[' false == false ']'
VAL=False
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core donot_pickle False
read -r LINE ++ echo core:executor=LocalExecutor ++ awk -F: '{print $1}'
SECTION=core ++ echo core:executor=LocalExecutor ++ sed -e 's|core:||'
KV=executor=LocalExecutor ++ echo executor=LocalExecutor ++ awk -F= '{print $1}'
KEY=executor ++ echo executor=LocalExecutor ++ awk -F= '{print $2}'
VAL=LocalExecutor
'[' core '!=' executor=LocalExecutor ']'
'[' LocalExecutor == true ']'
'[' LocalExecutor == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core executor LocalExecutor
read -r LINE ++ echo core:fernet_key= ++ awk -F: '{print $1}'
SECTION=core ++ echo core:fernet_key= ++ sed -e 's|core:||'
KV=fernet_key= ++ echo fernet_key= ++ awk -F= '{print $1}'
KEY=fernet_key ++ echo fernet_key= ++ awk -F= '{print $2}'
VAL=
'[' core '!=' fernet_key= ']'
'[' '' == true ']'
'[' '' == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core fernet_key ''
read -r LINE ++ echo core:load_examples=false ++ awk -F: '{print $1}'
SECTION=core ++ echo core:load_examples=false ++ sed -e 's|core:||'
KV=load_examples=false ++ echo load_examples=false ++ awk -F= '{print $1}'
KEY=load_examples ++ echo load_examples=false ++ awk -F= '{print $2}'
VAL=false
'[' core '!=' load_examples=false ']'
'[' false == true ']'
'[' false == false ']'
VAL=False
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core load_examples False
read -r LINE ++ echo core:max_active_runs_per_dag=16 ++ awk -F: '{print $1}'
SECTION=core ++ echo core:max_active_runs_per_dag=16 ++ sed -e 's|core:||'
KV=max_active_runs_per_dag=16 ++ echo max_active_runs_per_dag=16 ++ awk -F= '{print $1}'
KEY=max_active_runs_per_dag ++ echo max_active_runs_per_dag=16 ++ awk -F= '{print $2}'
VAL=16
'[' core '!=' max_active_runs_per_dag=16 ']'
'[' 16 == true ']'
'[' 16 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core max_active_runs_per_dag 16
read -r LINE ++ echo core:non_pooled_task_slot_count=128 ++ awk -F: '{print $1}'
SECTION=core ++ echo core:non_pooled_task_slot_count=128 ++ sed -e 's|core:||'
KV=non_pooled_task_slot_count=128 ++ echo non_pooled_task_slot_count=128 ++ awk -F= '{print $1}'
KEY=non_pooled_task_slot_count ++ echo non_pooled_task_slot_count=128 ++ awk -F= '{print $2}'
VAL=128
'[' core '!=' non_pooled_task_slot_count=128 ']'
'[' 128 == true ']'
'[' 128 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core non_pooled_task_slot_count 128
read -r LINE ++ echo core:parallelism=8 ++ awk -F: '{print $1}'
SECTION=core ++ echo core:parallelism=8 ++ sed -e 's|core:||'
KV=parallelism=8 ++ echo parallelism=8 ++ awk -F= '{print $1}'
KEY=parallelism ++ echo parallelism=8 ++ awk -F= '{print $2}'
VAL=8
'[' core '!=' parallelism=8 ']'
'[' 8 == true ']'
'[' 8 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core parallelism 8
read -r LINE ++ echo core:plugins_folder=/var/lib/airflow/plugins ++ awk -F: '{print $1}'
SECTION=core ++ echo core:plugins_folder=/var/lib/airflow/plugins ++ sed -e 's|core:||'
KV=plugins_folder=/var/lib/airflow/plugins ++ echo plugins_folder=/var/lib/airflow/plugins ++ awk -F= '{print $1}'
KEY=plugins_folder ++ echo plugins_folder=/var/lib/airflow/plugins ++ awk -F= '{print $2}'
VAL=/var/lib/airflow/plugins
'[' core '!=' plugins_folder=/var/lib/airflow/plugins ']'
'[' /var/lib/airflow/plugins == true ']'
'[' /var/lib/airflow/plugins == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core plugins_folder /var/lib/airflow/plugins
read -r LINE ++ echo core:sql_alchemy_pool_enabled=true ++ awk -F: '{print $1}'
SECTION=core ++ echo core:sql_alchemy_pool_enabled=true ++ sed -e 's|core:||'
KV=sql_alchemy_pool_enabled=true ++ echo sql_alchemy_pool_enabled=true ++ awk -F= '{print $1}'
KEY=sql_alchemy_pool_enabled ++ echo sql_alchemy_pool_enabled=true ++ awk -F= '{print $2}'
VAL=true
'[' core '!=' sql_alchemy_pool_enabled=true ']'
'[' true == true ']'
VAL=True
'[' True == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core sql_alchemy_pool_enabled True
read -r LINE ++ echo core:sql_alchemy_pool_recycle=1800 ++ awk -F: '{print $1}'
SECTION=core ++ echo core:sql_alchemy_pool_recycle=1800 ++ sed -e 's|core:||'
KV=sql_alchemy_pool_recycle=1800 ++ echo sql_alchemy_pool_recycle=1800 ++ awk -F= '{print $1}'
KEY=sql_alchemy_pool_recycle ++ echo sql_alchemy_pool_recycle=1800 ++ awk -F= '{print $2}'
VAL=1800
'[' core '!=' sql_alchemy_pool_recycle=1800 ']'
'[' 1800 == true ']'
'[' 1800 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core sql_alchemy_pool_recycle 1800
read -r LINE ++ echo core:sql_alchemy_pool_size=5 ++ awk -F: '{print $1}'
SECTION=core ++ echo core:sql_alchemy_pool_size=5 ++ sed -e 's|core:||'
KV=sql_alchemy_pool_size=5 ++ echo sql_alchemy_pool_size=5 ++ awk -F= '{print $1}'
KEY=sql_alchemy_pool_size ++ echo sql_alchemy_pool_size=5 ++ awk -F= '{print $2}'
VAL=5
'[' core '!=' sql_alchemy_pool_size=5 ']'
'[' 5 == true ']'
'[' 5 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core sql_alchemy_pool_size 5
read -r LINE ++ echo core:sql_alchemy_reconnect_timeout=300 ++ awk -F: '{print $1}'
SECTION=core ++ echo core:sql_alchemy_reconnect_timeout=300 ++ sed -e 's|core:||'
KV=sql_alchemy_reconnect_timeout=300 ++ echo sql_alchemy_reconnect_timeout=300 ++ awk -F= '{print $1}'
KEY=sql_alchemy_reconnect_timeout ++ echo sql_alchemy_reconnect_timeout=300 ++ awk -F= '{print $2}'
VAL=300
'[' core '!=' sql_alchemy_reconnect_timeout=300 ']'
'[' 300 == true ']'
'[' 300 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core sql_alchemy_reconnect_timeout 300
read -r LINE ++ awk -F: '{print $1}' ++ echo core:task_runner=BashTaskRunner
SECTION=core ++ echo core:task_runner=BashTaskRunner ++ sed -e 's|core:||'
KV=task_runner=BashTaskRunner ++ awk -F= '{print $1}' ++ echo task_runner=BashTaskRunner
KEY=task_runner ++ awk -F= '{print $2}' ++ echo task_runner=BashTaskRunner
VAL=BashTaskRunner
'[' core '!=' task_runner=BashTaskRunner ']'
'[' BashTaskRunner == true ']'
'[' BashTaskRunner == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core task_runner BashTaskRunner
read -r LINE ++ awk -F: '{print $1}' ++ echo hive:default_hive_mapred_queue=
SECTION=hive ++ sed -e 's|hive:||' ++ echo hive:default_hive_mapred_queue=
KV=default_hive_mapred_queue= ++ echo default_hive_mapred_queue= ++ awk -F= '{print $1}'
KEY=default_hive_mapred_queue ++ echo default_hive_mapred_queue= ++ awk -F= '{print $2}'
VAL=
'[' hive '!=' default_hive_mapred_queue= ']'
'[' '' == true ']'
'[' '' == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg hive default_hive_mapred_queue ''
read -r LINE ++ awk -F: '{print $1}' ++ echo log_dir=/var/log/airflow
SECTION=log_dir=/var/log/airflow ++ sed -e 's|log_dir=/var/log/airflow:||' ++ echo log_dir=/var/log/airflow
KV=log_dir=/var/log/airflow ++ awk -F= '{print $1}' ++ echo log_dir=/var/log/airflow
KEY=log_dir ++ awk -F= '{print $2}' ++ echo log_dir=/var/log/airflow
VAL=/var/log/airflow
'[' log_dir=/var/log/airflow '!=' log_dir=/var/log/airflow ']'
'[' -n log_dir=/var/log/airflow ']'
eval log_dir=/var/log/airflow ++ log_dir=/var/log/airflow
read -r LINE ++ echo operators:default_cpus=1 ++ awk -F: '{print $1}'
SECTION=operators ++ echo operators:default_cpus=1 ++ sed -e 's|operators:||'
KV=default_cpus=1 ++ awk -F= '{print $1}' ++ echo default_cpus=1
KEY=default_cpus ++ awk -F= '{print $2}' ++ echo default_cpus=1
VAL=1
'[' operators '!=' default_cpus=1 ']'
'[' 1 == true ']'
'[' 1 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg operators default_cpus 1
read -r LINE ++ awk -F: '{print $1}' ++ echo operators:default_disk=512
SECTION=operators ++ echo operators:default_disk=512 ++ sed -e 's|operators:||'
KV=default_disk=512 ++ echo default_disk=512 ++ awk -F= '{print $1}'
KEY=default_disk ++ echo default_disk=512 ++ awk -F= '{print $2}'
VAL=512
'[' operators '!=' default_disk=512 ']'
'[' 512 == true ']'
'[' 512 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg operators default_disk 512
read -r LINE ++ echo operators:default_gpus=0 ++ awk -F: '{print $1}'
SECTION=operators ++ echo operators:default_gpus=0 ++ sed -e 's|operators:||'
KV=default_gpus=0 ++ echo default_gpus=0 ++ awk -F= '{print $1}'
KEY=default_gpus ++ echo default_gpus=0 ++ awk -F= '{print $2}'
VAL=0
'[' operators '!=' default_gpus=0 ']'
'[' 0 == true ']'
'[' 0 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg operators default_gpus 0
read -r LINE ++ echo operators:default_owner=Airflow ++ awk -F: '{print $1}'
SECTION=operators ++ echo operators:default_owner=Airflow ++ sed -e 's|operators:||'
KV=default_owner=Airflow ++ echo default_owner=Airflow ++ awk -F= '{print $1}'
KEY=default_owner ++ echo default_owner=Airflow ++ awk -F= '{print $2}'
VAL=Airflow
'[' operators '!=' default_owner=Airflow ']'
'[' Airflow == true ']'
'[' Airflow == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg operators default_owner Airflow
read -r LINE ++ echo operators:default_ram=512 ++ awk -F: '{print $1}'
SECTION=operators ++ echo operators:default_ram=512 ++ sed -e 's|operators:||'
KV=default_ram=512 ++ echo default_ram=512 ++ awk -F= '{print $1}'
KEY=default_ram ++ echo default_ram=512 ++ awk -F= '{print $2}'
VAL=512
'[' operators '!=' default_ram=512 ']'
'[' 512 == true ']'
'[' 512 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg operators default_ram 512
read -r LINE ++ echo smtp:smtp_host=localhost ++ awk -F: '{print $1}'
SECTION=smtp ++ echo smtp:smtp_host=localhost ++ sed -e 's|smtp:||'
KV=smtp_host=localhost ++ echo smtp_host=localhost ++ awk -F= '{print $1}'
KEY=smtp_host ++ echo smtp_host=localhost ++ awk -F= '{print $2}'
VAL=localhost
'[' smtp '!=' smtp_host=localhost ']'
'[' localhost == true ']'
'[' localhost == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg smtp smtp_host localhost
read -r LINE ++ echo smtp:smtp_mail_from=airflow@localhost ++ awk -F: '{print $1}'
SECTION=smtp ++ echo smtp:smtp_mail_from=airflow@localhost ++ sed -e 's|smtp:||'
KV=smtp_mail_from=airflow@localhost ++ echo smtp_mail_from=airflow@localhost ++ awk -F= '{print $1}'
KEY=smtp_mail_from ++ echo smtp_mail_from=airflow@localhost ++ awk -F= '{print $2}'
VAL=airflow@localhost
'[' smtp '!=' smtp_mail_from=airflow@localhost ']'
'[' airflow@localhost == true ']'
'[' airflow@localhost == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg smtp smtp_mail_from airflow@localhost
read -r LINE ++ echo smtp:smtp_password= ++ awk -F: '{print $1}'
SECTION=smtp ++ echo smtp:smtp_password= ++ sed -e 's|smtp:||'
KV=smtp_password= ++ echo smtp_password= ++ awk -F= '{print $1}'
KEY=smtp_password ++ echo smtp_password= ++ awk -F= '{print $2}'
VAL=
'[' smtp '!=' smtp_password= ']'
'[' '' == true ']'
'[' '' == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg smtp smtp_password ''
read -r LINE ++ echo smtp:smtp_port=25 ++ awk -F: '{print $1}'
SECTION=smtp ++ echo smtp:smtp_port=25 ++ sed -e 's|smtp:||'
KV=smtp_port=25 ++ echo smtp_port=25 ++ awk -F= '{print $1}'
KEY=smtp_port ++ echo smtp_port=25 ++ awk -F= '{print $2}'
VAL=25
'[' smtp '!=' smtp_port=25 ']'
'[' 25 == true ']'
'[' 25 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg smtp smtp_port 25
read -r LINE ++ echo smtp:smtp_ssl=false ++ awk -F: '{print $1}'
SECTION=smtp ++ echo smtp:smtp_ssl=false ++ sed -e 's|smtp:||'
KV=smtp_ssl=false ++ echo smtp_ssl=false ++ awk -F= '{print $1}'
KEY=smtp_ssl ++ echo smtp_ssl=false ++ awk -F= '{print $2}'
VAL=false
'[' smtp '!=' smtp_ssl=false ']'
'[' false == true ']'
'[' false == false ']'
VAL=False
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg smtp smtp_ssl False
read -r LINE ++ echo smtp:smtp_starttls=true ++ awk -F: '{print $1}'
SECTION=smtp ++ echo smtp:smtp_starttls=true ++ sed -e 's|smtp:||'
KV=smtp_starttls=true ++ echo smtp_starttls=true ++ awk -F= '{print $1}'
KEY=smtp_starttls ++ echo smtp_starttls=true ++ awk -F= '{print $2}'
VAL=true
'[' smtp '!=' smtp_starttls=true ']'
'[' true == true ']'
VAL=True
'[' True == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg smtp smtp_starttls True
read -r LINE ++ echo smtp:smtp_user= ++ awk -F: '{print $1}'
SECTION=smtp ++ echo smtp:smtp_user= ++ sed -e 's|smtp:||'
KV=smtp_user= ++ echo smtp_user= ++ awk -F= '{print $1}'
KEY=smtp_user ++ echo smtp_user= ++ awk -F= '{print $2}'
VAL=
'[' smtp '!=' smtp_user= ']'
'[' '' == true ']'
'[' '' == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg smtp smtp_user ''
read -r LINE
'[' MySQL == SQLite3 ']'
'[' MySQL == MySQL ']'
VAL=mysql://airflow:p@ssw0rd123@usadc-vsedgdh03:3306/airflow
'[' -n mysql://airflow:p@ssw0rd123@usadc-vsedgdh03:3306/airflow ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core sql_alchemy_conn mysql://airflow:p@ssw0rd123@usadc-vsedgdh03:3306/airflow
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg celery result_backend db+mysql://airflow:p@ssw0rd123@usadc-vsedgdh03:3306/airflow
'[' RabbitMQ == RabbitMQ ']'
VAL=amqp://:@:/
'[' -n amqp://:@:/ ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg celery broker_url amqp://:@:/
echo -e '\nexport AIRFLOW_HOME=/var/lib/airflow'
echo -e '\nexport AIRFLOW_CONFIG=/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg'
chgrp airflow /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow-env.sh
log 'Starting Airflow worker...' ++ date '+%Y-%m-%d %H:%M:%S'
timestamp='2020-07-13 00:12:51'
echo '2020-07-13 00:12:51: Starting Airflow worker...'
su -s /bin/bash - airflow -c 'CONF_DIR=/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER exec /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/bin/airflow-cm.sh worker ' /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/airflow/configuration.py:214: FutureWarning: The task_runner setting in [core] has the old default value of u'BashTaskRunner'. This value has been changed to u'StandardTaskRunner' in the running config, but please update your config before Apache Airflow 2.0. FutureWarning, Traceback (most recent call last): File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/bin/.airflow", line 32, in args.func(args) File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/airflow/utils/cli.py", line 74, in wrapper return f(*args, kwargs) File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/airflow/bin/cli.py", line 1090, in worker worker.run(options) File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/celery/bin/worker.py", line 257, in run kwargs) File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/celery/worker/worker.py", line 101, in init self.setup_instance(self.prepare_args(kwargs)) File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/celery/worker/worker.py", line 122, in setup_instance self._conninfo = self.app.connection_for_read() File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/celery/app/base.py", line 743, in connection_for_read return self._connection(url or self.conf.broker_read_url, kwargs) File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/celery/app/base.py", line 819, in _connection 'broker_connection_timeout', connect_timeout File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/kombu/connection.py", line 189, in init url_params = parse_url(hostname) File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/kombu/utils/url.py", line 47, in parse_url scheme, host, port, user, password, path, query = _parse_url(url) File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/kombu/utils/url.py", line 81, in url_to_parts parts.port, File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/future/backports/urllib/parse.py", line 162, in port port = int(port, 10) File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/future/types/newint.py", line 76, in new return super(newint, cls).new(cls, val) ValueError: invalid literal for long() with base 10: '' [13/Jul/2020 00:12:55 +0000] 120851 MainThread redactor INFO Started launcher: /opt/cloudera/cm-agent/service/csd/csd.sh start_worker [13/Jul/2020 00:12:55 +0000] 120851 MainThread redactor ERROR Redaction rules file doesn't exist, not redacting logs. file: redaction-rules.json, directory: /run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER [13/Jul/2020 00:12:55 +0000] 120851 MainThread redactor INFO Re-exec watcher: /opt/cloudera/cm-agent/bin/cm proc_watcher 120860 Mon Jul 13 00:12:56 EDT 2020
locate_java_home
locate_java_home_no_verify
JAVA11_HOME_CANDIDATES=('/usr/java/jdk-11' '/usr/lib/jvm/jdk-11' '/usr/lib/jvm/java-11-oracle')
local JAVA11_HOME_CANDIDATES
OPENJAVA11_HOME_CANDIDATES=('/usr/lib/jvm/jdk-11' '/usr/lib64/jvm/jdk-11')
local OPENJAVA11_HOME_CANDIDATES
JAVA8_HOME_CANDIDATES=('/usr/java/jdk1.8' '/usr/java/jre1.8' '/usr/lib/jvm/j2sdk1.8-oracle' '/usr/lib/jvm/j2sdk1.8-oracle/jre' '/usr/lib/jvm/java-8-oracle')
local JAVA8_HOME_CANDIDATES
OPENJAVA8_HOME_CANDIDATES=('/usr/lib/jvm/java-1.8.0-openjdk' '/usr/lib/jvm/java-8-openjdk' '/usr/lib64/jvm/java-1.8.0-openjdk' '/usr/lib64/jvm/java-8-openjdk')
local OPENJAVA8_HOME_CANDIDATES
MISCJAVA_HOME_CANDIDATES=('/Library/Java/Home' '/usr/java/default' '/usr/lib/jvm/default-java' '/usr/lib/jvm/java-openjdk' '/usr/lib/jvm/jre-openjdk')
local MISCJAVA_HOME_CANDIDATES
case ${BIGTOP_JAVA_MAJOR} in
JAVA_HOME_CANDIDATES=(${JAVA8_HOME_CANDIDATES[@]} ${MISCJAVA_HOME_CANDIDATES[@]} ${OPENJAVA8_HOME_CANDIDATES[@]} ${JAVA11_HOME_CANDIDATES[@]} ${OPENJAVA11_HOME_CANDIDATES[@]})
'[' -z /usr/lib/jvm/jre-openjdk ']'
verify_java_home
'[' -z /usr/lib/jvm/jre-openjdk ']'
echo JAVA_HOME=/usr/lib/jvm/jre-openjdk
'[' -n '' ']'
source_parcel_environment
'[' '!' -z /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/meta/airflow_env.sh ']'
OLD_IFS='
'
IFS=:
SCRIPT_ARRAY=($SCM_DEFINES_SCRIPTS)
DIRNAME_ARRAY=($PARCEL_DIRNAMES)
IFS='
'
COUNT=1 ++ seq 1 1
for i in 'seq 1 $COUNT'
SCRIPT=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/meta/airflow_env.sh
PARCEL_DIRNAME=AIRFLOW-1.10.3-python2.7.15_b2
. /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/meta/airflow_env.sh ++ AIRFLOW_DIRNAME=AIRFLOW-1.10.3-python2.7.15_b2 ++ export AIRFLOW_DIR=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2 ++ AIRFLOW_DIR=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2 ++ export PATH=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin ++ PATH=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin ++ export PYTHONHOME=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2 ++ PYTHONHOME=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2 ++ export PYTHONPATH=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7 ++ PYTHONPATH=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7 ++ '[' -z '' ']' ++ export AIRFLOW_PYTHON=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/bin/python2.7 ++ AIRFLOW_PYTHON=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/bin/python2.7
echo 'Using /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER as conf dir'
echo 'Using scripts/control.sh as process script'
replace_conf_dir
echo CONF_DIR=/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER
echo CMF_CONF_DIR=
EXCLUDE_CMF_FILES=('cloudera-config.sh' 'hue.sh' 'impala.sh' 'sqoop.sh' 'supervisor.conf' 'config.zip' 'proc.json' '.log' '.keytab' 'jceks') ++ printf '! -name %s ' cloudera-config.sh hue.sh impala.sh sqoop.sh supervisor.conf config.zip proc.json '.log' '.keytab' 'jceks'
find /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER -type f '!' -path '/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/logs/' '!' -name cloudera-config.sh '!' -name hue.sh '!' -name impala.sh '!' -name sqoop.sh '!' -name supervisor.conf '!' -name config.zip '!' -name proc.json '!' -name '.log' '!' -name '.keytab' '!' -name 'jceks' -exec perl -pi -e 's#{{CMF_CONF_DIR}}#/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER#g' '{}' ';'
make_scripts_executable
find /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER -regex '.*.(py|sh)$' -exec chmod u+x '{}' ';'
RUN_DIR=/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER
'[' '' == true ']'
chmod u+x /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/scripts/control.sh
export COMMON_SCRIPT=/opt/cloudera/cm-agent/service/common/cloudera-config.sh
COMMON_SCRIPT=/opt/cloudera/cm-agent/service/common/cloudera-config.sh
exec /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/scripts/control.sh start_worker
date Mon Jul 13 00:12:56 EDT 2020
CMD=start_worker
OPTS=
log '*** AIRFLOW_DIR: /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2' ++ date '+%Y-%m-%d %H:%M:%S'
timestamp='2020-07-13 00:12:56'
echo '2020-07-13 00:12:56: *** AIRFLOW_DIR: /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2'
log '*** AIRFLOW_HOME: ' ++ date '+%Y-%m-%d %H:%M:%S'
timestamp='2020-07-13 00:12:56'
echo '2020-07-13 00:12:56: *** AIRFLOW_HOME: '
log '*** AIRFLOW_CONFIG: ' ++ date '+%Y-%m-%d %H:%M:%S'
timestamp='2020-07-13 00:12:56'
echo '2020-07-13 00:12:56: *** AIRFLOW_CONFIG: '
log '*** PYTHONHOME: /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2' ++ date '+%Y-%m-%d %H:%M:%S'
timestamp='2020-07-13 00:12:56'
echo '2020-07-13 00:12:56: *** PYTHONHOME: /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2'
log '*** PYTHONPATH: /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7' ++ date '+%Y-%m-%d %H:%M:%S'
timestamp='2020-07-13 00:12:56'
echo '2020-07-13 00:12:56: *** PYTHONPATH: /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7'
case $CMD in
update_daemon_config /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER
local DIR=/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER
export AIRFLOW_CONFIG=/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg
AIRFLOW_CONFIG=/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg
log '** AIRFLOW_CONFIG: /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg' ++ date '+%Y-%m-%d %H:%M:%S'
timestamp='2020-07-13 00:12:56'
echo '2020-07-13 00:12:56: ** AIRFLOW_CONFIG: /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg'
deploy_client_config /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER
log 'Converting airflow.properties to airflow.cfg ...' ++ date '+%Y-%m-%d %H:%M:%S'
timestamp='2020-07-13 00:12:56'
echo '2020-07-13 00:12:56: Converting airflow.properties to airflow.cfg ...'
local DIR=/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER
local SECTION
local KV
local KEY
local VAL
read -r LINE ++ echo AIRFLOW_HOME=/var/lib/airflow ++ awk -F: '{print $1}'
SECTION=AIRFLOW_HOME=/var/lib/airflow ++ echo AIRFLOW_HOME=/var/lib/airflow ++ sed -e 's|AIRFLOW_HOME=/var/lib/airflow:||'
KV=AIRFLOW_HOME=/var/lib/airflow ++ echo AIRFLOW_HOME=/var/lib/airflow ++ awk -F= '{print $1}'
KEY=AIRFLOW_HOME ++ echo AIRFLOW_HOME=/var/lib/airflow ++ awk -F= '{print $2}'
VAL=/var/lib/airflow
'[' AIRFLOW_HOME=/var/lib/airflow '!=' AIRFLOW_HOME=/var/lib/airflow ']'
'[' -n AIRFLOW_HOME=/var/lib/airflow ']'
eval AIRFLOW_HOME=/var/lib/airflow ++ AIRFLOW_HOME=/var/lib/airflow
read -r LINE ++ echo CELERY_BROKER=RabbitMQ ++ awk -F: '{print $1}'
SECTION=CELERY_BROKER=RabbitMQ ++ echo CELERY_BROKER=RabbitMQ ++ sed -e 's|CELERY_BROKER=RabbitMQ:||'
KV=CELERY_BROKER=RabbitMQ ++ echo CELERY_BROKER=RabbitMQ ++ awk -F= '{print $1}'
KEY=CELERY_BROKER ++ echo CELERY_BROKER=RabbitMQ ++ awk -F= '{print $2}'
VAL=RabbitMQ
'[' CELERY_BROKER=RabbitMQ '!=' CELERY_BROKER=RabbitMQ ']'
'[' -n CELERY_BROKER=RabbitMQ ']'
eval CELERY_BROKER=RabbitMQ ++ CELERY_BROKER=RabbitMQ
read -r LINE ++ echo CELERY_BROKER_HOST= ++ awk -F: '{print $1}'
SECTION=CELERY_BROKER_HOST= ++ echo CELERY_BROKER_HOST= ++ sed -e 's|CELERY_BROKER_HOST=:||'
KV=CELERY_BROKER_HOST= ++ echo CELERY_BROKER_HOST= ++ awk -F= '{print $1}'
KEY=CELERY_BROKER_HOST ++ echo CELERY_BROKER_HOST= ++ awk -F= '{print $2}'
VAL=
'[' CELERY_BROKER_HOST= '!=' CELERY_BROKER_HOST= ']'
'[' -n CELERY_BROKER_HOST= ']'
eval CELERY_BROKER_HOST= ++ CELERY_BROKER_HOST=
read -r LINE ++ echo CELERY_BROKER_PASS= ++ awk -F: '{print $1}'
SECTION=CELERY_BROKER_PASS= ++ echo CELERY_BROKER_PASS= ++ sed -e 's|CELERY_BROKER_PASS=:||'
KV=CELERY_BROKER_PASS= ++ echo CELERY_BROKER_PASS= ++ awk -F= '{print $1}'
KEY=CELERY_BROKER_PASS ++ echo CELERY_BROKER_PASS= ++ awk -F= '{print $2}'
VAL=
'[' CELERY_BROKER_PASS= '!=' CELERY_BROKER_PASS= ']'
'[' -n CELERY_BROKER_PASS= ']'
eval CELERY_BROKER_PASS= ++ CELERY_BROKER_PASS=
read -r LINE ++ echo CELERY_BROKER_PORT= ++ awk -F: '{print $1}'
SECTION=CELERY_BROKER_PORT= ++ echo CELERY_BROKER_PORT= ++ sed -e 's|CELERY_BROKER_PORT=:||'
KV=CELERY_BROKER_PORT= ++ awk -F= '{print $1}' ++ echo CELERY_BROKER_PORT=
KEY=CELERY_BROKER_PORT ++ awk -F= '{print $2}' ++ echo CELERY_BROKER_PORT=
VAL=
'[' CELERY_BROKER_PORT= '!=' CELERY_BROKER_PORT= ']'
'[' -n CELERY_BROKER_PORT= ']'
eval CELERY_BROKER_PORT= ++ CELERY_BROKER_PORT=
read -r LINE ++ echo CELERY_BROKER_USER= ++ awk -F: '{print $1}'
SECTION=CELERY_BROKER_USER= ++ echo CELERY_BROKER_USER= ++ sed -e 's|CELERY_BROKER_USER=:||'
KV=CELERY_BROKER_USER= ++ echo CELERY_BROKER_USER= ++ awk -F= '{print $1}'
KEY=CELERY_BROKER_USER ++ echo CELERY_BROKER_USER= ++ awk -F= '{print $2}'
VAL=
'[' CELERY_BROKER_USER= '!=' CELERY_BROKER_USER= ']'
'[' -n CELERY_BROKER_USER= ']'
eval CELERY_BROKER_USER= ++ CELERY_BROKER_USER=
read -r LINE ++ echo DB_HOST=usadc-vsedgdh03 ++ awk -F: '{print $1}'
SECTION=DB_HOST=usadc-vsedgdh03 ++ echo DB_HOST=usadc-vsedgdh03 ++ sed -e 's|DB_HOST=usadc-vsedgdh03:||'
KV=DB_HOST=usadc-vsedgdh03 ++ echo DB_HOST=usadc-vsedgdh03 ++ awk -F= '{print $1}'
KEY=DB_HOST ++ echo DB_HOST=usadc-vsedgdh03 ++ awk -F= '{print $2}'
VAL=usadc-vsedgdh03
'[' DB_HOST=usadc-vsedgdh03 '!=' DB_HOST=usadc-vsedgdh03 ']'
'[' -n DB_HOST=usadc-vsedgdh03 ']'
eval DB_HOST=usadc-vsedgdh03 ++ DB_HOST=usadc-vsedgdh03
read -r LINE ++ echo DB_NAME=airflow ++ awk -F: '{print $1}'
SECTION=DB_NAME=airflow ++ echo DB_NAME=airflow ++ sed -e 's|DB_NAME=airflow:||'
KV=DB_NAME=airflow ++ echo DB_NAME=airflow ++ awk -F= '{print $1}'
KEY=DB_NAME ++ echo DB_NAME=airflow ++ awk -F= '{print $2}'
VAL=airflow
'[' DB_NAME=airflow '!=' DB_NAME=airflow ']'
'[' -n DB_NAME=airflow ']'
eval DB_NAME=airflow ++ DB_NAME=airflow
read -r LINE ++ echo DB_PASS=p@ssw0rd123 ++ awk -F: '{print $1}'
SECTION=DB_PASS=p@ssw0rd123 ++ echo DB_PASS=p@ssw0rd123 ++ sed -e 's|DB_PASS=p@ssw0rd123:||'
KV=DB_PASS=p@ssw0rd123 ++ echo DB_PASS=p@ssw0rd123 ++ awk -F= '{print $1}'
KEY=DB_PASS ++ echo DB_PASS=p@ssw0rd123 ++ awk -F= '{print $2}'
VAL=p@ssw0rd123
'[' DB_PASS=p@ssw0rd123 '!=' DB_PASS=p@ssw0rd123 ']'
'[' -n DB_PASS=p@ssw0rd123 ']'
eval DB_PASS=p@ssw0rd123 ++ DB_PASS=p@ssw0rd123
read -r LINE ++ echo DB_PORT=3306 ++ awk -F: '{print $1}'
SECTION=DB_PORT=3306 ++ sed -e 's|DB_PORT=3306:||' ++ echo DB_PORT=3306
KV=DB_PORT=3306 ++ echo DB_PORT=3306 ++ awk -F= '{print $1}'
KEY=DB_PORT ++ echo DB_PORT=3306 ++ awk -F= '{print $2}'
VAL=3306
'[' DB_PORT=3306 '!=' DB_PORT=3306 ']'
'[' -n DB_PORT=3306 ']'
eval DB_PORT=3306 ++ DB_PORT=3306
read -r LINE ++ echo DB_TYPE=MySQL ++ awk -F: '{print $1}'
SECTION=DB_TYPE=MySQL ++ echo DB_TYPE=MySQL ++ sed -e 's|DB_TYPE=MySQL:||'
KV=DB_TYPE=MySQL ++ echo DB_TYPE=MySQL ++ awk -F= '{print $1}'
KEY=DB_TYPE ++ echo DB_TYPE=MySQL ++ awk -F= '{print $2}'
VAL=MySQL
'[' DB_TYPE=MySQL '!=' DB_TYPE=MySQL ']'
'[' -n DB_TYPE=MySQL ']'
eval DB_TYPE=MySQL ++ DB_TYPE=MySQL
read -r LINE ++ echo DB_USER=airflow ++ awk -F: '{print $1}'
SECTION=DB_USER=airflow ++ echo DB_USER=airflow ++ sed -e 's|DB_USER=airflow:||'
KV=DB_USER=airflow ++ echo DB_USER=airflow ++ awk -F= '{print $1}'
KEY=DB_USER ++ echo DB_USER=airflow ++ awk -F= '{print $2}'
VAL=airflow
'[' DB_USER=airflow '!=' DB_USER=airflow ']'
'[' -n DB_USER=airflow ']'
eval DB_USER=airflow ++ DB_USER=airflow
read -r LINE ++ echo api:auth_backend=airflow.api.auth.backend.default ++ awk -F: '{print $1}'
SECTION=api ++ echo api:auth_backend=airflow.api.auth.backend.default ++ sed -e 's|api:||'
KV=auth_backend=airflow.api.auth.backend.default ++ echo auth_backend=airflow.api.auth.backend.default ++ awk -F= '{print $1}'
KEY=auth_backend ++ echo auth_backend=airflow.api.auth.backend.default ++ awk -F= '{print $2}'
VAL=airflow.api.auth.backend.default
'[' api '!=' auth_backend=airflow.api.auth.backend.default ']'
'[' airflow.api.auth.backend.default == true ']'
'[' airflow.api.auth.backend.default == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg api auth_backend airflow.api.auth.backend.default
read -r LINE ++ echo celery:default_queue=default ++ awk -F: '{print $1}'
SECTION=celery ++ echo celery:default_queue=default ++ sed -e 's|celery:||'
KV=default_queue=default ++ echo default_queue=default ++ awk -F= '{print $1}'
KEY=default_queue ++ echo default_queue=default ++ awk -F= '{print $2}'
VAL=default
'[' celery '!=' default_queue=default ']'
'[' default == true ']'
'[' default == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg celery default_queue default
read -r LINE ++ echo celery:worker_concurrency=16 ++ awk -F: '{print $1}'
SECTION=celery ++ echo celery:worker_concurrency=16 ++ sed -e 's|celery:||'
KV=worker_concurrency=16 ++ echo worker_concurrency=16 ++ awk -F= '{print $1}'
KEY=worker_concurrency ++ echo worker_concurrency=16 ++ awk -F= '{print $2}'
VAL=16
'[' celery '!=' worker_concurrency=16 ']'
'[' 16 == true ']'
'[' 16 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg celery worker_concurrency 16
read -r LINE ++ echo celery:worker_log_server_port=8793 ++ awk -F: '{print $1}'
SECTION=celery ++ echo celery:worker_log_server_port=8793 ++ sed -e 's|celery:||'
KV=worker_log_server_port=8793 ++ echo worker_log_server_port=8793 ++ awk -F= '{print $1}'
KEY=worker_log_server_port ++ echo worker_log_server_port=8793 ++ awk -F= '{print $2}'
VAL=8793
'[' celery '!=' worker_log_server_port=8793 ']'
'[' 8793 == true ']'
'[' 8793 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg celery worker_log_server_port 8793
read -r LINE ++ echo cli:api_client=airflow.api.client.local_client ++ awk -F: '{print $1}'
SECTION=cli ++ sed -e 's|cli:||' ++ echo cli:api_client=airflow.api.client.local_client
KV=api_client=airflow.api.client.local_client ++ echo api_client=airflow.api.client.local_client ++ awk -F= '{print $1}'
KEY=api_client ++ echo api_client=airflow.api.client.local_client ++ awk -F= '{print $2}'
VAL=airflow.api.client.local_client
'[' cli '!=' api_client=airflow.api.client.local_client ']'
'[' airflow.api.client.local_client == true ']'
'[' airflow.api.client.local_client == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg cli api_client airflow.api.client.local_client
read -r LINE ++ echo cli:endpoint_url=http://localhost:8080 ++ awk -F: '{print $1}'
SECTION=cli ++ echo cli:endpoint_url=http://localhost:8080 ++ sed -e 's|cli:||'
KV=endpoint_url=http://localhost:8080 ++ awk -F= '{print $1}' ++ echo endpoint_url=http://localhost:8080
KEY=endpoint_url ++ echo endpoint_url=http://localhost:8080 ++ awk -F= '{print $2}'
VAL=http://localhost:8080
'[' cli '!=' endpoint_url=http://localhost:8080 ']'
'[' http://localhost:8080 == true ']'
'[' http://localhost:8080 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg cli endpoint_url http://localhost:8080
read -r LINE ++ echo core:base_log_folder=/var/log/airflow ++ awk -F: '{print $1}'
SECTION=core ++ echo core:base_log_folder=/var/log/airflow ++ sed -e 's|core:||'
KV=base_log_folder=/var/log/airflow ++ echo base_log_folder=/var/log/airflow ++ awk -F= '{print $1}'
KEY=base_log_folder ++ echo base_log_folder=/var/log/airflow ++ awk -F= '{print $2}'
VAL=/var/log/airflow
'[' core '!=' base_log_folder=/var/log/airflow ']'
'[' /var/log/airflow == true ']'
'[' /var/log/airflow == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core base_log_folder /var/log/airflow
read -r LINE ++ echo core:dag_concurrency=4 ++ awk -F: '{print $1}'
SECTION=core ++ echo core:dag_concurrency=4 ++ sed -e 's|core:||'
KV=dag_concurrency=4 ++ awk -F= '{print $1}' ++ echo dag_concurrency=4
KEY=dag_concurrency ++ echo dag_concurrency=4 ++ awk -F= '{print $2}'
VAL=4
'[' core '!=' dag_concurrency=4 ']'
'[' 4 == true ']'
'[' 4 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core dag_concurrency 4
read -r LINE ++ echo core:dagbag_import_timeout=30 ++ awk -F: '{print $1}'
SECTION=core ++ echo core:dagbag_import_timeout=30 ++ sed -e 's|core:||'
KV=dagbag_import_timeout=30 ++ echo dagbag_import_timeout=30 ++ awk -F= '{print $1}'
KEY=dagbag_import_timeout ++ echo dagbag_import_timeout=30 ++ awk -F= '{print $2}'
VAL=30
'[' core '!=' dagbag_import_timeout=30 ']'
'[' 30 == true ']'
'[' 30 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core dagbag_import_timeout 30
read -r LINE ++ echo core:dags_are_paused_at_creation=true ++ awk -F: '{print $1}'
SECTION=core ++ sed -e 's|core:||' ++ echo core:dags_are_paused_at_creation=true
KV=dags_are_paused_at_creation=true ++ echo dags_are_paused_at_creation=true ++ awk -F= '{print $1}'
KEY=dags_are_paused_at_creation ++ awk -F= '{print $2}' ++ echo dags_are_paused_at_creation=true
VAL=true
'[' core '!=' dags_are_paused_at_creation=true ']'
'[' true == true ']'
VAL=True
'[' True == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core dags_are_paused_at_creation True
read -r LINE ++ echo core:dags_folder=/var/lib/airflow/dags ++ awk -F: '{print $1}'
SECTION=core ++ echo core:dags_folder=/var/lib/airflow/dags ++ sed -e 's|core:||'
KV=dags_folder=/var/lib/airflow/dags ++ echo dags_folder=/var/lib/airflow/dags ++ awk -F= '{print $1}'
KEY=dags_folder ++ echo dags_folder=/var/lib/airflow/dags ++ awk -F= '{print $2}'
VAL=/var/lib/airflow/dags
'[' core '!=' dags_folder=/var/lib/airflow/dags ']'
'[' /var/lib/airflow/dags == true ']'
'[' /var/lib/airflow/dags == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core dags_folder /var/lib/airflow/dags
read -r LINE ++ echo core:default_impersonation= ++ awk -F: '{print $1}'
SECTION=core ++ echo core:default_impersonation= ++ sed -e 's|core:||'
KV=default_impersonation= ++ awk -F= '{print $1}' ++ echo default_impersonation=
KEY=default_impersonation ++ awk -F= '{print $2}' ++ echo default_impersonation=
VAL=
'[' core '!=' default_impersonation= ']'
'[' '' == true ']'
'[' '' == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core default_impersonation ''
read -r LINE ++ awk -F: '{print $1}' ++ echo core:donot_pickle=false
SECTION=core ++ echo core:donot_pickle=false ++ sed -e 's|core:||'
KV=donot_pickle=false ++ echo donot_pickle=false ++ awk -F= '{print $1}'
KEY=donot_pickle ++ echo donot_pickle=false ++ awk -F= '{print $2}'
VAL=false
'[' core '!=' donot_pickle=false ']'
'[' false == true ']'
'[' false == false ']'
VAL=False
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core donot_pickle False
read -r LINE ++ echo core:executor=LocalExecutor ++ awk -F: '{print $1}'
SECTION=core ++ echo core:executor=LocalExecutor ++ sed -e 's|core:||'
KV=executor=LocalExecutor ++ echo executor=LocalExecutor ++ awk -F= '{print $1}'
KEY=executor ++ echo executor=LocalExecutor ++ awk -F= '{print $2}'
VAL=LocalExecutor
'[' core '!=' executor=LocalExecutor ']'
'[' LocalExecutor == true ']'
'[' LocalExecutor == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core executor LocalExecutor
read -r LINE ++ awk -F: '{print $1}' ++ echo core:fernet_key=
SECTION=core ++ echo core:fernet_key= ++ sed -e 's|core:||'
KV=fernet_key= ++ echo fernet_key= ++ awk -F= '{print $1}'
KEY=fernet_key ++ awk -F= '{print $2}' ++ echo fernet_key=
VAL=
'[' core '!=' fernet_key= ']'
'[' '' == true ']'
'[' '' == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core fernet_key ''
read -r LINE ++ echo core:load_examples=false ++ awk -F: '{print $1}'
SECTION=core ++ echo core:load_examples=false ++ sed -e 's|core:||'
KV=load_examples=false ++ echo load_examples=false ++ awk -F= '{print $1}'
KEY=load_examples ++ echo load_examples=false ++ awk -F= '{print $2}'
VAL=false
'[' core '!=' load_examples=false ']'
'[' false == true ']'
'[' false == false ']'
VAL=False
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core load_examples False
read -r LINE ++ awk -F: '{print $1}' ++ echo core:max_active_runs_per_dag=16
SECTION=core ++ echo core:max_active_runs_per_dag=16 ++ sed -e 's|core:||'
KV=max_active_runs_per_dag=16 ++ echo max_active_runs_per_dag=16 ++ awk -F= '{print $1}'
KEY=max_active_runs_per_dag ++ awk -F= '{print $2}' ++ echo max_active_runs_per_dag=16
VAL=16
'[' core '!=' max_active_runs_per_dag=16 ']'
'[' 16 == true ']'
'[' 16 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core max_active_runs_per_dag 16
read -r LINE ++ echo core:non_pooled_task_slot_count=128 ++ awk -F: '{print $1}'
SECTION=core ++ echo core:non_pooled_task_slot_count=128 ++ sed -e 's|core:||'
KV=non_pooled_task_slot_count=128 ++ echo non_pooled_task_slot_count=128 ++ awk -F= '{print $1}'
KEY=non_pooled_task_slot_count ++ echo non_pooled_task_slot_count=128 ++ awk -F= '{print $2}'
VAL=128
'[' core '!=' non_pooled_task_slot_count=128 ']'
'[' 128 == true ']'
'[' 128 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core non_pooled_task_slot_count 128
read -r LINE ++ awk -F: '{print $1}' ++ echo core:parallelism=8
SECTION=core ++ echo core:parallelism=8 ++ sed -e 's|core:||'
KV=parallelism=8 ++ awk -F= '{print $1}' ++ echo parallelism=8
KEY=parallelism ++ echo parallelism=8 ++ awk -F= '{print $2}'
VAL=8
'[' core '!=' parallelism=8 ']'
'[' 8 == true ']'
'[' 8 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core parallelism 8
read -r LINE ++ echo core:plugins_folder=/var/lib/airflow/plugins ++ awk -F: '{print $1}'
SECTION=core ++ echo core:plugins_folder=/var/lib/airflow/plugins ++ sed -e 's|core:||'
KV=plugins_folder=/var/lib/airflow/plugins ++ awk -F= '{print $1}' ++ echo plugins_folder=/var/lib/airflow/plugins
KEY=plugins_folder ++ echo plugins_folder=/var/lib/airflow/plugins ++ awk -F= '{print $2}'
VAL=/var/lib/airflow/plugins
'[' core '!=' plugins_folder=/var/lib/airflow/plugins ']'
'[' /var/lib/airflow/plugins == true ']'
'[' /var/lib/airflow/plugins == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core plugins_folder /var/lib/airflow/plugins
read -r LINE ++ echo core:sql_alchemy_pool_enabled=true ++ awk -F: '{print $1}'
SECTION=core ++ sed -e 's|core:||' ++ echo core:sql_alchemy_pool_enabled=true
KV=sql_alchemy_pool_enabled=true ++ echo sql_alchemy_pool_enabled=true ++ awk -F= '{print $1}'
KEY=sql_alchemy_pool_enabled ++ echo sql_alchemy_pool_enabled=true ++ awk -F= '{print $2}'
VAL=true
'[' core '!=' sql_alchemy_pool_enabled=true ']'
'[' true == true ']'
VAL=True
'[' True == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core sql_alchemy_pool_enabled True
read -r LINE ++ echo core:sql_alchemy_pool_recycle=1800 ++ awk -F: '{print $1}'
SECTION=core ++ echo core:sql_alchemy_pool_recycle=1800 ++ sed -e 's|core:||'
KV=sql_alchemy_pool_recycle=1800 ++ awk -F= '{print $1}' ++ echo sql_alchemy_pool_recycle=1800
KEY=sql_alchemy_pool_recycle ++ echo sql_alchemy_pool_recycle=1800 ++ awk -F= '{print $2}'
VAL=1800
'[' core '!=' sql_alchemy_pool_recycle=1800 ']'
'[' 1800 == true ']'
'[' 1800 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core sql_alchemy_pool_recycle 1800
read -r LINE ++ echo core:sql_alchemy_pool_size=5 ++ awk -F: '{print $1}'
SECTION=core ++ echo core:sql_alchemy_pool_size=5 ++ sed -e 's|core:||'
KV=sql_alchemy_pool_size=5 ++ echo sql_alchemy_pool_size=5 ++ awk -F= '{print $1}'
KEY=sql_alchemy_pool_size ++ echo sql_alchemy_pool_size=5 ++ awk -F= '{print $2}'
VAL=5
'[' core '!=' sql_alchemy_pool_size=5 ']'
'[' 5 == true ']'
'[' 5 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core sql_alchemy_pool_size 5
read -r LINE ++ echo core:sql_alchemy_reconnect_timeout=300 ++ awk -F: '{print $1}'
SECTION=core ++ sed -e 's|core:||' ++ echo core:sql_alchemy_reconnect_timeout=300
KV=sql_alchemy_reconnect_timeout=300 ++ echo sql_alchemy_reconnect_timeout=300 ++ awk -F= '{print $1}'
KEY=sql_alchemy_reconnect_timeout ++ echo sql_alchemy_reconnect_timeout=300 ++ awk -F= '{print $2}'
VAL=300
'[' core '!=' sql_alchemy_reconnect_timeout=300 ']'
'[' 300 == true ']'
'[' 300 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core sql_alchemy_reconnect_timeout 300
read -r LINE ++ awk -F: '{print $1}' ++ echo core:task_runner=BashTaskRunner
SECTION=core ++ echo core:task_runner=BashTaskRunner ++ sed -e 's|core:||'
KV=task_runner=BashTaskRunner ++ echo task_runner=BashTaskRunner ++ awk -F= '{print $1}'
KEY=task_runner ++ echo task_runner=BashTaskRunner ++ awk -F= '{print $2}'
VAL=BashTaskRunner
'[' core '!=' task_runner=BashTaskRunner ']'
'[' BashTaskRunner == true ']'
'[' BashTaskRunner == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core task_runner BashTaskRunner
read -r LINE ++ awk -F: '{print $1}' ++ echo hive:default_hive_mapred_queue=
SECTION=hive ++ echo hive:default_hive_mapred_queue= ++ sed -e 's|hive:||'
KV=default_hive_mapred_queue= ++ echo default_hive_mapred_queue= ++ awk -F= '{print $1}'
KEY=default_hive_mapred_queue ++ awk -F= '{print $2}' ++ echo default_hive_mapred_queue=
VAL=
'[' hive '!=' default_hive_mapred_queue= ']'
'[' '' == true ']'
'[' '' == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg hive default_hive_mapred_queue ''
read -r LINE ++ awk -F: '{print $1}' ++ echo log_dir=/var/log/airflow
SECTION=log_dir=/var/log/airflow ++ echo log_dir=/var/log/airflow ++ sed -e 's|log_dir=/var/log/airflow:||'
KV=log_dir=/var/log/airflow ++ echo log_dir=/var/log/airflow ++ awk -F= '{print $1}'
KEY=log_dir ++ awk -F= '{print $2}' ++ echo log_dir=/var/log/airflow
VAL=/var/log/airflow
'[' log_dir=/var/log/airflow '!=' log_dir=/var/log/airflow ']'
'[' -n log_dir=/var/log/airflow ']'
eval log_dir=/var/log/airflow ++ log_dir=/var/log/airflow
read -r LINE ++ echo operators:default_cpus=1 ++ awk -F: '{print $1}'
SECTION=operators ++ echo operators:default_cpus=1 ++ sed -e 's|operators:||'
KV=default_cpus=1 ++ awk -F= '{print $1}' ++ echo default_cpus=1
KEY=default_cpus ++ echo default_cpus=1 ++ awk -F= '{print $2}'
VAL=1
'[' operators '!=' default_cpus=1 ']'
'[' 1 == true ']'
'[' 1 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg operators default_cpus 1
read -r LINE ++ awk -F: '{print $1}' ++ echo operators:default_disk=512
SECTION=operators ++ echo operators:default_disk=512 ++ sed -e 's|operators:||'
KV=default_disk=512 ++ awk -F= '{print $1}' ++ echo default_disk=512
KEY=default_disk ++ echo default_disk=512 ++ awk -F= '{print $2}'
VAL=512
'[' operators '!=' default_disk=512 ']'
'[' 512 == true ']'
'[' 512 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg operators default_disk 512
read -r LINE ++ awk -F: '{print $1}' ++ echo operators:default_gpus=0
SECTION=operators ++ sed -e 's|operators:||' ++ echo operators:default_gpus=0
KV=default_gpus=0 ++ awk -F= '{print $1}' ++ echo default_gpus=0
KEY=default_gpus ++ awk -F= '{print $2}' ++ echo default_gpus=0
VAL=0
'[' operators '!=' default_gpus=0 ']'
'[' 0 == true ']'
'[' 0 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg operators default_gpus 0
read -r LINE ++ echo operators:default_owner=Airflow ++ awk -F: '{print $1}'
SECTION=operators ++ sed -e 's|operators:||' ++ echo operators:default_owner=Airflow
KV=default_owner=Airflow ++ echo default_owner=Airflow ++ awk -F= '{print $1}'
KEY=default_owner ++ awk -F= '{print $2}' ++ echo default_owner=Airflow
VAL=Airflow
'[' operators '!=' default_owner=Airflow ']'
'[' Airflow == true ']'
'[' Airflow == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg operators default_owner Airflow
read -r LINE ++ echo operators:default_ram=512 ++ awk -F: '{print $1}'
SECTION=operators ++ echo operators:default_ram=512 ++ sed -e 's|operators:||'
KV=default_ram=512 ++ echo default_ram=512 ++ awk -F= '{print $1}'
KEY=default_ram ++ echo default_ram=512 ++ awk -F= '{print $2}'
VAL=512
'[' operators '!=' default_ram=512 ']'
'[' 512 == true ']'
'[' 512 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg operators default_ram 512
read -r LINE ++ echo smtp:smtp_host=localhost ++ awk -F: '{print $1}'
SECTION=smtp ++ sed -e 's|smtp:||' ++ echo smtp:smtp_host=localhost
KV=smtp_host=localhost ++ awk -F= '{print $1}' ++ echo smtp_host=localhost
KEY=smtp_host ++ awk -F= '{print $2}' ++ echo smtp_host=localhost
VAL=localhost
'[' smtp '!=' smtp_host=localhost ']'
'[' localhost == true ']'
'[' localhost == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg smtp smtp_host localhost
read -r LINE ++ echo smtp:smtp_mail_from=airflow@localhost ++ awk -F: '{print $1}'
SECTION=smtp ++ sed -e 's|smtp:||' ++ echo smtp:smtp_mail_from=airflow@localhost
KV=smtp_mail_from=airflow@localhost ++ echo smtp_mail_from=airflow@localhost ++ awk -F= '{print $1}'
KEY=smtp_mail_from ++ echo smtp_mail_from=airflow@localhost ++ awk -F= '{print $2}'
VAL=airflow@localhost
'[' smtp '!=' smtp_mail_from=airflow@localhost ']'
'[' airflow@localhost == true ']'
'[' airflow@localhost == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg smtp smtp_mail_from airflow@localhost
read -r LINE ++ echo smtp:smtp_password= ++ awk -F: '{print $1}'
SECTION=smtp ++ echo smtp:smtp_password= ++ sed -e 's|smtp:||'
KV=smtp_password= ++ echo smtp_password= ++ awk -F= '{print $1}'
KEY=smtp_password ++ echo smtp_password= ++ awk -F= '{print $2}'
VAL=
'[' smtp '!=' smtp_password= ']'
'[' '' == true ']'
'[' '' == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg smtp smtp_password ''
read -r LINE ++ awk -F: '{print $1}' ++ echo smtp:smtp_port=25
SECTION=smtp ++ echo smtp:smtp_port=25 ++ sed -e 's|smtp:||'
KV=smtp_port=25 ++ awk -F= '{print $1}' ++ echo smtp_port=25
KEY=smtp_port ++ awk -F= '{print $2}' ++ echo smtp_port=25
VAL=25
'[' smtp '!=' smtp_port=25 ']'
'[' 25 == true ']'
'[' 25 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg smtp smtp_port 25
read -r LINE ++ echo smtp:smtp_ssl=false ++ awk -F: '{print $1}'
SECTION=smtp ++ echo smtp:smtp_ssl=false ++ sed -e 's|smtp:||'
KV=smtp_ssl=false ++ echo smtp_ssl=false ++ awk -F= '{print $1}'
KEY=smtp_ssl ++ echo smtp_ssl=false ++ awk -F= '{print $2}'
VAL=false
'[' smtp '!=' smtp_ssl=false ']'
'[' false == true ']'
'[' false == false ']'
VAL=False
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg smtp smtp_ssl False
read -r LINE ++ echo smtp:smtp_starttls=true ++ awk -F: '{print $1}'
SECTION=smtp ++ echo smtp:smtp_starttls=true ++ sed -e 's|smtp:||'
KV=smtp_starttls=true ++ awk -F= '{print $1}' ++ echo smtp_starttls=true
KEY=smtp_starttls ++ echo smtp_starttls=true ++ awk -F= '{print $2}'
VAL=true
'[' smtp '!=' smtp_starttls=true ']'
'[' true == true ']'
VAL=True
'[' True == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg smtp smtp_starttls True
read -r LINE ++ echo smtp:smtp_user= ++ awk -F: '{print $1}'
SECTION=smtp ++ sed -e 's|smtp:||' ++ echo smtp:smtp_user=
KV=smtp_user= ++ echo smtp_user= ++ awk -F= '{print $1}'
KEY=smtp_user ++ echo smtp_user= ++ awk -F= '{print $2}'
VAL=
'[' smtp '!=' smtp_user= ']'
'[' '' == true ']'
'[' '' == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg smtp smtp_user ''
read -r LINE
'[' MySQL == SQLite3 ']'
'[' MySQL == MySQL ']'
VAL=mysql://airflow:p@ssw0rd123@usadc-vsedgdh03:3306/airflow
'[' -n mysql://airflow:p@ssw0rd123@usadc-vsedgdh03:3306/airflow ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core sql_alchemy_conn mysql://airflow:p@ssw0rd123@usadc-vsedgdh03:3306/airflow
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg celery result_backend db+mysql://airflow:p@ssw0rd123@usadc-vsedgdh03:3306/airflow
'[' RabbitMQ == RabbitMQ ']'
VAL=amqp://:@:/
'[' -n amqp://:@:/ ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg celery broker_url amqp://:@:/
echo -e '\nexport AIRFLOW_HOME=/var/lib/airflow'
echo -e '\nexport AIRFLOW_CONFIG=/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg'
chgrp airflow /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow-env.sh
log 'Starting Airflow worker...' ++ date '+%Y-%m-%d %H:%M:%S'
timestamp='2020-07-13 00:12:59'
echo '2020-07-13 00:12:59: Starting Airflow worker...'
su -s /bin/bash - airflow -c 'CONF_DIR=/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER exec /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/bin/airflow-cm.sh worker ' /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/airflow/configuration.py:214: FutureWarning: The task_runner setting in [core] has the old default value of u'BashTaskRunner'. This value has been changed to u'StandardTaskRunner' in the running config, but please update your config before Apache Airflow 2.0. FutureWarning, Traceback (most recent call last): File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/bin/.airflow", line 32, in args.func(args) File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/airflow/utils/cli.py", line 74, in wrapper return f(*args, kwargs) File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/airflow/bin/cli.py", line 1090, in worker worker.run(options) File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/celery/bin/worker.py", line 257, in run kwargs) File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/celery/worker/worker.py", line 101, in init self.setup_instance(self.prepare_args(kwargs)) File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/celery/worker/worker.py", line 122, in setup_instance self._conninfo = self.app.connection_for_read() File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/celery/app/base.py", line 743, in connection_for_read return self._connection(url or self.conf.broker_read_url, kwargs) File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/celery/app/base.py", line 819, in _connection 'broker_connection_timeout', connect_timeout File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/kombu/connection.py", line 189, in init url_params = parse_url(hostname) File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/kombu/utils/url.py", line 47, in parse_url scheme, host, port, user, password, path, query = _parse_url(url) File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/kombu/utils/url.py", line 81, in url_to_parts parts.port, File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/future/backports/urllib/parse.py", line 162, in port port = int(port, 10) File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/future/types/newint.py", line 76, in new return super(newint, cls).new(cls, val) ValueError: invalid literal for long() with base 10: '' [13/Jul/2020 00:13:04 +0000] 121677 MainThread redactor INFO Started launcher: /opt/cloudera/cm-agent/service/csd/csd.sh start_worker [13/Jul/2020 00:13:04 +0000] 121677 MainThread redactor ERROR Redaction rules file doesn't exist, not redacting logs. file: redaction-rules.json, directory: /run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER [13/Jul/2020 00:13:04 +0000] 121677 MainThread redactor INFO Re-exec watcher: /opt/cloudera/cm-agent/bin/cm proc_watcher 121689 Mon Jul 13 00:13:04 EDT 2020
locate_java_home
locate_java_home_no_verify
JAVA11_HOME_CANDIDATES=('/usr/java/jdk-11' '/usr/lib/jvm/jdk-11' '/usr/lib/jvm/java-11-oracle')
local JAVA11_HOME_CANDIDATES
OPENJAVA11_HOME_CANDIDATES=('/usr/lib/jvm/jdk-11' '/usr/lib64/jvm/jdk-11')
local OPENJAVA11_HOME_CANDIDATES
JAVA8_HOME_CANDIDATES=('/usr/java/jdk1.8' '/usr/java/jre1.8' '/usr/lib/jvm/j2sdk1.8-oracle' '/usr/lib/jvm/j2sdk1.8-oracle/jre' '/usr/lib/jvm/java-8-oracle')
local JAVA8_HOME_CANDIDATES
OPENJAVA8_HOME_CANDIDATES=('/usr/lib/jvm/java-1.8.0-openjdk' '/usr/lib/jvm/java-8-openjdk' '/usr/lib64/jvm/java-1.8.0-openjdk' '/usr/lib64/jvm/java-8-openjdk')
local OPENJAVA8_HOME_CANDIDATES
MISCJAVA_HOME_CANDIDATES=('/Library/Java/Home' '/usr/java/default' '/usr/lib/jvm/default-java' '/usr/lib/jvm/java-openjdk' '/usr/lib/jvm/jre-openjdk')
local MISCJAVA_HOME_CANDIDATES
case ${BIGTOP_JAVA_MAJOR} in
JAVA_HOME_CANDIDATES=(${JAVA8_HOME_CANDIDATES[@]} ${MISCJAVA_HOME_CANDIDATES[@]} ${OPENJAVA8_HOME_CANDIDATES[@]} ${JAVA11_HOME_CANDIDATES[@]} ${OPENJAVA11_HOME_CANDIDATES[@]})
'[' -z /usr/lib/jvm/jre-openjdk ']'
verify_java_home
'[' -z /usr/lib/jvm/jre-openjdk ']'
echo JAVA_HOME=/usr/lib/jvm/jre-openjdk
'[' -n '' ']'
source_parcel_environment
'[' '!' -z /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/meta/airflow_env.sh ']'
OLD_IFS='
'
IFS=:
SCRIPT_ARRAY=($SCM_DEFINES_SCRIPTS)
DIRNAME_ARRAY=($PARCEL_DIRNAMES)
IFS='
'
COUNT=1 ++ seq 1 1
for i in 'seq 1 $COUNT'
SCRIPT=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/meta/airflow_env.sh
PARCEL_DIRNAME=AIRFLOW-1.10.3-python2.7.15_b2
. /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/meta/airflow_env.sh ++ AIRFLOW_DIRNAME=AIRFLOW-1.10.3-python2.7.15_b2 ++ export AIRFLOW_DIR=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2 ++ AIRFLOW_DIR=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2 ++ export PATH=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin ++ PATH=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin ++ export PYTHONHOME=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2 ++ PYTHONHOME=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2 ++ export PYTHONPATH=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7 ++ PYTHONPATH=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7 ++ '[' -z '' ']' ++ export AIRFLOW_PYTHON=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/bin/python2.7 ++ AIRFLOW_PYTHON=/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/bin/python2.7
echo 'Using /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER as conf dir'
echo 'Using scripts/control.sh as process script'
replace_conf_dir
echo CONF_DIR=/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER
echo CMF_CONF_DIR=
EXCLUDE_CMF_FILES=('cloudera-config.sh' 'hue.sh' 'impala.sh' 'sqoop.sh' 'supervisor.conf' 'config.zip' 'proc.json' '.log' '.keytab' 'jceks') ++ printf '! -name %s ' cloudera-config.sh hue.sh impala.sh sqoop.sh supervisor.conf config.zip proc.json '.log' '.keytab' 'jceks'
find /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER -type f '!' -path '/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/logs/' '!' -name cloudera-config.sh '!' -name hue.sh '!' -name impala.sh '!' -name sqoop.sh '!' -name supervisor.conf '!' -name config.zip '!' -name proc.json '!' -name '.log' '!' -name '.keytab' '!' -name 'jceks' -exec perl -pi -e 's#{{CMF_CONF_DIR}}#/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER#g' '{}' ';'
make_scripts_executable
find /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER -regex '.*.(py|sh)$' -exec chmod u+x '{}' ';'
RUN_DIR=/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER
'[' '' == true ']'
chmod u+x /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/scripts/control.sh
export COMMON_SCRIPT=/opt/cloudera/cm-agent/service/common/cloudera-config.sh
COMMON_SCRIPT=/opt/cloudera/cm-agent/service/common/cloudera-config.sh
exec /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/scripts/control.sh start_worker
date Mon Jul 13 00:13:05 EDT 2020
CMD=start_worker
OPTS=
log '*** AIRFLOW_DIR: /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2' ++ date '+%Y-%m-%d %H:%M:%S'
timestamp='2020-07-13 00:13:05'
echo '2020-07-13 00:13:05: *** AIRFLOW_DIR: /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2'
log '*** AIRFLOW_HOME: ' ++ date '+%Y-%m-%d %H:%M:%S'
timestamp='2020-07-13 00:13:05'
echo '2020-07-13 00:13:05: *** AIRFLOW_HOME: '
log '*** AIRFLOW_CONFIG: ' ++ date '+%Y-%m-%d %H:%M:%S'
timestamp='2020-07-13 00:13:05'
echo '2020-07-13 00:13:05: *** AIRFLOW_CONFIG: '
log '*** PYTHONHOME: /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2' ++ date '+%Y-%m-%d %H:%M:%S'
timestamp='2020-07-13 00:13:05'
echo '2020-07-13 00:13:05: *** PYTHONHOME: /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2'
log '*** PYTHONPATH: /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7' ++ date '+%Y-%m-%d %H:%M:%S'
timestamp='2020-07-13 00:13:05'
echo '2020-07-13 00:13:05: *** PYTHONPATH: /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7'
case $CMD in
update_daemon_config /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER
local DIR=/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER
export AIRFLOW_CONFIG=/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg
AIRFLOW_CONFIG=/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg
log '** AIRFLOW_CONFIG: /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg' ++ date '+%Y-%m-%d %H:%M:%S'
timestamp='2020-07-13 00:13:05'
echo '2020-07-13 00:13:05: ** AIRFLOW_CONFIG: /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg'
deploy_client_config /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER
log 'Converting airflow.properties to airflow.cfg ...' ++ date '+%Y-%m-%d %H:%M:%S'
timestamp='2020-07-13 00:13:05'
echo '2020-07-13 00:13:05: Converting airflow.properties to airflow.cfg ...'
local DIR=/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER
local SECTION
local KV
local KEY
local VAL
read -r LINE ++ echo AIRFLOW_HOME=/var/lib/airflow ++ awk -F: '{print $1}'
SECTION=AIRFLOW_HOME=/var/lib/airflow ++ echo AIRFLOW_HOME=/var/lib/airflow ++ sed -e 's|AIRFLOW_HOME=/var/lib/airflow:||'
KV=AIRFLOW_HOME=/var/lib/airflow ++ awk -F= '{print $1}' ++ echo AIRFLOW_HOME=/var/lib/airflow
KEY=AIRFLOW_HOME ++ awk -F= '{print $2}' ++ echo AIRFLOW_HOME=/var/lib/airflow
VAL=/var/lib/airflow
'[' AIRFLOW_HOME=/var/lib/airflow '!=' AIRFLOW_HOME=/var/lib/airflow ']'
'[' -n AIRFLOW_HOME=/var/lib/airflow ']'
eval AIRFLOW_HOME=/var/lib/airflow ++ AIRFLOW_HOME=/var/lib/airflow
read -r LINE ++ awk -F: '{print $1}' ++ echo CELERY_BROKER=RabbitMQ
SECTION=CELERY_BROKER=RabbitMQ ++ echo CELERY_BROKER=RabbitMQ ++ sed -e 's|CELERY_BROKER=RabbitMQ:||'
KV=CELERY_BROKER=RabbitMQ ++ echo CELERY_BROKER=RabbitMQ ++ awk -F= '{print $1}'
KEY=CELERY_BROKER ++ awk -F= '{print $2}' ++ echo CELERY_BROKER=RabbitMQ
VAL=RabbitMQ
'[' CELERY_BROKER=RabbitMQ '!=' CELERY_BROKER=RabbitMQ ']'
'[' -n CELERY_BROKER=RabbitMQ ']'
eval CELERY_BROKER=RabbitMQ ++ CELERY_BROKER=RabbitMQ
read -r LINE ++ echo CELERY_BROKER_HOST= ++ awk -F: '{print $1}'
SECTION=CELERY_BROKER_HOST= ++ echo CELERY_BROKER_HOST= ++ sed -e 's|CELERY_BROKER_HOST=:||'
KV=CELERY_BROKER_HOST= ++ echo CELERY_BROKER_HOST= ++ awk -F= '{print $1}'
KEY=CELERY_BROKER_HOST ++ awk -F= '{print $2}' ++ echo CELERY_BROKER_HOST=
VAL=
'[' CELERY_BROKER_HOST= '!=' CELERY_BROKER_HOST= ']'
'[' -n CELERY_BROKER_HOST= ']'
eval CELERY_BROKER_HOST= ++ CELERY_BROKER_HOST=
read -r LINE ++ echo CELERY_BROKER_PASS= ++ awk -F: '{print $1}'
SECTION=CELERY_BROKER_PASS= ++ sed -e 's|CELERY_BROKER_PASS=:||' ++ echo CELERY_BROKER_PASS=
KV=CELERY_BROKER_PASS= ++ awk -F= '{print $1}' ++ echo CELERY_BROKER_PASS=
KEY=CELERY_BROKER_PASS ++ awk -F= '{print $2}' ++ echo CELERY_BROKER_PASS=
VAL=
'[' CELERY_BROKER_PASS= '!=' CELERY_BROKER_PASS= ']'
'[' -n CELERY_BROKER_PASS= ']'
eval CELERY_BROKER_PASS= ++ CELERY_BROKER_PASS=
read -r LINE ++ awk -F: '{print $1}' ++ echo CELERY_BROKER_PORT=
SECTION=CELERY_BROKER_PORT= ++ echo CELERY_BROKER_PORT= ++ sed -e 's|CELERY_BROKER_PORT=:||'
KV=CELERY_BROKER_PORT= ++ awk -F= '{print $1}' ++ echo CELERY_BROKER_PORT=
KEY=CELERY_BROKER_PORT ++ echo CELERY_BROKER_PORT= ++ awk -F= '{print $2}'
VAL=
'[' CELERY_BROKER_PORT= '!=' CELERY_BROKER_PORT= ']'
'[' -n CELERY_BROKER_PORT= ']'
eval CELERY_BROKER_PORT= ++ CELERY_BROKER_PORT=
read -r LINE ++ awk -F: '{print $1}' ++ echo CELERY_BROKER_USER=
SECTION=CELERY_BROKER_USER= ++ echo CELERY_BROKER_USER= ++ sed -e 's|CELERY_BROKER_USER=:||'
KV=CELERY_BROKER_USER= ++ echo CELERY_BROKER_USER= ++ awk -F= '{print $1}'
KEY=CELERY_BROKER_USER ++ awk -F= '{print $2}' ++ echo CELERY_BROKER_USER=
VAL=
'[' CELERY_BROKER_USER= '!=' CELERY_BROKER_USER= ']'
'[' -n CELERY_BROKER_USER= ']'
eval CELERY_BROKER_USER= ++ CELERY_BROKER_USER=
read -r LINE ++ awk -F: '{print $1}' ++ echo DB_HOST=usadc-vsedgdh03
SECTION=DB_HOST=usadc-vsedgdh03 ++ sed -e 's|DB_HOST=usadc-vsedgdh03:||' ++ echo DB_HOST=usadc-vsedgdh03
KV=DB_HOST=usadc-vsedgdh03 ++ echo DB_HOST=usadc-vsedgdh03 ++ awk -F= '{print $1}'
KEY=DB_HOST ++ echo DB_HOST=usadc-vsedgdh03 ++ awk -F= '{print $2}'
VAL=usadc-vsedgdh03
'[' DB_HOST=usadc-vsedgdh03 '!=' DB_HOST=usadc-vsedgdh03 ']'
'[' -n DB_HOST=usadc-vsedgdh03 ']'
eval DB_HOST=usadc-vsedgdh03 ++ DB_HOST=usadc-vsedgdh03
read -r LINE ++ echo DB_NAME=airflow ++ awk -F: '{print $1}'
SECTION=DB_NAME=airflow ++ sed -e 's|DB_NAME=airflow:||' ++ echo DB_NAME=airflow
KV=DB_NAME=airflow ++ echo DB_NAME=airflow ++ awk -F= '{print $1}'
KEY=DB_NAME ++ awk -F= '{print $2}' ++ echo DB_NAME=airflow
VAL=airflow
'[' DB_NAME=airflow '!=' DB_NAME=airflow ']'
'[' -n DB_NAME=airflow ']'
eval DB_NAME=airflow ++ DB_NAME=airflow
read -r LINE ++ awk -F: '{print $1}' ++ echo DB_PASS=p@ssw0rd123
SECTION=DB_PASS=p@ssw0rd123 ++ echo DB_PASS=p@ssw0rd123 ++ sed -e 's|DB_PASS=p@ssw0rd123:||'
KV=DB_PASS=p@ssw0rd123 ++ awk -F= '{print $1}' ++ echo DB_PASS=p@ssw0rd123
KEY=DB_PASS ++ awk -F= '{print $2}' ++ echo DB_PASS=p@ssw0rd123
VAL=p@ssw0rd123
'[' DB_PASS=p@ssw0rd123 '!=' DB_PASS=p@ssw0rd123 ']'
'[' -n DB_PASS=p@ssw0rd123 ']'
eval DB_PASS=p@ssw0rd123 ++ DB_PASS=p@ssw0rd123
read -r LINE ++ echo DB_PORT=3306 ++ awk -F: '{print $1}'
SECTION=DB_PORT=3306 ++ sed -e 's|DB_PORT=3306:||' ++ echo DB_PORT=3306
KV=DB_PORT=3306 ++ echo DB_PORT=3306 ++ awk -F= '{print $1}'
KEY=DB_PORT ++ echo DB_PORT=3306 ++ awk -F= '{print $2}'
VAL=3306
'[' DB_PORT=3306 '!=' DB_PORT=3306 ']'
'[' -n DB_PORT=3306 ']'
eval DB_PORT=3306 ++ DB_PORT=3306
read -r LINE ++ echo DB_TYPE=MySQL ++ awk -F: '{print $1}'
SECTION=DB_TYPE=MySQL ++ echo DB_TYPE=MySQL ++ sed -e 's|DB_TYPE=MySQL:||'
KV=DB_TYPE=MySQL ++ echo DB_TYPE=MySQL ++ awk -F= '{print $1}'
KEY=DB_TYPE ++ awk -F= '{print $2}' ++ echo DB_TYPE=MySQL
VAL=MySQL
'[' DB_TYPE=MySQL '!=' DB_TYPE=MySQL ']'
'[' -n DB_TYPE=MySQL ']'
eval DB_TYPE=MySQL ++ DB_TYPE=MySQL
read -r LINE ++ echo DB_USER=airflow ++ awk -F: '{print $1}'
SECTION=DB_USER=airflow ++ sed -e 's|DB_USER=airflow:||' ++ echo DB_USER=airflow
KV=DB_USER=airflow ++ echo DB_USER=airflow ++ awk -F= '{print $1}'
KEY=DB_USER ++ echo DB_USER=airflow ++ awk -F= '{print $2}'
VAL=airflow
'[' DB_USER=airflow '!=' DB_USER=airflow ']'
'[' -n DB_USER=airflow ']'
eval DB_USER=airflow ++ DB_USER=airflow
read -r LINE ++ echo api:auth_backend=airflow.api.auth.backend.default ++ awk -F: '{print $1}'
SECTION=api ++ echo api:auth_backend=airflow.api.auth.backend.default ++ sed -e 's|api:||'
KV=auth_backend=airflow.api.auth.backend.default ++ awk -F= '{print $1}' ++ echo auth_backend=airflow.api.auth.backend.default
KEY=auth_backend ++ echo auth_backend=airflow.api.auth.backend.default ++ awk -F= '{print $2}'
VAL=airflow.api.auth.backend.default
'[' api '!=' auth_backend=airflow.api.auth.backend.default ']'
'[' airflow.api.auth.backend.default == true ']'
'[' airflow.api.auth.backend.default == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg api auth_backend airflow.api.auth.backend.default
read -r LINE ++ echo celery:default_queue=default ++ awk -F: '{print $1}'
SECTION=celery ++ sed -e 's|celery:||' ++ echo celery:default_queue=default
KV=default_queue=default ++ echo default_queue=default ++ awk -F= '{print $1}'
KEY=default_queue ++ echo default_queue=default ++ awk -F= '{print $2}'
VAL=default
'[' celery '!=' default_queue=default ']'
'[' default == true ']'
'[' default == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg celery default_queue default
read -r LINE ++ echo celery:worker_concurrency=16 ++ awk -F: '{print $1}'
SECTION=celery ++ echo celery:worker_concurrency=16 ++ sed -e 's|celery:||'
KV=worker_concurrency=16 ++ awk -F= '{print $1}' ++ echo worker_concurrency=16
KEY=worker_concurrency ++ awk -F= '{print $2}' ++ echo worker_concurrency=16
VAL=16
'[' celery '!=' worker_concurrency=16 ']'
'[' 16 == true ']'
'[' 16 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg celery worker_concurrency 16
read -r LINE ++ awk -F: '{print $1}' ++ echo celery:worker_log_server_port=8793
SECTION=celery ++ echo celery:worker_log_server_port=8793 ++ sed -e 's|celery:||'
KV=worker_log_server_port=8793 ++ echo worker_log_server_port=8793 ++ awk -F= '{print $1}'
KEY=worker_log_server_port ++ echo worker_log_server_port=8793 ++ awk -F= '{print $2}'
VAL=8793
'[' celery '!=' worker_log_server_port=8793 ']'
'[' 8793 == true ']'
'[' 8793 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg celery worker_log_server_port 8793
read -r LINE ++ echo cli:api_client=airflow.api.client.local_client ++ awk -F: '{print $1}'
SECTION=cli ++ echo cli:api_client=airflow.api.client.local_client ++ sed -e 's|cli:||'
KV=api_client=airflow.api.client.local_client ++ echo api_client=airflow.api.client.local_client ++ awk -F= '{print $1}'
KEY=api_client ++ echo api_client=airflow.api.client.local_client ++ awk -F= '{print $2}'
VAL=airflow.api.client.local_client
'[' cli '!=' api_client=airflow.api.client.local_client ']'
'[' airflow.api.client.local_client == true ']'
'[' airflow.api.client.local_client == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg cli api_client airflow.api.client.local_client
read -r LINE ++ echo cli:endpoint_url=http://localhost:8080 ++ awk -F: '{print $1}'
SECTION=cli ++ echo cli:endpoint_url=http://localhost:8080 ++ sed -e 's|cli:||'
KV=endpoint_url=http://localhost:8080 ++ awk -F= '{print $1}' ++ echo endpoint_url=http://localhost:8080
KEY=endpoint_url ++ echo endpoint_url=http://localhost:8080 ++ awk -F= '{print $2}'
VAL=http://localhost:8080
'[' cli '!=' endpoint_url=http://localhost:8080 ']'
'[' http://localhost:8080 == true ']'
'[' http://localhost:8080 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg cli endpoint_url http://localhost:8080
read -r LINE ++ echo core:base_log_folder=/var/log/airflow ++ awk -F: '{print $1}'
SECTION=core ++ echo core:base_log_folder=/var/log/airflow ++ sed -e 's|core:||'
KV=base_log_folder=/var/log/airflow ++ echo base_log_folder=/var/log/airflow ++ awk -F= '{print $1}'
KEY=base_log_folder ++ echo base_log_folder=/var/log/airflow ++ awk -F= '{print $2}'
VAL=/var/log/airflow
'[' core '!=' base_log_folder=/var/log/airflow ']'
'[' /var/log/airflow == true ']'
'[' /var/log/airflow == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core base_log_folder /var/log/airflow
read -r LINE ++ echo core:dag_concurrency=4 ++ awk -F: '{print $1}'
SECTION=core ++ echo core:dag_concurrency=4 ++ sed -e 's|core:||'
KV=dag_concurrency=4 ++ awk -F= '{print $1}' ++ echo dag_concurrency=4
KEY=dag_concurrency ++ awk -F= '{print $2}' ++ echo dag_concurrency=4
VAL=4
'[' core '!=' dag_concurrency=4 ']'
'[' 4 == true ']'
'[' 4 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core dag_concurrency 4
read -r LINE ++ awk -F: '{print $1}' ++ echo core:dagbag_import_timeout=30
SECTION=core ++ sed -e 's|core:||' ++ echo core:dagbag_import_timeout=30
KV=dagbag_import_timeout=30 ++ awk -F= '{print $1}' ++ echo dagbag_import_timeout=30
KEY=dagbag_import_timeout ++ echo dagbag_import_timeout=30 ++ awk -F= '{print $2}'
VAL=30
'[' core '!=' dagbag_import_timeout=30 ']'
'[' 30 == true ']'
'[' 30 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core dagbag_import_timeout 30
read -r LINE ++ echo core:dags_are_paused_at_creation=true ++ awk -F: '{print $1}'
SECTION=core ++ sed -e 's|core:||' ++ echo core:dags_are_paused_at_creation=true
KV=dags_are_paused_at_creation=true ++ awk -F= '{print $1}' ++ echo dags_are_paused_at_creation=true
KEY=dags_are_paused_at_creation ++ awk -F= '{print $2}' ++ echo dags_are_paused_at_creation=true
VAL=true
'[' core '!=' dags_are_paused_at_creation=true ']'
'[' true == true ']'
VAL=True
'[' True == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core dags_are_paused_at_creation True
read -r LINE ++ awk -F: '{print $1}' ++ echo core:dags_folder=/var/lib/airflow/dags
SECTION=core ++ echo core:dags_folder=/var/lib/airflow/dags ++ sed -e 's|core:||'
KV=dags_folder=/var/lib/airflow/dags ++ echo dags_folder=/var/lib/airflow/dags ++ awk -F= '{print $1}'
KEY=dags_folder ++ echo dags_folder=/var/lib/airflow/dags ++ awk -F= '{print $2}'
VAL=/var/lib/airflow/dags
'[' core '!=' dags_folder=/var/lib/airflow/dags ']'
'[' /var/lib/airflow/dags == true ']'
'[' /var/lib/airflow/dags == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core dags_folder /var/lib/airflow/dags
read -r LINE ++ echo core:default_impersonation= ++ awk -F: '{print $1}'
SECTION=core ++ echo core:default_impersonation= ++ sed -e 's|core:||'
KV=default_impersonation= ++ awk -F= '{print $1}' ++ echo default_impersonation=
KEY=default_impersonation ++ awk -F= '{print $2}' ++ echo default_impersonation=
VAL=
'[' core '!=' default_impersonation= ']'
'[' '' == true ']'
'[' '' == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core default_impersonation ''
read -r LINE ++ echo core:donot_pickle=false ++ awk -F: '{print $1}'
SECTION=core ++ sed -e 's|core:||' ++ echo core:donot_pickle=false
KV=donot_pickle=false ++ awk -F= '{print $1}' ++ echo donot_pickle=false
KEY=donot_pickle ++ echo donot_pickle=false ++ awk -F= '{print $2}'
VAL=false
'[' core '!=' donot_pickle=false ']'
'[' false == true ']'
'[' false == false ']'
VAL=False
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core donot_pickle False
read -r LINE ++ echo core:executor=LocalExecutor ++ awk -F: '{print $1}'
SECTION=core ++ sed -e 's|core:||' ++ echo core:executor=LocalExecutor
KV=executor=LocalExecutor ++ awk -F= '{print $1}' ++ echo executor=LocalExecutor
KEY=executor ++ awk -F= '{print $2}' ++ echo executor=LocalExecutor
VAL=LocalExecutor
'[' core '!=' executor=LocalExecutor ']'
'[' LocalExecutor == true ']'
'[' LocalExecutor == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core executor LocalExecutor
read -r LINE ++ awk -F: '{print $1}' ++ echo core:fernet_key=
SECTION=core ++ echo core:fernet_key= ++ sed -e 's|core:||'
KV=fernet_key= ++ echo fernet_key= ++ awk -F= '{print $1}'
KEY=fernet_key ++ echo fernet_key= ++ awk -F= '{print $2}'
VAL=
'[' core '!=' fernet_key= ']'
'[' '' == true ']'
'[' '' == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core fernet_key ''
read -r LINE ++ awk -F: '{print $1}' ++ echo core:load_examples=false
SECTION=core ++ sed -e 's|core:||' ++ echo core:load_examples=false
KV=load_examples=false ++ awk -F= '{print $1}' ++ echo load_examples=false
KEY=load_examples ++ awk -F= '{print $2}' ++ echo load_examples=false
VAL=false
'[' core '!=' load_examples=false ']'
'[' false == true ']'
'[' false == false ']'
VAL=False
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core load_examples False
read -r LINE ++ echo core:max_active_runs_per_dag=16 ++ awk -F: '{print $1}'
SECTION=core ++ sed -e 's|core:||' ++ echo core:max_active_runs_per_dag=16
KV=max_active_runs_per_dag=16 ++ echo max_active_runs_per_dag=16 ++ awk -F= '{print $1}'
KEY=max_active_runs_per_dag ++ echo max_active_runs_per_dag=16 ++ awk -F= '{print $2}'
VAL=16
'[' core '!=' max_active_runs_per_dag=16 ']'
'[' 16 == true ']'
'[' 16 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core max_active_runs_per_dag 16
read -r LINE ++ echo core:non_pooled_task_slot_count=128 ++ awk -F: '{print $1}'
SECTION=core ++ echo core:non_pooled_task_slot_count=128 ++ sed -e 's|core:||'
KV=non_pooled_task_slot_count=128 ++ echo non_pooled_task_slot_count=128 ++ awk -F= '{print $1}'
KEY=non_pooled_task_slot_count ++ echo non_pooled_task_slot_count=128 ++ awk -F= '{print $2}'
VAL=128
'[' core '!=' non_pooled_task_slot_count=128 ']'
'[' 128 == true ']'
'[' 128 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core non_pooled_task_slot_count 128
read -r LINE ++ echo core:parallelism=8 ++ awk -F: '{print $1}'
SECTION=core ++ sed -e 's|core:||' ++ echo core:parallelism=8
KV=parallelism=8 ++ awk -F= '{print $1}' ++ echo parallelism=8
KEY=parallelism ++ awk -F= '{print $2}' ++ echo parallelism=8
VAL=8
'[' core '!=' parallelism=8 ']'
'[' 8 == true ']'
'[' 8 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core parallelism 8
read -r LINE ++ echo core:plugins_folder=/var/lib/airflow/plugins ++ awk -F: '{print $1}'
SECTION=core ++ echo core:plugins_folder=/var/lib/airflow/plugins ++ sed -e 's|core:||'
KV=plugins_folder=/var/lib/airflow/plugins ++ echo plugins_folder=/var/lib/airflow/plugins ++ awk -F= '{print $1}'
KEY=plugins_folder ++ echo plugins_folder=/var/lib/airflow/plugins ++ awk -F= '{print $2}'
VAL=/var/lib/airflow/plugins
'[' core '!=' plugins_folder=/var/lib/airflow/plugins ']'
'[' /var/lib/airflow/plugins == true ']'
'[' /var/lib/airflow/plugins == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core plugins_folder /var/lib/airflow/plugins
read -r LINE ++ awk -F: '{print $1}' ++ echo core:sql_alchemy_pool_enabled=true
SECTION=core ++ echo core:sql_alchemy_pool_enabled=true ++ sed -e 's|core:||'
KV=sql_alchemy_pool_enabled=true ++ awk -F= '{print $1}' ++ echo sql_alchemy_pool_enabled=true
KEY=sql_alchemy_pool_enabled ++ awk -F= '{print $2}' ++ echo sql_alchemy_pool_enabled=true
VAL=true
'[' core '!=' sql_alchemy_pool_enabled=true ']'
'[' true == true ']'
VAL=True
'[' True == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core sql_alchemy_pool_enabled True
read -r LINE ++ echo core:sql_alchemy_pool_recycle=1800 ++ awk -F: '{print $1}'
SECTION=core ++ sed -e 's|core:||' ++ echo core:sql_alchemy_pool_recycle=1800
KV=sql_alchemy_pool_recycle=1800 ++ echo sql_alchemy_pool_recycle=1800 ++ awk -F= '{print $1}'
KEY=sql_alchemy_pool_recycle ++ awk -F= '{print $2}' ++ echo sql_alchemy_pool_recycle=1800
VAL=1800
'[' core '!=' sql_alchemy_pool_recycle=1800 ']'
'[' 1800 == true ']'
'[' 1800 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core sql_alchemy_pool_recycle 1800
read -r LINE ++ awk -F: '{print $1}' ++ echo core:sql_alchemy_pool_size=5
SECTION=core ++ echo core:sql_alchemy_pool_size=5 ++ sed -e 's|core:||'
KV=sql_alchemy_pool_size=5 ++ echo sql_alchemy_pool_size=5 ++ awk -F= '{print $1}'
KEY=sql_alchemy_pool_size ++ echo sql_alchemy_pool_size=5 ++ awk -F= '{print $2}'
VAL=5
'[' core '!=' sql_alchemy_pool_size=5 ']'
'[' 5 == true ']'
'[' 5 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core sql_alchemy_pool_size 5
read -r LINE ++ awk -F: '{print $1}' ++ echo core:sql_alchemy_reconnect_timeout=300
SECTION=core ++ echo core:sql_alchemy_reconnect_timeout=300 ++ sed -e 's|core:||'
KV=sql_alchemy_reconnect_timeout=300 ++ echo sql_alchemy_reconnect_timeout=300 ++ awk -F= '{print $1}'
KEY=sql_alchemy_reconnect_timeout ++ echo sql_alchemy_reconnect_timeout=300 ++ awk -F= '{print $2}'
VAL=300
'[' core '!=' sql_alchemy_reconnect_timeout=300 ']'
'[' 300 == true ']'
'[' 300 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core sql_alchemy_reconnect_timeout 300
read -r LINE ++ echo core:task_runner=BashTaskRunner ++ awk -F: '{print $1}'
SECTION=core ++ sed -e 's|core:||' ++ echo core:task_runner=BashTaskRunner
KV=task_runner=BashTaskRunner ++ echo task_runner=BashTaskRunner ++ awk -F= '{print $1}'
KEY=task_runner ++ echo task_runner=BashTaskRunner ++ awk -F= '{print $2}'
VAL=BashTaskRunner
'[' core '!=' task_runner=BashTaskRunner ']'
'[' BashTaskRunner == true ']'
'[' BashTaskRunner == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core task_runner BashTaskRunner
read -r LINE ++ echo hive:default_hive_mapred_queue= ++ awk -F: '{print $1}'
SECTION=hive ++ echo hive:default_hive_mapred_queue= ++ sed -e 's|hive:||'
KV=default_hive_mapred_queue= ++ echo default_hive_mapred_queue= ++ awk -F= '{print $1}'
KEY=default_hive_mapred_queue ++ awk -F= '{print $2}' ++ echo default_hive_mapred_queue=
VAL=
'[' hive '!=' default_hive_mapred_queue= ']'
'[' '' == true ']'
'[' '' == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg hive default_hive_mapred_queue ''
read -r LINE ++ echo log_dir=/var/log/airflow ++ awk -F: '{print $1}'
SECTION=log_dir=/var/log/airflow ++ echo log_dir=/var/log/airflow ++ sed -e 's|log_dir=/var/log/airflow:||'
KV=log_dir=/var/log/airflow ++ echo log_dir=/var/log/airflow ++ awk -F= '{print $1}'
KEY=log_dir ++ echo log_dir=/var/log/airflow ++ awk -F= '{print $2}'
VAL=/var/log/airflow
'[' log_dir=/var/log/airflow '!=' log_dir=/var/log/airflow ']'
'[' -n log_dir=/var/log/airflow ']'
eval log_dir=/var/log/airflow ++ log_dir=/var/log/airflow
read -r LINE ++ echo operators:default_cpus=1 ++ awk -F: '{print $1}'
SECTION=operators ++ echo operators:default_cpus=1 ++ sed -e 's|operators:||'
KV=default_cpus=1 ++ echo default_cpus=1 ++ awk -F= '{print $1}'
KEY=default_cpus ++ echo default_cpus=1 ++ awk -F= '{print $2}'
VAL=1
'[' operators '!=' default_cpus=1 ']'
'[' 1 == true ']'
'[' 1 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg operators default_cpus 1
read -r LINE ++ echo operators:default_disk=512 ++ awk -F: '{print $1}'
SECTION=operators ++ sed -e 's|operators:||' ++ echo operators:default_disk=512
KV=default_disk=512 ++ awk -F= '{print $1}' ++ echo default_disk=512
KEY=default_disk ++ awk -F= '{print $2}' ++ echo default_disk=512
VAL=512
'[' operators '!=' default_disk=512 ']'
'[' 512 == true ']'
'[' 512 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg operators default_disk 512
read -r LINE ++ echo operators:default_gpus=0 ++ awk -F: '{print $1}'
SECTION=operators ++ echo operators:default_gpus=0 ++ sed -e 's|operators:||'
KV=default_gpus=0 ++ echo default_gpus=0 ++ awk -F= '{print $1}'
KEY=default_gpus ++ awk -F= '{print $2}' ++ echo default_gpus=0
VAL=0
'[' operators '!=' default_gpus=0 ']'
'[' 0 == true ']'
'[' 0 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg operators default_gpus 0
read -r LINE ++ awk -F: '{print $1}' ++ echo operators:default_owner=Airflow
SECTION=operators ++ echo operators:default_owner=Airflow ++ sed -e 's|operators:||'
KV=default_owner=Airflow ++ awk -F= '{print $1}' ++ echo default_owner=Airflow
KEY=default_owner ++ echo default_owner=Airflow ++ awk -F= '{print $2}'
VAL=Airflow
'[' operators '!=' default_owner=Airflow ']'
'[' Airflow == true ']'
'[' Airflow == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg operators default_owner Airflow
read -r LINE ++ echo operators:default_ram=512 ++ awk -F: '{print $1}'
SECTION=operators ++ echo operators:default_ram=512 ++ sed -e 's|operators:||'
KV=default_ram=512 ++ echo default_ram=512 ++ awk -F= '{print $1}'
KEY=default_ram ++ echo default_ram=512 ++ awk -F= '{print $2}'
VAL=512
'[' operators '!=' default_ram=512 ']'
'[' 512 == true ']'
'[' 512 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg operators default_ram 512
read -r LINE ++ echo smtp:smtp_host=localhost ++ awk -F: '{print $1}'
SECTION=smtp ++ echo smtp:smtp_host=localhost ++ sed -e 's|smtp:||'
KV=smtp_host=localhost ++ echo smtp_host=localhost ++ awk -F= '{print $1}'
KEY=smtp_host ++ echo smtp_host=localhost ++ awk -F= '{print $2}'
VAL=localhost
'[' smtp '!=' smtp_host=localhost ']'
'[' localhost == true ']'
'[' localhost == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg smtp smtp_host localhost
read -r LINE ++ echo smtp:smtp_mail_from=airflow@localhost ++ awk -F: '{print $1}'
SECTION=smtp ++ echo smtp:smtp_mail_from=airflow@localhost ++ sed -e 's|smtp:||'
KV=smtp_mail_from=airflow@localhost ++ echo smtp_mail_from=airflow@localhost ++ awk -F= '{print $1}'
KEY=smtp_mail_from ++ echo smtp_mail_from=airflow@localhost ++ awk -F= '{print $2}'
VAL=airflow@localhost
'[' smtp '!=' smtp_mail_from=airflow@localhost ']'
'[' airflow@localhost == true ']'
'[' airflow@localhost == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg smtp smtp_mail_from airflow@localhost
read -r LINE ++ awk -F: '{print $1}' ++ echo smtp:smtp_password=
SECTION=smtp ++ sed -e 's|smtp:||' ++ echo smtp:smtp_password=
KV=smtp_password= ++ echo smtp_password= ++ awk -F= '{print $1}'
KEY=smtp_password ++ echo smtp_password= ++ awk -F= '{print $2}'
VAL=
'[' smtp '!=' smtp_password= ']'
'[' '' == true ']'
'[' '' == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg smtp smtp_password ''
read -r LINE ++ echo smtp:smtp_port=25 ++ awk -F: '{print $1}'
SECTION=smtp ++ echo smtp:smtp_port=25 ++ sed -e 's|smtp:||'
KV=smtp_port=25 ++ echo smtp_port=25 ++ awk -F= '{print $1}'
KEY=smtp_port ++ awk -F= '{print $2}' ++ echo smtp_port=25
VAL=25
'[' smtp '!=' smtp_port=25 ']'
'[' 25 == true ']'
'[' 25 == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg smtp smtp_port 25
read -r LINE ++ awk -F: '{print $1}' ++ echo smtp:smtp_ssl=false
SECTION=smtp ++ sed -e 's|smtp:||' ++ echo smtp:smtp_ssl=false
KV=smtp_ssl=false ++ echo smtp_ssl=false ++ awk -F= '{print $1}'
KEY=smtp_ssl ++ echo smtp_ssl=false ++ awk -F= '{print $2}'
VAL=false
'[' smtp '!=' smtp_ssl=false ']'
'[' false == true ']'
'[' false == false ']'
VAL=False
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg smtp smtp_ssl False
read -r LINE ++ echo smtp:smtp_starttls=true ++ awk -F: '{print $1}'
SECTION=smtp ++ echo smtp:smtp_starttls=true ++ sed -e 's|smtp:||'
KV=smtp_starttls=true ++ echo smtp_starttls=true ++ awk -F= '{print $1}'
KEY=smtp_starttls ++ echo smtp_starttls=true ++ awk -F= '{print $2}'
VAL=true
'[' smtp '!=' smtp_starttls=true ']'
'[' true == true ']'
VAL=True
'[' True == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg smtp smtp_starttls True
read -r LINE ++ echo smtp:smtp_user= ++ awk -F: '{print $1}'
SECTION=smtp ++ echo smtp:smtp_user= ++ sed -e 's|smtp:||'
KV=smtp_user= ++ awk -F= '{print $1}' ++ echo smtp_user=
KEY=smtp_user ++ echo smtp_user= ++ awk -F= '{print $2}'
VAL=
'[' smtp '!=' smtp_user= ']'
'[' '' == true ']'
'[' '' == false ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg smtp smtp_user ''
read -r LINE
'[' MySQL == SQLite3 ']'
'[' MySQL == MySQL ']'
VAL=mysql://airflow:p@ssw0rd123@usadc-vsedgdh03:3306/airflow
'[' -n mysql://airflow:p@ssw0rd123@usadc-vsedgdh03:3306/airflow ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg core sql_alchemy_conn mysql://airflow:p@ssw0rd123@usadc-vsedgdh03:3306/airflow
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg celery result_backend db+mysql://airflow:p@ssw0rd123@usadc-vsedgdh03:3306/airflow
'[' RabbitMQ == RabbitMQ ']'
VAL=amqp://:@:/
'[' -n amqp://:@:/ ']'
crudini --set /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg celery broker_url amqp://:@:/
echo -e '\nexport AIRFLOW_HOME=/var/lib/airflow'
echo -e '\nexport AIRFLOW_CONFIG=/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg'
chgrp airflow /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow.cfg /var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER/airflow-env.sh
log 'Starting Airflow worker...' ++ date '+%Y-%m-%d %H:%M:%S'
timestamp='2020-07-13 00:13:10'
echo '2020-07-13 00:13:10: Starting Airflow worker...'
su -s /bin/bash - airflow -c 'CONF_DIR=/var/run/cloudera-scm-agent/process/12038-airflow-AIRFLOW_WORKER exec /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/bin/airflow-cm.sh worker ' /opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/airflow/configuration.py:214: FutureWarning: The task_runner setting in [core] has the old default value of u'BashTaskRunner'. This value has been changed to u'StandardTaskRunner' in the running config, but please update your config before Apache Airflow 2.0. FutureWarning, Traceback (most recent call last): File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/bin/.airflow", line 32, in args.func(args) File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/airflow/utils/cli.py", line 74, in wrapper return f(*args, kwargs) File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/airflow/bin/cli.py", line 1090, in worker worker.run(options) File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/celery/bin/worker.py", line 257, in run kwargs) File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/celery/worker/worker.py", line 101, in init self.setup_instance(self.prepare_args(kwargs)) File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/celery/worker/worker.py", line 122, in setup_instance self._conninfo = self.app.connection_for_read() File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/celery/app/base.py", line 743, in connection_for_read return self._connection(url or self.conf.broker_read_url, kwargs) File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/celery/app/base.py", line 819, in _connection 'broker_connection_timeout', connect_timeout File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/kombu/connection.py", line 189, in init url_params = parse_url(hostname) File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/kombu/utils/url.py", line 47, in parse_url scheme, host, port, user, password, path, query = _parse_url(url) File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/kombu/utils/url.py", line 81, in url_to_parts parts.port, File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/future/backports/urllib/parse.py", line 162, in port port = int(port, 10) File "/opt/cloudera/parcels/AIRFLOW-1.10.3-python2.7.15_b2/lib/python2.7/site-packages/future/types/newint.py", line 76, in new return super(newint, cls).new(cls, val) ValueError: invalid literal for long() with base 10: ''

teamclairvoyant / apache-airflow-cloudera-csd

worker service not starting on cdh6.3 #10