hkvision
commented
1 year ago From @zpeng1898 After adding shutdown() of estimator, the errors are eliminated.
Open lalalapotter opened 1 year ago
hkvision
commented
1 year ago From @zpeng1898 After adding shutdown() of estimator, the errors are eliminated.
zpeng1898
commented
1 year ago I encountered the same issue when I run python tf_train_xshards.py --data_dir /local/path/to/NCF --cluster_mode local with backend="ray" on my laptop, and it was caused by the directories train_processed_xshards and test_processed_xshards which already existed in /local/path/to/NCF. The error disappeared after removing the two directories.
So the issue seems has nothing to do with estimator.shutdown(), because I test it again with and without estimator.shutdown(), both run normally without any error.
python tf_train_xshards.py --data_dir /local/path/to/NCF --cluster_mode local.
tf_train_xshards.py:
# Step 0: Import necessary libraries
import mathimport tensorflow as tf
from process_xshards import prepare_data from tf_model import ncf_model from utils import *
from bigdl.orca.learn.tf2 import Estimator
args = parse_args("TensorFlow NCF Training with Orca XShards") args.backend = "ray" # TODO: fix spark backend for saving optimizer states init_orca(args.cluster_mode, extra_python_lib="tf_model.py,process_xshards.py")
train_data, test_data, user_num, item_num, sparse_feats_input_dims, num_dense_feats, \ feature_cols, label_cols = prepare_data(args.data_dir, args.dataset, num_ng=4)
config = dict( user_num=user_num, item_num=item_num, factor_num=16, num_layers=3, dropout=0.5, lr=0.01, sparse_feats_input_dims=sparse_feats_input_dims, sparse_feats_embed_dims=8, num_dense_feats=num_dense_feats )
def model_creator(config): model = ncf_model(user_num=config["user_num"], item_num=config["item_num"], num_layers=config["num_layers"], factor_num=config["factor_num"], dropout=config["dropout"], lr=config["lr"], sparse_feats_input_dims=config["sparse_feats_input_dims"], sparse_feats_embed_dims=config["sparse_feats_embed_dims"], num_dense_feats=config["num_dense_feats"]) return model
est = Estimator.from_keras(model_creator=model_creator, config=config, backend=args.backend, workers_per_node=args.workers_per_node)
batch_size = 10240 train_steps = math.ceil(len(train_data) / batch_size) val_steps = math.ceil(len(test_data) / batch_size) callbacks = [tf.keras.callbacks.TensorBoard(log_dir=os.path.join(args.model_dir, "logs"))] \ if args.tensorboard else []
if args.lr_scheduler: lr_callback = tf.keras.callbacks.LearningRateScheduler(schedule_func, verbose=1) callbacks.append(lr_callback)
train_stats = est.fit(train_data, epochs=2, batch_size=batch_size, feature_cols=feature_cols, label_cols=label_cols, steps_per_epoch=train_steps, validation_data=test_data, validation_steps=val_steps, callbacks=callbacks) print("Train results:") for k, v in train_stats.items(): print("{}: {}".format(k, v))
eval_stats = est.evaluate(test_data, feature_cols=feature_cols, label_cols=label_cols, batch_size=batch_size, num_steps=val_steps) print("Evaluation results:") for k, v in eval_stats.items(): print("{}: {}".format(k, v))
est.save(os.path.join(args.model_dir, "NCF_model")) save_model_config(config, args.model_dir, "config.json") train_data.save_pickle(os.path.join(args.data_dir, "train_processed_xshards")) test_data.save_pickle(os.path.join(args.data_dir, "test_processed_xshards"))
est.shutdown() stop_orca_context()
- Error message if don't remove `train_processed_xshards` and `test_processed_xshards`
(Worker pid=6458) WARNING:absl:Found untraced functions such as _update_step_xla while saving (showing 1 of 1).
These functions will not be directly callable after loading.
Traceback (most recent call last): File "tf_train_xshards.py", line 112, in
2023-02-09 22:15:52,176 ERROR worker.py:1247 -- listen_error_messages_raylet: Connection closed by server. 2023-02-09 22:15:52,177 ERROR worker.py:1247 -- listen_error_messages_raylet: Connection closed by server. 2023-02-09 22:15:52,177 ERROR worker.py:1247 -- listen_error_messages_raylet: Connection closed by server. 2023-02-09 22:15:52,178 ERROR import_thread.py:89 -- ImportThread: Connection closed by server. 2023-02-09 22:15:52,178 ERROR import_thread.py:89 -- ImportThread: Connection closed by server. 2023-02-09 22:15:52,178 ERROR worker.py:478 -- print_logs: Connection closed by server. 2023-02-09 22:15:52,178 ERROR worker.py:1247 -- listen_error_messages_raylet: Connection closed by server. 2023-02-09 22:15:52,178 ERROR worker.py:1247 -- listen_error_messages_raylet: Connection closed by server. 2023-02-09 22:15:52,178 ERROR worker.py:478 -- print_logs: Connection closed by server. 2023-02-09 22:15:52,179 ERROR import_thread.py:89 -- ImportThread: Connection closed by server. 2023-02-09 22:15:52,179 ERROR worker.py:1247 -- listen_error_messages_raylet: Connection closed by server. 2023-02-09 22:15:52,179 ERROR worker.py:1247 -- listen_error_messages_raylet: Connection closed by server. 2023-02-09 22:15:52,179 ERROR import_thread.py:89 -- ImportThread: Connection closed by server. 2023-02-09 22:15:52,179 ERROR worker.py:1247 -- listen_error_messages_raylet: Connection closed by server. 2023-02-09 22:15:52,179 ERROR worker.py:478 -- print_logs: Connection closed by server. 2023-02-09 22:15:52,179 ERROR worker.py:1247 -- listen_error_messages_raylet: Connection closed by server. 2023-02-09 22:15:52,180 ERROR worker.py:478 -- print_logs: Connection closed by server. 2023-02-09 22:15:52,180 ERROR import_thread.py:89 -- ImportThread: Connection closed by server. 2023-02-09 22:15:52,180 ERROR worker.py:1247 -- listen_error_messages_raylet: Connection closed by server. 2023-02-09 22:15:52,180 ERROR import_thread.py:89 -- ImportThread: Connection closed by server. 2023-02-09 22:15:52,180 ERROR import_thread.py:89 -- ImportThread: Connection closed by server. 2023-02-09 22:15:52,180 ERROR import_thread.py:89 -- ImportThread: Connection closed by server. 2023-02-09 22:15:52,180 ERROR worker.py:478 -- print_logs: Connection closed by server. 2023-02-09 22:15:52,180 ERROR worker.py:478 -- print_logs: Connection closed by server. 2023-02-09 22:15:52,180 ERROR worker.py:1247 -- listen_error_messages_raylet: Connection closed by server. 2023-02-09 22:15:52,180 ERROR worker.py:1247 -- listen_error_messages_raylet: Connection closed by server. 2023-02-09 22:15:52,181 ERROR worker.py:1247 -- listen_error_messages_raylet: Connection closed by server. 2023-02-09 22:15:52,181 ERROR import_thread.py:89 -- ImportThread: Connection closed by server. 2023-02-09 22:15:52,181 ERROR worker.py:478 -- print_logs: Connection closed by server. 2023-02-09 22:15:52,181 ERROR worker.py:478 -- print_logs: Connection closed by server. 2023-02-09 22:15:52,181 ERROR worker.py:478 -- print_logs: Connection closed by server. 2023-02-09 22:15:52,181 ERROR import_thread.py:89 -- ImportThread: Connection closed by server. 2023-02-09 22:15:52,181 ERROR worker.py:1247 -- listen_error_messages_raylet: Connection closed by server. 2023-02-09 22:15:52,181 ERROR worker.py:478 -- print_logs: Connection closed by server. 2023-02-09 22:15:52,183 ERROR import_thread.py:89 -- ImportThread: Connection closed by server. 2023-02-09 22:15:52,183 ERROR import_thread.py:89 -- ImportThread: Connection closed by server. 2023-02-09 22:15:52,183 ERROR worker.py:478 -- print_logs: Connection closed by server. 2023-02-09 22:15:52,184 ERROR worker.py:478 -- print_logs: Connection closed by server. 2023-02-09 22:15:52,184 ERROR import_thread.py:89 -- ImportThread: Connection closed by server. 2023-02-09 22:15:52,184 ERROR worker.py:1247 -- listen_error_messages_raylet: Connection closed by server. 2023-02-09 22:15:52,184 ERROR worker.py:1247 -- listen_error_messages_raylet: Connection closed by server. 2023-02-09 22:15:52,185 ERROR import_thread.py:89 -- ImportThread: Connection closed by server. 2023-02-09 22:15:52,185 ERROR worker.py:478 -- print_logs: Connection closed by server. 2023-02-09 22:15:52,185 ERROR worker.py:478 -- print_logs: Connection closed by server. 2023-02-09 22:15:52,185 ERROR import_thread.py:89 -- ImportThread: Connection closed by server. 2023-02-09 22:15:52,185 ERROR worker.py:478 -- print_logs: Connection closed by server. 2023-02-09 22:15:52,185 ERROR worker.py:478 -- print_logs: Connection closed by server. 2023-02-09 22:15:52,185 ERROR worker.py:478 -- print_logs: Connection closed by server. 2023-02-09 22:15:52,186 ERROR import_thread.py:89 -- ImportThread: Connection closed by server. 2023-02-09 22:15:52,186 ERROR worker.py:1247 -- listen_error_messages_raylet: Connection closed by server. 2023-02-09 22:15:52,186 ERROR worker.py:478 -- print_logs: Connection closed by server. 2023-02-09 22:15:52,186 ERROR worker.py:478 -- print_logs: Connection closed by server. 2023-02-09 22:15:52,186 ERROR worker.py:1247 -- listen_error_messages_raylet: Connection closed by server. 2023-02-09 22:15:52,186 ERROR worker.py:478 -- print_logs: Connection closed by server. 2023-02-09 22:15:52,186 ERROR worker.py:1247 -- listen_error_messages_raylet: Connection closed by server. 2023-02-09 22:15:52,186 ERROR worker.py:478 -- print_logs: Connection closed by server. 2023-02-09 22:15:52,187 ERROR worker.py:1247 -- listen_error_messages_raylet: Connection closed by server. 2023-02-09 22:15:52,187 ERROR import_thread.py:89 -- ImportThread: Connection closed by server. 2023-02-09 22:15:52,187 ERROR import_thread.py:89 -- ImportThread: Connection closed by server. 2023-02-09 22:15:52,188 ERROR worker.py:1247 -- listen_error_messages_raylet: Connection closed by server. 2023-02-09 22:15:52,188 ERROR import_thread.py:89 -- ImportThread: Connection closed by server. 2023-02-09 22:15:52,188 ERROR worker.py:1247 -- listen_error_messages_raylet: Connection closed by server. 2023-02-09 22:15:52,189 ERROR import_thread.py:89 -- ImportThread: Connection closed by server. 2023-02-09 22:15:52,189 ERROR import_thread.py:89 -- ImportThread: Connection closed by server. 2023-02-09 22:15:52,189 ERROR import_thread.py:89 -- ImportThread: Connection closed by server. 2023-02-09 22:15:52,190 ERROR worker.py:1247 -- listen_error_messages_raylet: Connection closed by server. 2023-02-09 22:15:52,190 ERROR worker.py:478 -- print_logs: Connection closed by server. 2023-02-09 22:15:52,190 ERROR worker.py:478 -- print_logs: Connection closed by server. 2023-02-09 22:15:52,189 ERROR import_thread.py:89 -- ImportThread: Connection closed by server. 2023-02-09 22:15:52,190 ERROR worker.py:478 -- print_logs: Connection closed by server. 2023-02-09 22:15:52,195 ERROR import_thread.py:89 -- ImportThread: Connection closed by server. 2023-02-09 22:15:52,195 ERROR worker.py:1247 -- listen_error_messages_raylet: Connection closed by server.
Stopping orca context
- Results after removing 2 directories
conda create -n NCF_env python=3.7
pip install --pre --upgrade bigdl-orca-spark3[ray]
pip install tensorflow torch torchvision torchmetrics==0.10.0
pip install tqdm pandas scikit-learn pyarrow
zpeng1898
commented
1 year ago python tf_train_spark_dataframe.py
2023-02-13 15:32:10.143756: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. 2023-02-13 15:32:10.270621: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: :/root/.mujoco/mujoco200/bin 2023-02-13 15:32:10.270675: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. 2023-02-13 15:32:11.010399: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: :/root/.mujoco/mujoco200/bin 2023-02-13 15:32:11.014429: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: :/root/.mujoco/mujoco200/bin 2023-02-13 15:32:11.014478: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. Initializing orca context For Spark local mode, default to use all the cores on the node. Current pyspark location is : /root/miniconda3/envs/py375/lib/python3.7/site-packages/pyspark/__init__.py Start to getOrCreate SparkContext pyspark_submit_args is: --driver-class-path /root/miniconda3/envs/py375/lib/python3.7/site-packages/bigdl/share/orca/lib/bigdl-orca-spark_3.1.3-2.3.0-SNAPSHOT-jar-with-dependencies.jar:/root/miniconda3/envs/py375/lib/python3.7/site-packages/bigdl/share/core/lib/all-2.3.0-20221216.060121-1.jar:/root/miniconda3/envs/py375/lib/python3.7/site-packages/bigdl/share/dllib/lib/bigdl-dllib-spark_3.1.3-2.3.0-SNAPSHOT-jar-with-dependencies.jar pyspark-shell [main] WARN org.apache.spark.util.Utils - Your hostname, ziyaopen-mobl resolves to a loopback address: 127.0.1.1; using 172.26.34.55 instead (on interface eth0) [main] WARN org.apache.spark.util.Utils - Set SPARK_LOCAL_IP if you need to bind to another address [main] WARN org.apache.hadoop.util.NativeCodeLoader - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable Setting default log level to "WARN". To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel). 2023-02-13 15:32:16,318 Thread-4 WARN The bufferSize is set to 4000 but bufferedIo is false: false 2023-02-13 15:32:16,329 Thread-4 WARN The bufferSize is set to 4000 but bufferedIo is false: false 2023-02-13 15:32:16,329 Thread-4 WARN The bufferSize is set to 4000 but bufferedIo is false: false 2023-02-13 15:32:16,330 Thread-4 WARN The bufferSize is set to 4000 but bufferedIo is false: false 23-02-13 15:32:16 [Thread-4] INFO Engine$:122 - Auto detect executor number and executor cores number 23-02-13 15:32:16 [Thread-4] INFO Engine$:124 - Executor number is 1 and executor cores number is 4 User settings: KMP_AFFINITY=granularity=fine,compact,1,0 KMP_BLOCKTIME=0 KMP_SETTINGS=1 OMP_NUM_THREADS=1 Effective settings: KMP_ABORT_DELAY=0 KMP_ADAPTIVE_LOCK_PROPS='1,1024' KMP_ALIGN_ALLOC=64 KMP_ALL_THREADPRIVATE=128 KMP_ATOMIC_MODE=2 KMP_BLOCKTIME=0 KMP_CPUINFO_FILE: value is not defined KMP_DETERMINISTIC_REDUCTION=false KMP_DEVICE_THREAD_LIMIT=2147483647 KMP_DISP_HAND_THREAD=false KMP_DISP_NUM_BUFFERS=7 KMP_DUPLICATE_LIB_OK=false KMP_FORCE_REDUCTION: value is not defined KMP_FOREIGN_THREADS_THREADPRIVATE=true KMP_FORKJOIN_BARRIER='2,2' KMP_FORKJOIN_BARRIER_PATTERN='hyper,hyper' KMP_FORKJOIN_FRAMES=true KMP_FORKJOIN_FRAMES_MODE=3 KMP_GTID_MODE=3 KMP_HANDLE_SIGNALS=false KMP_HOT_TEAMS_MAX_LEVEL=1 KMP_HOT_TEAMS_MODE=0 KMP_INIT_AT_FORK=true KMP_ITT_PREPARE_DELAY=0 KMP_LIBRARY=throughput KMP_LOCK_KIND=queuing KMP_MALLOC_POOL_INCR=1M KMP_MWAIT_HINTS=0 KMP_NUM_LOCKS_IN_BLOCK=1 KMP_PLAIN_BARRIER='2,2' KMP_PLAIN_BARRIER_PATTERN='hyper,hyper' KMP_REDUCTION_BARRIER='1,1' KMP_REDUCTION_BARRIER_PATTERN='hyper,hyper' KMP_SCHEDULE='static,balanced;guided,iterative' KMP_SETTINGS=true KMP_SPIN_BACKOFF_PARAMS='4096,100' KMP_STACKOFFSET=64 KMP_STACKPAD=0 KMP_STACKSIZE=8M KMP_STORAGE_MAP=false KMP_TASKING=2 KMP_TASKLOOP_MIN_TASKS=0 KMP_TASK_STEALING_CONSTRAINT=1 KMP_TEAMS_THREAD_LIMIT=8 KMP_TOPOLOGY_METHOD=all KMP_USER_LEVEL_MWAIT=false KMP_USE_YIELD=1 KMP_VERSION=false KMP_WARNINGS=true OMP_AFFINITY_FORMAT='OMP: pid %P tid %i thread %n bound to OS proc set {%A}' OMP_ALLOCATOR=omp_default_mem_alloc OMP_CANCELLATION=false OMP_DEBUG=disabled OMP_DEFAULT_DEVICE=0 OMP_DISPLAY_AFFINITY=false OMP_DISPLAY_ENV=false OMP_DYNAMIC=false OMP_MAX_ACTIVE_LEVELS=2147483647 OMP_MAX_TASK_PRIORITY=0 OMP_NESTED=false OMP_NUM_THREADS='1' OMP_PLACES: value is not defined OMP_PROC_BIND='intel' OMP_SCHEDULE='static' OMP_STACKSIZE=8M OMP_TARGET_OFFLOAD=DEFAULT OMP_THREAD_LIMIT=2147483647 OMP_TOOL=enabled OMP_TOOL_LIBRARIES: value is not defined OMP_WAIT_POLICY=PASSIVE KMP_AFFINITY='noverbose,warnings,respect,granularity=fine,compact,1,0' 23-02-13 15:32:16 [Thread-4] INFO ThreadPool$:95 - Set mkl threads to 1 on thread 16 [Thread-4] WARN org.apache.spark.SparkContext - Using an existing SparkContext; some configuration may not take effect. 23-02-13 15:32:16 [Thread-4] INFO Engine$:461 - Find existing spark context. Checking the spark conf... cls.getname: com.intel.analytics.bigdl.dllib.utils.python.api.Sample BigDLBasePickler registering: bigdl.dllib.utils.common Sample cls.getname: com.intel.analytics.bigdl.dllib.utils.python.api.EvaluatedResult BigDLBasePickler registering: bigdl.dllib.utils.common EvaluatedResult cls.getname: com.intel.analytics.bigdl.dllib.utils.python.api.JTensor BigDLBasePickler registering: bigdl.dllib.utils.common JTensor cls.getname: com.intel.analytics.bigdl.dllib.utils.python.api.JActivity BigDLBasePickler registering: bigdl.dllib.utils.common JActivity Successfully got a SparkContext Loading data... 2023-02-13 15:32:47,820 INFO services.py:1340 -- View the Ray dashboard at http://172.26.34.55:8265 {'node_ip_address': '172.26.34.55', 'raylet_ip_address': '172.26.34.55', 'redis_address': '172.26.34.55:6379', 'object_store_address': '/tmp/ray/session_2023-02-13_15-32-45_289362_376/sockets/plasma_store', 'raylet_socket_name': '/tmp/ray/session_2023-02-13_15-32-45_289362_376/sockets/raylet', 'webui_url': '172.26.34.55:8265', 'session_dir': '/tmp/ray/session_2023-02-13_15-32-45_289362_376', 'metrics_export_port': 55692, 'node_id': '4e27130341441f0ae5e5a7d06d93c833459e8c5ec1c89a3e7ec34ee1'} (Worker pid=1347) 2023-02-13 15:32:49.497897: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA (Worker pid=1347) To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. (Worker pid=1347) 2023-02-13 15:32:49.633262: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: :/root/.mujoco/mujoco200/bin (Worker pid=1347) 2023-02-13 15:32:49.633292: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. (Worker pid=1347) 2023-02-13 15:32:50.279241: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: :/root/.mujoco/mujoco200/bin (Worker pid=1347) 2023-02-13 15:32:50.283090: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: :/root/.mujoco/mujoco200/bin (Worker pid=1347) 2023-02-13 15:32:50.283120: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. (Worker pid=1347) WARNING:tensorflow:From /root/miniconda3/envs/py375/lib/python3.7/site-packages/bigdl/orca/learn/tf2/tf_runner.py:337: _CollectiveAllReduceStrategyExperimental.__init__ (from tensorflow.python.distribute.collective_all_reduce_strategy) is deprecated and will be removed in a future version. (Worker pid=1347) Instructions for updating: (Worker pid=1347) use distribute.MultiWorkerMirroredStrategy instead (Worker pid=1347) 2023-02-13 15:32:51.118952: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: :/root/.mujoco/mujoco200/bin (Worker pid=1347) 2023-02-13 15:32:51.119744: W tensorflow/compiler/xla/stream_executor/cuda/cuda_driver.cc:265] failed call to cuInit: UNKNOWN ERROR (303) (Worker pid=1347) 2023-02-13 15:32:51.119776: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (ziyaopen-mobl): /proc/driver/nvidia/version does not exist (Worker pid=1347) 2023-02-13 15:32:51.122356: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA (Worker pid=1347) To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. (Worker pid=1347) 2023-02-13 15:32:51.147693: I tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:447] Started server with target: grpc://172.26.34.55:59881 (Worker pid=1347) 2023-02-13 15:32:51.157024: I tensorflow/core/distributed_runtime/coordination/coordination_service.cc:502] /job:worker/replica:0/task:0 has connected to coordination service. Incarnation: 10720989701530919701 (Worker pid=1347) 2023-02-13 15:32:51.157848: I tensorflow/core/distributed_runtime/coordination/coordination_service_agent.cc:277] Coordination agent has successfully connected. 2023-02-13 15:33:58,497 INFO worker.py:843 -- Connecting to existing Ray cluster at address: 172.26.34.55:6379 2023-02-13 15:33:58,520 INFO worker.py:843 -- Connecting to existing Ray cluster at address: 172.26.34.55:6379 2023-02-13 15:33:58,579 INFO worker.py:843 -- Connecting to existing Ray cluster at address: 172.26.34.55:6379 2023-02-13 15:33:58,584 INFO worker.py:843 -- Connecting to existing Ray cluster at address: 172.26.34.55:6379 2023-02-13 15:33:58,588 INFO worker.py:843 -- Connecting to existing Ray cluster at address: 172.26.34.55:6379 2023-02-13 15:33:58,604 INFO worker.py:843 -- Connecting to existing Ray cluster at address: 172.26.34.55:6379 2023-02-13 15:33:58,651 INFO worker.py:843 -- Connecting to existing Ray cluster at address: 172.26.34.55:6379 2023-02-13 15:33:58,678 INFO worker.py:843 -- Connecting to existing Ray cluster at address: 172.26.34.55:6379 (Worker pid=1347) WARNING:tensorflow:From /root/miniconda3/envs/py375/lib/python3.7/site-packages/bigdl/orca/learn/tf2/tf_runner.py:206: StrategyBase.experimental_distribute_datasets_from_function (from tensorflow.python.distribute.distribute_lib) is deprecated and will be removed in a future version. (Worker pid=1347) Instructions for updating: (Worker pid=1347) rename to distribute_datasets_from_function (Worker pid=1347) 2023-02-13 15:36:47.136740: W tensorflow/core/framework/dataset.cc:769] Input of GeneratorDatasetOp::Dataset will not be optimized because the dataset does not implement the AsGraphDefInternal() method needed to apply optimizations. (Worker pid=1347) Epoch 1/2 1/385 [..............................] - ETA: 19:10 - loss: 0.7110 - accuracy: 0.3308 - auc: 0.5089 - precision: 0.2068 - recall: 0.8042 2/385 [..............................] - ETA: 30s - loss: 0.6975 - accuracy: 0.5051 - auc: 0.5066 - precision: 0.2096 - recall: 0.5202 3/385 [..............................] - ETA: 42s - loss: 0.6861 - accuracy: 0.5974 - auc: 0.5075 - precision: 0.2095 - recall: 0.3616 4/385 [..............................] - ETA: 47s - loss: 0.6747 - accuracy: 0.6452 - auc: 0.5040 - precision: 0.2101 - recall: 0.2740 5/385 [..............................] - ETA: 50s - loss: 0.6612 - accuracy: 0.6744 - auc: 0.5039 - precision: 0.2099 - recall: 0.2202 6/385 [..............................] - ETA: 51s - loss: 0.6455 - accuracy: 0.6947 - auc: 0.5058 - precision: 0.2097 - recall: 0.1846 7/385 [..............................] - ETA: 52s - loss: 0.6302 - accuracy: 0.7086 - auc: 0.5063 - precision: 0.2095 - recall: 0.1583 8/385 [..............................] - ETA: 53s - loss: 0.6190 - accuracy: 0.7185 - auc: 0.5060 - precision: 0.2094 - recall: 0.1380 9/385 [..............................] - ETA: 54s - loss: 0.6098 - accuracy: 0.7270 - auc: 0.5096 - precision: 0.2092 - recall: 0.1226 10/385 [..............................] - ETA: 55s - loss: 0.5987 - accuracy: 0.7346 - auc: 0.5173 - precision: 0.2091 - recall: 0.1108 11/385 [..............................] - ETA: 55s - loss: 0.5879 - accuracy: 0.7407 - auc: 0.5264 - precision: 0.2090 - recall: 0.1009 12/385 [..............................] - ETA: 56s - loss: 0.5786 - accuracy: 0.7447 - auc: 0.5359 - precision: 0.2090 - recall: 0.0922 13/385 [>.............................] - ETA: 56s - loss: 0.5691 - accuracy: 0.7488 - auc: 0.5469 - precision: 0.2089 - recall: 0.0851 14/385 [>.............................] - ETA: 56s - loss: 0.5602 - accuracy: 0.7520 - auc: 0.5585 - precision: 0.2089 - recall: 0.0790 15/385 [>.............................] - ETA: 57s - loss: 0.5524 - accuracy: 0.7548 - auc: 0.5692 - precision: 0.2089 - recall: 0.0736 16/385 [>.............................] - ETA: 57s - loss: 0.5448 - accuracy: 0.7573 - auc: 0.5805 - precision: 0.2089 - recall: 0.0690 17/385 [>.............................] - ETA: 57s - loss: 0.5381 - accuracy: 0.7591 - auc: 0.5911 - precision: 0.2089 - recall: 0.0648 18/385 [>.............................] - ETA: 58s - loss: 0.5323 - accuracy: 0.7610 - auc: 0.6006 - precision: 0.2089 - recall: 0.0611 19/385 [>.............................] - ETA: 58s - loss: 0.5267 - accuracy: 0.7628 - auc: 0.6100 - precision: 0.2089 - recall: 0.0579 20/385 [>.............................] - ETA: 58s - loss: 0.5223 - accuracy: 0.7640 - auc: 0.6184 - precision: 0.2088 - recall: 0.0549 21/385 [>.............................] - ETA: 58s - loss: 0.5173 - accuracy: 0.7659 - auc: 0.6264 - precision: 0.2088 - recall: 0.0524 22/385 [>.............................] - ETA: 58s - loss: 0.5127 - accuracy: 0.7675 - auc: 0.6338 - precision: 0.2088 - recall: 0.0501 23/385 [>.............................] - ETA: 58s - loss: 0.5085 - accuracy: 0.7691 - auc: 0.6404 - precision: 0.2088 - recall: 0.0480 24/385 [>.............................] - ETA: 58s - loss: 0.5048 - accuracy: 0.7702 - auc: 0.6470 - precision: 0.2088 - recall: 0.0460 25/385 [>.............................] - ETA: 58s - loss: 0.5011 - accuracy: 0.7715 - auc: 0.6528 - precision: 0.2088 - recall: 0.0442 26/385 [=>............................] - ETA: 58s - loss: 0.4982 - accuracy: 0.7723 - auc: 0.6581 - precision: 0.2088 - recall: 0.0424 27/385 [=>............................] - ETA: 59s - loss: 0.4954 - accuracy: 0.7733 - auc: 0.6631 - precision: 0.2088 - recall: 0.0409 28/385 [=>............................] - ETA: 59s - loss: 0.4924 - accuracy: 0.7743 - auc: 0.6680 - precision: 0.2088 - recall: 0.0395 29/385 [=>............................] - ETA: 59s - loss: 0.4895 - accuracy: 0.7750 - auc: 0.6729 - precision: 0.2088 - recall: 0.0381 30/385 [=>............................] - ETA: 59s - loss: 0.4870 - accuracy: 0.7757 - auc: 0.6774 - precision: 0.2088 - recall: 0.0368 31/385 [=>............................] - ETA: 59s - loss: 0.4846 - accuracy: 0.7765 - auc: 0.6816 - precision: 0.2088 - recall: 0.0357 32/385 [=>............................] - ETA: 59s - loss: 0.4823 - accuracy: 0.7771 - auc: 0.6858 - precision: 0.2088 - recall: 0.0345 33/385 [=>............................] - ETA: 59s - loss: 0.4801 - accuracy: 0.7777 - auc: 0.6899 - precision: 0.2089 - recall: 0.0335 34/385 [=>............................] - ETA: 59s - loss: 0.4777 - accuracy: 0.7785 - auc: 0.6937 - precision: 0.2090 - recall: 0.0326 35/385 [=>............................] - ETA: 59s - loss: 0.4755 - accuracy: 0.7789 - auc: 0.6974 - precision: 0.2091 - recall: 0.0316 36/385 [=>............................] - ETA: 59s - loss: 0.4737 - accuracy: 0.7793 - auc: 0.7006 - precision: 0.2094 - recall: 0.0308 37/385 [=>............................] - ETA: 1:00 - loss: 0.4718 - accuracy: 0.7796 - auc: 0.7042 - precision: 0.2108 - recall: 0.0302 38/385 [=>............................] - ETA: 1:00 - loss: 0.4699 - accuracy: 0.7802 - auc: 0.7073 - precision: 0.2126 - recall: 0.0298 39/385 [==>...........................] - ETA: 1:00 - loss: 0.4683 - accuracy: 0.7806 - auc: 0.7102 - precision: 0.2154 - recall: 0.0296 40/385 [==>...........................] - ETA: 1:00 - loss: 0.4666 - accuracy: 0.7810 - auc: 0.7132 - precision: 0.2196 - recall: 0.0298 41/385 [==>...........................] - ETA: 1:00 - loss: 0.4653 - accuracy: 0.7815 - auc: 0.7159 - precision: 0.2278 - recall: 0.0306 42/385 [==>...........................] - ETA: 1:00 - loss: 0.4638 - accuracy: 0.7820 - auc: 0.7185 - precision: 0.2371 - recall: 0.0318 43/385 [==>...........................] - ETA: 1:00 - loss: 0.4624 - accuracy: 0.7825 - auc: 0.7210 - precision: 0.2456 - recall: 0.0330 44/385 [==>...........................] - ETA: 1:00 - loss: 0.4608 - accuracy: 0.7830 - auc: 0.7236 - precision: 0.2552 - recall: 0.0346 45/385 [==>...........................] - ETA: 1:00 - loss: 0.4598 - accuracy: 0.7834 - auc: 0.7258 - precision: 0.2660 - recall: 0.0364 46/385 [==>...........................] - ETA: 1:00 - loss: 0.4586 - accuracy: 0.7839 - auc: 0.7279 - precision: 0.2750 - recall: 0.0379 47/385 [==>...........................] - ETA: 1:00 - loss: 0.4573 - accuracy: 0.7843 - auc: 0.7300 - precision: 0.2848 - recall: 0.0397 48/385 [==>...........................] - ETA: 59s - loss: 0.4561 - accuracy: 0.7847 - auc: 0.7320 - precision: 0.2941 - recall: 0.0415 49/385 [==>...........................] - ETA: 59s - loss: 0.4551 - accuracy: 0.7852 - auc: 0.7339 - precision: 0.3027 - recall: 0.0430 50/385 [==>...........................] - ETA: 59s - loss: 0.4541 - accuracy: 0.7854 - auc: 0.7357 - precision: 0.3118 - recall: 0.0448 51/385 [==>...........................] - ETA: 59s - loss: 0.4530 - accuracy: 0.7859 - auc: 0.7376 - precision: 0.3202 - recall: 0.0465 52/385 [===>..........................] - ETA: 59s - loss: 0.4519 - accuracy: 0.7862 - auc: 0.7393 - precision: 0.3265 - recall: 0.0478 53/385 [===>..........................] - ETA: 59s - loss: 0.4509 - accuracy: 0.7864 - auc: 0.7411 - precision: 0.3337 - recall: 0.0491 54/385 [===>..........................] - ETA: 59s - loss: 0.4497 - accuracy: 0.7869 - auc: 0.7430 - precision: 0.3410 - recall: 0.0506 55/385 [===>..........................] - ETA: 59s - loss: 0.4486 - accuracy: 0.7873 - auc: 0.7446 - precision: 0.3479 - recall: 0.0521 56/385 [===>..........................] - ETA: 59s - loss: 0.4479 - accuracy: 0.7875 - auc: 0.7460 - precision: 0.3534 - recall: 0.0532 57/385 [===>..........................] - ETA: 59s - loss: 0.4469 - accuracy: 0.7879 - auc: 0.7477 - precision: 0.3595 - recall: 0.0546 58/385 [===>..........................] - ETA: 58s - loss: 0.4460 - accuracy: 0.7881 - auc: 0.7492 - precision: 0.3640 - recall: 0.0557 59/385 [===>..........................] - ETA: 58s - loss: 0.4449 - accuracy: 0.7884 - auc: 0.7506 - precision: 0.3685 - recall: 0.0566 60/385 [===>..........................] - ETA: 58s - loss: 0.4442 - accuracy: 0.7887 - auc: 0.7520 - precision: 0.3741 - recall: 0.0577 61/385 [===>..........................] - ETA: 58s - loss: 0.4434 - accuracy: 0.7890 - auc: 0.7533 - precision: 0.3792 - recall: 0.0587 62/385 [===>..........................] - ETA: 58s - loss: 0.4427 - accuracy: 0.7892 - auc: 0.7545 - precision: 0.3835 - recall: 0.0594 63/385 [===>..........................] - ETA: 58s - loss: 0.4418 - accuracy: 0.7895 - auc: 0.7559 - precision: 0.3879 - recall: 0.0602 64/385 [===>..........................] - ETA: 58s - loss: 0.4410 - accuracy: 0.7898 - auc: 0.7572 - precision: 0.3909 - recall: 0.0607 65/385 [====>.........................] - ETA: 58s - loss: 0.4403 - accuracy: 0.7900 - auc: 0.7583 - precision: 0.3949 - recall: 0.0612 66/385 [====>.........................] - ETA: 57s - loss: 0.4396 - accuracy: 0.7901 - auc: 0.7595 - precision: 0.3985 - recall: 0.0617 67/385 [====>.........................] - ETA: 57s - loss: 0.4388 - accuracy: 0.7905 - auc: 0.7606 - precision: 0.4020 - recall: 0.0624 68/385 [====>.........................] - ETA: 57s - loss: 0.4380 - accuracy: 0.7907 - auc: 0.7618 - precision: 0.4063 - recall: 0.0633 69/385 [====>.........................] - ETA: 57s - loss: 0.4374 - accuracy: 0.7909 - auc: 0.7629 - precision: 0.4097 - recall: 0.0640 70/385 [====>.........................] - ETA: 57s - loss: 0.4368 - accuracy: 0.7909 - auc: 0.7639 - precision: 0.4123 - recall: 0.0645 71/385 [====>.........................] - ETA: 57s - loss: 0.4362 - accuracy: 0.7911 - auc: 0.7650 - precision: 0.4158 - recall: 0.0652 72/385 [====>.........................] - ETA: 57s - loss: 0.4355 - accuracy: 0.7913 - auc: 0.7659 - precision: 0.4190 - recall: 0.0658 73/385 [====>.........................] - ETA: 56s - loss: 0.4349 - accuracy: 0.7916 - auc: 0.7668 - precision: 0.4219 - recall: 0.0663 74/385 [====>.........................] - ETA: 56s - loss: 0.4343 - accuracy: 0.7918 - auc: 0.7677 - precision: 0.4251 - recall: 0.0668 75/385 [====>.........................] - ETA: 56s - loss: 0.4337 - accuracy: 0.7920 - auc: 0.7687 - precision: 0.4280 - recall: 0.0673 76/385 [====>.........................] - ETA: 56s - loss: 0.4331 - accuracy: 0.7921 - auc: 0.7696 - precision: 0.4311 - recall: 0.0677 77/385 [=====>........................] - ETA: 56s - loss: 0.4325 - accuracy: 0.7923 - auc: 0.7706 - precision: 0.4346 - recall: 0.0683 78/385 [=====>........................] - ETA: 56s - loss: 0.4319 - accuracy: 0.7925 - auc: 0.7716 - precision: 0.4373 - recall: 0.0688 79/385 [=====>........................] - ETA: 56s - loss: 0.4315 - accuracy: 0.7926 - auc: 0.7723 - precision: 0.4396 - recall: 0.0694 80/385 [=====>........................] - ETA: 56s - loss: 0.4310 - accuracy: 0.7928 - auc: 0.7732 - precision: 0.4428 - recall: 0.0700 81/385 [=====>........................] - ETA: 56s - loss: 0.4305 - accuracy: 0.7930 - auc: 0.7741 - precision: 0.4459 - recall: 0.0707 82/385 [=====>........................] - ETA: 55s - loss: 0.4299 - accuracy: 0.7932 - auc: 0.7750 - precision: 0.4488 - recall: 0.0713 83/385 [=====>........................] - ETA: 55s - loss: 0.4294 - accuracy: 0.7933 - auc: 0.7759 - precision: 0.4517 - recall: 0.0720 84/385 [=====>........................] - ETA: 55s - loss: 0.4289 - accuracy: 0.7935 - auc: 0.7766 - precision: 0.4544 - recall: 0.0727 85/385 [=====>........................] - ETA: 55s - loss: 0.4283 - accuracy: 0.7938 - auc: 0.7775 - precision: 0.4579 - recall: 0.0735 86/385 [=====>........................] - ETA: 55s - loss: 0.4276 - accuracy: 0.7940 - auc: 0.7785 - precision: 0.4617 - recall: 0.0744 87/385 [=====>........................] - ETA: 55s - loss: 0.4271 - accuracy: 0.7942 - auc: 0.7792 - precision: 0.4650 - recall: 0.0753 88/385 [=====>........................] - ETA: 55s - loss: 0.4265 - accuracy: 0.7945 - auc: 0.7801 - precision: 0.4693 - recall: 0.0765 89/385 [=====>........................] - ETA: 54s - loss: 0.4259 - accuracy: 0.7949 - auc: 0.7809 - precision: 0.4734 - recall: 0.0776 90/385 [======>.......................] - ETA: 54s - loss: 0.4254 - accuracy: 0.7951 - auc: 0.7817 - precision: 0.4776 - recall: 0.0789 91/385 [======>.......................] - ETA: 54s - loss: 0.4249 - accuracy: 0.7954 - auc: 0.7824 - precision: 0.4815 - recall: 0.0802 92/385 [======>.......................] - ETA: 54s - loss: 0.4244 - accuracy: 0.7957 - auc: 0.7831 - precision: 0.4856 - recall: 0.0815 93/385 [======>.......................] - ETA: 54s - loss: 0.4240 - accuracy: 0.7959 - auc: 0.7838 - precision: 0.4900 - recall: 0.0832 94/385 [======>.......................] - ETA: 54s - loss: 0.4235 - accuracy: 0.7963 - auc: 0.7845 - precision: 0.4946 - recall: 0.0850 95/385 [======>.......................] - ETA: 53s - loss: 0.4231 - accuracy: 0.7966 - auc: 0.7851 - precision: 0.4994 - recall: 0.0869 96/385 [======>.......................] - ETA: 53s - loss: 0.4227 - accuracy: 0.7969 - auc: 0.7858 - precision: 0.5035 - recall: 0.0888 97/385 [======>.......................] - ETA: 53s - loss: 0.4221 - accuracy: 0.7973 - auc: 0.7865 - precision: 0.5075 - recall: 0.0907 98/385 [======>.......................] - ETA: 53s - loss: 0.4217 - accuracy: 0.7976 - auc: 0.7872 - precision: 0.5128 - recall: 0.0930 99/385 [======>.......................] - ETA: 53s - loss: 0.4212 - accuracy: 0.7980 - auc: 0.7879 - precision: 0.5167 - recall: 0.0952 100/385 [======>.......................] - ETA: 53s - loss: 0.4206 - accuracy: 0.7984 - auc: 0.7886 - precision: 0.5206 - recall: 0.0974 101/385 [======>.......................] - ETA: 52s - loss: 0.4202 - accuracy: 0.7987 - auc: 0.7892 - precision: 0.5242 - recall: 0.0996 102/385 [======>.......................] - ETA: 52s - loss: 0.4197 - accuracy: 0.7990 - auc: 0.7898 - precision: 0.5279 - recall: 0.1017 103/385 [=======>......................] - ETA: 52s - loss: 0.4194 - accuracy: 0.7993 - auc: 0.7904 - precision: 0.5317 - recall: 0.1038 104/385 [=======>......................] - ETA: 52s - loss: 0.4189 - accuracy: 0.7996 - auc: 0.7911 - precision: 0.5356 - recall: 0.1061 105/385 [=======>......................] - ETA: 52s - loss: 0.4184 - accuracy: 0.8000 - auc: 0.7917 - precision: 0.5393 - recall: 0.1083 106/385 [=======>......................] - ETA: 52s - loss: 0.4180 - accuracy: 0.8003 - auc: 0.7923 - precision: 0.5426 - recall: 0.1106 107/385 [=======>......................] - ETA: 51s - loss: 0.4175 - accuracy: 0.8007 - auc: 0.7930 - precision: 0.5461 - recall: 0.1130 108/385 [=======>......................] - ETA: 51s - loss: 0.4171 - accuracy: 0.8010 - auc: 0.7935 - precision: 0.5486 - recall: 0.1152 109/385 [=======>......................] - ETA: 51s - loss: 0.4166 - accuracy: 0.8013 - auc: 0.7941 - precision: 0.5517 - recall: 0.1176 110/385 [=======>......................] - ETA: 51s - loss: 0.4162 - accuracy: 0.8016 - auc: 0.7947 - precision: 0.5546 - recall: 0.1198 111/385 [=======>......................] - ETA: 51s - loss: 0.4157 - accuracy: 0.8019 - auc: 0.7953 - precision: 0.5573 - recall: 0.1220 112/385 [=======>......................] - ETA: 51s - loss: 0.4153 - accuracy: 0.8022 - auc: 0.7959 - precision: 0.5605 - recall: 0.1244 113/385 [=======>......................] - ETA: 50s - loss: 0.4149 - accuracy: 0.8026 - auc: 0.7965 - precision: 0.5631 - recall: 0.1266 114/385 [=======>......................] - ETA: 50s - loss: 0.4145 - accuracy: 0.8029 - auc: 0.7970 - precision: 0.5661 - recall: 0.1288 115/385 [=======>......................] - ETA: 50s - loss: 0.4141 - accuracy: 0.8032 - auc: 0.7976 - precision: 0.5684 - recall: 0.1312 116/385 [========>.....................] - ETA: 50s - loss: 0.4137 - accuracy: 0.8034 - auc: 0.7981 - precision: 0.5708 - recall: 0.1335 117/385 [========>.....................] - ETA: 50s - loss: 0.4133 - accuracy: 0.8037 - auc: 0.7987 - precision: 0.5732 - recall: 0.1358 118/385 [========>.....................] - ETA: 50s - loss: 0.4129 - accuracy: 0.8040 - auc: 0.7992 - precision: 0.5748 - recall: 0.1379 119/385 [========>.....................] - ETA: 49s - loss: 0.4124 - accuracy: 0.8043 - auc: 0.7998 - precision: 0.5768 - recall: 0.1401 120/385 [========>.....................] - ETA: 49s - loss: 0.4120 - accuracy: 0.8046 - auc: 0.8003 - precision: 0.5790 - recall: 0.1423 121/385 [========>.....................] - ETA: 49s - loss: 0.4116 - accuracy: 0.8049 - auc: 0.8008 - precision: 0.5809 - recall: 0.1444 122/385 [========>.....................] - ETA: 49s - loss: 0.4112 - accuracy: 0.8051 - auc: 0.8013 - precision: 0.5829 - recall: 0.1465 123/385 [========>.....................] - ETA: 49s - loss: 0.4108 - accuracy: 0.8055 - auc: 0.8019 - precision: 0.5850 - recall: 0.1486 124/385 [========>.....................] - ETA: 49s - loss: 0.4103 - accuracy: 0.8057 - auc: 0.8024 - precision: 0.5868 - recall: 0.1506 125/385 [========>.....................] - ETA: 48s - loss: 0.4099 - accuracy: 0.8060 - auc: 0.8030 - precision: 0.5887 - recall: 0.1526 126/385 [========>.....................] - ETA: 48s - loss: 0.4095 - accuracy: 0.8063 - auc: 0.8035 - precision: 0.5907 - recall: 0.1549 127/385 [========>.....................] - ETA: 48s - loss: 0.4091 - accuracy: 0.8065 - auc: 0.8040 - precision: 0.5924 - recall: 0.1570 128/385 [========>.....................] - ETA: 48s - loss: 0.4087 - accuracy: 0.8069 - auc: 0.8045 - precision: 0.5942 - recall: 0.1592 129/385 [=========>....................] - ETA: 48s - loss: 0.4083 - accuracy: 0.8071 - auc: 0.8050 - precision: 0.5958 - recall: 0.1611 130/385 [=========>....................] - ETA: 47s - loss: 0.4079 - accuracy: 0.8074 - auc: 0.8055 - precision: 0.5972 - recall: 0.1633 131/385 [=========>....................] - ETA: 47s - loss: 0.4074 - accuracy: 0.8077 - auc: 0.8060 - precision: 0.5988 - recall: 0.1654 132/385 [=========>....................] - ETA: 47s - loss: 0.4071 - accuracy: 0.8080 - auc: 0.8065 - precision: 0.5999 - recall: 0.1675 133/385 [=========>....................] - ETA: 47s - loss: 0.4067 - accuracy: 0.8082 - auc: 0.8070 - precision: 0.6012 - recall: 0.1697 134/385 [=========>....................] - ETA: 47s - loss: 0.4063 - accuracy: 0.8085 - auc: 0.8074 - precision: 0.6023 - recall: 0.1717 135/385 [=========>....................] - ETA: 47s - loss: 0.4059 - accuracy: 0.8088 - auc: 0.8079 - precision: 0.6040 - recall: 0.1739 136/385 [=========>....................] - ETA: 46s - loss: 0.4056 - accuracy: 0.8090 - auc: 0.8083 - precision: 0.6052 - recall: 0.1757 137/385 [=========>....................] - ETA: 46s - loss: 0.4051 - accuracy: 0.8093 - auc: 0.8088 - precision: 0.6065 - recall: 0.1777 138/385 [=========>....................] - ETA: 46s - loss: 0.4047 - accuracy: 0.8096 - auc: 0.8093 - precision: 0.6079 - recall: 0.1799 139/385 [=========>....................] - ETA: 46s - loss: 0.4043 - accuracy: 0.8098 - auc: 0.8098 - precision: 0.6096 - recall: 0.1821 140/385 [=========>....................] - ETA: 46s - loss: 0.4039 - accuracy: 0.8101 - auc: 0.8103 - precision: 0.6108 - recall: 0.1841 141/385 [=========>....................] - ETA: 46s - loss: 0.4034 - accuracy: 0.8104 - auc: 0.8109 - precision: 0.6123 - recall: 0.1861 142/385 [==========>...................] - ETA: 45s - loss: 0.4029 - accuracy: 0.8107 - auc: 0.8114 - precision: 0.6136 - recall: 0.1879 143/385 [==========>...................] - ETA: 45s - loss: 0.4026 - accuracy: 0.8110 - auc: 0.8118 - precision: 0.6146 - recall: 0.1897 144/385 [==========>...................] - ETA: 45s - loss: 0.4021 - accuracy: 0.8112 - auc: 0.8123 - precision: 0.6157 - recall: 0.1916 145/385 [==========>...................] - ETA: 45s - loss: 0.4018 - accuracy: 0.8114 - auc: 0.8127 - precision: 0.6167 - recall: 0.1935 146/385 [==========>...................] - ETA: 45s - loss: 0.4014 - accuracy: 0.8117 - auc: 0.8132 - precision: 0.6176 - recall: 0.1953 147/385 [==========>...................] - ETA: 44s - loss: 0.4011 - accuracy: 0.8119 - auc: 0.8136 - precision: 0.6186 - recall: 0.1972 148/385 [==========>...................] - ETA: 44s - loss: 0.4007 - accuracy: 0.8121 - auc: 0.8140 - precision: 0.6194 - recall: 0.1991 149/385 [==========>...................] - ETA: 44s - loss: 0.4003 - accuracy: 0.8124 - auc: 0.8144 - precision: 0.6202 - recall: 0.2008 150/385 [==========>...................] - ETA: 44s - loss: 0.3999 - accuracy: 0.8127 - auc: 0.8149 - precision: 0.6213 - recall: 0.2027 151/385 [==========>...................] - ETA: 44s - loss: 0.3995 - accuracy: 0.8129 - auc: 0.8153 - precision: 0.6224 - recall: 0.2046 152/385 [==========>...................] - ETA: 44s - loss: 0.3991 - accuracy: 0.8132 - auc: 0.8158 - precision: 0.6234 - recall: 0.2064 153/385 [==========>...................] - ETA: 43s - loss: 0.3987 - accuracy: 0.8134 - auc: 0.8162 - precision: 0.6244 - recall: 0.2082 154/385 [===========>..................] - ETA: 43s - loss: 0.3983 - accuracy: 0.8137 - auc: 0.8167 - precision: 0.6255 - recall: 0.2101 155/385 [===========>..................] - ETA: 43s - loss: 0.3978 - accuracy: 0.8140 - auc: 0.8172 - precision: 0.6263 - recall: 0.2119 156/385 [===========>..................] - ETA: 43s - loss: 0.3975 - accuracy: 0.8142 - auc: 0.8176 - precision: 0.6272 - recall: 0.2137 157/385 [===========>..................] - ETA: 43s - loss: 0.3971 - accuracy: 0.8145 - auc: 0.8180 - precision: 0.6280 - recall: 0.2153 158/385 [===========>..................] - ETA: 42s - loss: 0.3967 - accuracy: 0.8147 - auc: 0.8185 - precision: 0.6289 - recall: 0.2171 159/385 [===========>..................] - ETA: 42s - loss: 0.3963 - accuracy: 0.8149 - auc: 0.8189 - precision: 0.6296 - recall: 0.2188 160/385 [===========>..................] - ETA: 42s - loss: 0.3959 - accuracy: 0.8152 - auc: 0.8193 - precision: 0.6306 - recall: 0.2206 161/385 [===========>..................] - ETA: 42s - loss: 0.3955 - accuracy: 0.8154 - auc: 0.8197 - precision: 0.6310 - recall: 0.2222 162/385 [===========>..................] - ETA: 42s - loss: 0.3952 - accuracy: 0.8157 - auc: 0.8201 - precision: 0.6319 - recall: 0.2239 163/385 [===========>..................] - ETA: 42s - loss: 0.3949 - accuracy: 0.8159 - auc: 0.8205 - precision: 0.6329 - recall: 0.2256 164/385 [===========>..................] - ETA: 41s - loss: 0.3944 - accuracy: 0.8161 - auc: 0.8209 - precision: 0.6337 - recall: 0.2273 165/385 [===========>..................] - ETA: 41s - loss: 0.3940 - accuracy: 0.8164 - auc: 0.8213 - precision: 0.6344 - recall: 0.2290 166/385 [===========>..................] - ETA: 41s - loss: 0.3937 - accuracy: 0.8166 - auc: 0.8217 - precision: 0.6351 - recall: 0.2308 167/385 [============>.................] - ETA: 41s - loss: 0.3934 - accuracy: 0.8168 - auc: 0.8221 - precision: 0.6360 - recall: 0.2325 168/385 [============>.................] - ETA: 41s - loss: 0.3930 - accuracy: 0.8171 - auc: 0.8225 - precision: 0.6369 - recall: 0.2343 169/385 [============>.................] - ETA: 40s - loss: 0.3926 - accuracy: 0.8173 - auc: 0.8230 - precision: 0.6376 - recall: 0.2361 170/385 [============>.................] - ETA: 40s - loss: 0.3922 - accuracy: 0.8175 - auc: 0.8234 - precision: 0.6384 - recall: 0.2377 171/385 [============>.................] - ETA: 40s - loss: 0.3919 - accuracy: 0.8178 - auc: 0.8237 - precision: 0.6388 - recall: 0.2393 172/385 [============>.................] - ETA: 40s - loss: 0.3916 - accuracy: 0.8180 - auc: 0.8241 - precision: 0.6393 - recall: 0.2410 173/385 [============>.................] - ETA: 40s - loss: 0.3912 - accuracy: 0.8182 - auc: 0.8245 - precision: 0.6399 - recall: 0.2427 174/385 [============>.................] - ETA: 40s - loss: 0.3908 - accuracy: 0.8184 - auc: 0.8248 - precision: 0.6402 - recall: 0.2442 175/385 [============>.................] - ETA: 39s - loss: 0.3904 - accuracy: 0.8186 - auc: 0.8253 - precision: 0.6408 - recall: 0.2458 176/385 [============>.................] - ETA: 39s - loss: 0.3900 - accuracy: 0.8188 - auc: 0.8257 - precision: 0.6414 - recall: 0.2475 177/385 [============>.................] - ETA: 39s - loss: 0.3896 - accuracy: 0.8190 - auc: 0.8261 - precision: 0.6419 - recall: 0.2490 178/385 [============>.................] - ETA: 39s - loss: 0.3893 - accuracy: 0.8192 - auc: 0.8264 - precision: 0.6424 - recall: 0.2504 179/385 [============>.................] - ETA: 39s - loss: 0.3890 - accuracy: 0.8195 - auc: 0.8268 - precision: 0.6433 - recall: 0.2519 180/385 [=============>................] - ETA: 38s - loss: 0.3886 - accuracy: 0.8197 - auc: 0.8272 - precision: 0.6440 - recall: 0.2535 181/385 [=============>................] - ETA: 38s - loss: 0.3883 - accuracy: 0.8199 - auc: 0.8275 - precision: 0.6447 - recall: 0.2550 182/385 [=============>................] - ETA: 38s - loss: 0.3879 - accuracy: 0.8201 - auc: 0.8279 - precision: 0.6451 - recall: 0.2565 183/385 [=============>................] - ETA: 38s - loss: 0.3876 - accuracy: 0.8203 - auc: 0.8283 - precision: 0.6459 - recall: 0.2582 184/385 [=============>................] - ETA: 38s - loss: 0.3872 - accuracy: 0.8205 - auc: 0.8287 - precision: 0.6466 - recall: 0.2596 185/385 [=============>................] - ETA: 37s - loss: 0.3869 - accuracy: 0.8207 - auc: 0.8290 - precision: 0.6470 - recall: 0.2609 186/385 [=============>................] - ETA: 37s - loss: 0.3866 - accuracy: 0.8209 - auc: 0.8293 - precision: 0.6475 - recall: 0.2623 187/385 [=============>................] - ETA: 37s - loss: 0.3862 - accuracy: 0.8211 - auc: 0.8297 - precision: 0.6482 - recall: 0.2637 188/385 [=============>................] - ETA: 37s - loss: 0.3859 - accuracy: 0.8213 - auc: 0.8301 - precision: 0.6487 - recall: 0.2650 189/385 [=============>................] - ETA: 37s - loss: 0.3855 - accuracy: 0.8215 - auc: 0.8304 - precision: 0.6491 - recall: 0.2664 190/385 [=============>................] - ETA: 37s - loss: 0.3852 - accuracy: 0.8217 - auc: 0.8308 - precision: 0.6497 - recall: 0.2679 191/385 [=============>................] - ETA: 36s - loss: 0.3849 - accuracy: 0.8219 - auc: 0.8311 - precision: 0.6503 - recall: 0.2694 192/385 [=============>................] - ETA: 36s - loss: 0.3845 - accuracy: 0.8221 - auc: 0.8315 - precision: 0.6508 - recall: 0.2708 193/385 [==============>...............] - ETA: 36s - loss: 0.3842 - accuracy: 0.8223 - auc: 0.8318 - precision: 0.6514 - recall: 0.2723 194/385 [==============>...............] - ETA: 36s - loss: 0.3838 - accuracy: 0.8225 - auc: 0.8322 - precision: 0.6517 - recall: 0.2737 195/385 [==============>...............] - ETA: 36s - loss: 0.3836 - accuracy: 0.8227 - auc: 0.8325 - precision: 0.6521 - recall: 0.2751 196/385 [==============>...............] - ETA: 35s - loss: 0.3832 - accuracy: 0.8229 - auc: 0.8328 - precision: 0.6524 - recall: 0.2765 197/385 [==============>...............] - ETA: 35s - loss: 0.3829 - accuracy: 0.8231 - auc: 0.8331 - precision: 0.6527 - recall: 0.2778 198/385 [==============>...............] - ETA: 35s - loss: 0.3826 - accuracy: 0.8232 - auc: 0.8334 - precision: 0.6533 - recall: 0.2791 199/385 [==============>...............] - ETA: 35s - loss: 0.3823 - accuracy: 0.8234 - auc: 0.8337 - precision: 0.6536 - recall: 0.2804 200/385 [==============>...............] - ETA: 35s - loss: 0.3820 - accuracy: 0.8236 - auc: 0.8340 - precision: 0.6540 - recall: 0.2816 201/385 [==============>...............] - ETA: 34s - loss: 0.3817 - accuracy: 0.8238 - auc: 0.8343 - precision: 0.6543 - recall: 0.2828 202/385 [==============>...............] - ETA: 34s - loss: 0.3814 - accuracy: 0.8240 - auc: 0.8346 - precision: 0.6547 - recall: 0.2841 203/385 [==============>...............] - ETA: 34s - loss: 0.3811 - accuracy: 0.8242 - auc: 0.8350 - precision: 0.6553 - recall: 0.2855 204/385 [==============>...............] - ETA: 34s - loss: 0.3807 - accuracy: 0.8244 - auc: 0.8353 - precision: 0.6559 - recall: 0.2867 205/385 [==============>...............] - ETA: 34s - loss: 0.3804 - accuracy: 0.8245 - auc: 0.8356 - precision: 0.6564 - recall: 0.2879 206/385 [===============>..............] - ETA: 34s - loss: 0.3801 - accuracy: 0.8247 - auc: 0.8360 - precision: 0.6570 - recall: 0.2891 207/385 [===============>..............] - ETA: 33s - loss: 0.3797 - accuracy: 0.8249 - auc: 0.8363 - precision: 0.6573 - recall: 0.2903 208/385 [===============>..............] - ETA: 33s - loss: 0.3794 - accuracy: 0.8251 - auc: 0.8366 - precision: 0.6577 - recall: 0.2915 209/385 [===============>..............] - ETA: 33s - loss: 0.3791 - accuracy: 0.8253 - auc: 0.8368 - precision: 0.6580 - recall: 0.2927 210/385 [===============>..............] - ETA: 33s - loss: 0.3789 - accuracy: 0.8255 - auc: 0.8371 - precision: 0.6584 - recall: 0.2939 211/385 [===============>..............] - ETA: 33s - loss: 0.3786 - accuracy: 0.8256 - auc: 0.8374 - precision: 0.6588 - recall: 0.2952 212/385 [===============>..............] - ETA: 32s - loss: 0.3783 - accuracy: 0.8258 - auc: 0.8377 - precision: 0.6591 - recall: 0.2964 213/385 [===============>..............] - ETA: 32s - loss: 0.3780 - accuracy: 0.8259 - auc: 0.8380 - precision: 0.6595 - recall: 0.2977 214/385 [===============>..............] - ETA: 32s - loss: 0.3777 - accuracy: 0.8261 - auc: 0.8383 - precision: 0.6597 - recall: 0.2988 215/385 [===============>..............] - ETA: 32s - loss: 0.3775 - accuracy: 0.8263 - auc: 0.8386 - precision: 0.6602 - recall: 0.3000 216/385 [===============>..............] - ETA: 32s - loss: 0.3771 - accuracy: 0.8264 - auc: 0.8389 - precision: 0.6607 - recall: 0.3012 217/385 [===============>..............] - ETA: 31s - loss: 0.3768 - accuracy: 0.8266 - auc: 0.8392 - precision: 0.6609 - recall: 0.3023 218/385 [===============>..............] - ETA: 31s - loss: 0.3766 - accuracy: 0.8267 - auc: 0.8394 - precision: 0.6613 - recall: 0.3034 219/385 [================>.............] - ETA: 31s - loss: 0.3763 - accuracy: 0.8269 - auc: 0.8397 - precision: 0.6615 - recall: 0.3046 220/385 [================>.............] - ETA: 31s - loss: 0.3760 - accuracy: 0.8271 - auc: 0.8400 - precision: 0.6619 - recall: 0.3057 221/385 [================>.............] - ETA: 31s - loss: 0.3757 - accuracy: 0.8272 - auc: 0.8402 - precision: 0.6623 - recall: 0.3068 222/385 [================>.............] - ETA: 31s - loss: 0.3755 - accuracy: 0.8274 - auc: 0.8405 - precision: 0.6626 - recall: 0.3079 223/385 [================>.............] - ETA: 30s - loss: 0.3751 - accuracy: 0.8275 - auc: 0.8408 - precision: 0.6629 - recall: 0.3091 224/385 [================>.............] - ETA: 30s - loss: 0.3749 - accuracy: 0.8277 - auc: 0.8411 - precision: 0.6632 - recall: 0.3102 225/385 [================>.............] - ETA: 30s - loss: 0.3746 - accuracy: 0.8279 - auc: 0.8414 - precision: 0.6637 - recall: 0.3113 226/385 [================>.............] - ETA: 30s - loss: 0.3743 - accuracy: 0.8280 - auc: 0.8417 - precision: 0.6640 - recall: 0.3123 227/385 [================>.............] - ETA: 30s - loss: 0.3741 - accuracy: 0.8281 - auc: 0.8419 - precision: 0.6643 - recall: 0.3133 228/385 [================>.............] - ETA: 29s - loss: 0.3738 - accuracy: 0.8283 - auc: 0.8422 - precision: 0.6647 - recall: 0.3144 229/385 [================>.............] - ETA: 29s - loss: 0.3735 - accuracy: 0.8284 - auc: 0.8424 - precision: 0.6650 - recall: 0.3154 230/385 [================>.............] - ETA: 29s - loss: 0.3733 - accuracy: 0.8286 - auc: 0.8426 - precision: 0.6652 - recall: 0.3164 231/385 [=================>............] - ETA: 29s - loss: 0.3730 - accuracy: 0.8288 - auc: 0.8429 - precision: 0.6656 - recall: 0.3175 232/385 [=================>............] - ETA: 29s - loss: 0.3727 - accuracy: 0.8289 - auc: 0.8432 - precision: 0.6660 - recall: 0.3186 233/385 [=================>............] - ETA: 29s - loss: 0.3725 - accuracy: 0.8290 - auc: 0.8434 - precision: 0.6662 - recall: 0.3196 234/385 [=================>............] - ETA: 28s - loss: 0.3722 - accuracy: 0.8292 - auc: 0.8437 - precision: 0.6667 - recall: 0.3206 235/385 [=================>............] - ETA: 28s - loss: 0.3719 - accuracy: 0.8294 - auc: 0.8440 - precision: 0.6670 - recall: 0.3216 236/385 [=================>............] - ETA: 28s - loss: 0.3716 - accuracy: 0.8296 - auc: 0.8443 - precision: 0.6675 - recall: 0.3228 237/385 [=================>............] - ETA: 28s - loss: 0.3714 - accuracy: 0.8297 - auc: 0.8445 - precision: 0.6679 - recall: 0.3237 238/385 [=================>............] - ETA: 28s - loss: 0.3711 - accuracy: 0.8299 - auc: 0.8448 - precision: 0.6683 - recall: 0.3248 239/385 [=================>............] - ETA: 27s - loss: 0.3708 - accuracy: 0.8300 - auc: 0.8450 - precision: 0.6685 - recall: 0.3256 240/385 [=================>............] - ETA: 27s - loss: 0.3705 - accuracy: 0.8302 - auc: 0.8453 - precision: 0.6688 - recall: 0.3266 241/385 [=================>............] - ETA: 27s - loss: 0.3703 - accuracy: 0.8303 - auc: 0.8455 - precision: 0.6692 - recall: 0.3277 242/385 [=================>............] - ETA: 27s - loss: 0.3701 - accuracy: 0.8305 - auc: 0.8458 - precision: 0.6695 - recall: 0.3287 243/385 [=================>............] - ETA: 27s - loss: 0.3698 - accuracy: 0.8306 - auc: 0.8460 - precision: 0.6698 - recall: 0.3298 244/385 [==================>...........] - ETA: 27s - loss: 0.3696 - accuracy: 0.8308 - auc: 0.8462 - precision: 0.6700 - recall: 0.3308 245/385 [==================>...........] - ETA: 26s - loss: 0.3694 - accuracy: 0.8309 - auc: 0.8465 - precision: 0.6703 - recall: 0.3318 246/385 [==================>...........] - ETA: 26s - loss: 0.3691 - accuracy: 0.8311 - auc: 0.8468 - precision: 0.6707 - recall: 0.3328 247/385 [==================>...........] - ETA: 26s - loss: 0.3688 - accuracy: 0.8312 - auc: 0.8470 - precision: 0.6709 - recall: 0.3337 248/385 [==================>...........] - ETA: 26s - loss: 0.3686 - accuracy: 0.8313 - auc: 0.8472 - precision: 0.6710 - recall: 0.3345 249/385 [==================>...........] - ETA: 26s - loss: 0.3684 - accuracy: 0.8314 - auc: 0.8474 - precision: 0.6713 - recall: 0.3354 250/385 [==================>...........] - ETA: 25s - loss: 0.3681 - accuracy: 0.8316 - auc: 0.8476 - precision: 0.6715 - recall: 0.3363 251/385 [==================>...........] - ETA: 25s - loss: 0.3679 - accuracy: 0.8317 - auc: 0.8479 - precision: 0.6719 - recall: 0.3371 252/385 [==================>...........] - ETA: 25s - loss: 0.3676 - accuracy: 0.8319 - auc: 0.8481 - precision: 0.6722 - recall: 0.3380 253/385 [==================>...........] - ETA: 25s - loss: 0.3674 - accuracy: 0.8320 - auc: 0.8483 - precision: 0.6725 - recall: 0.3390 254/385 [==================>...........] - ETA: 25s - loss: 0.3672 - accuracy: 0.8321 - auc: 0.8486 - precision: 0.6728 - recall: 0.3399 255/385 [==================>...........] - ETA: 24s - loss: 0.3669 - accuracy: 0.8323 - auc: 0.8488 - precision: 0.6729 - recall: 0.3408 256/385 [==================>...........] - ETA: 24s - loss: 0.3667 - accuracy: 0.8324 - auc: 0.8490 - precision: 0.6733 - recall: 0.3417 257/385 [===================>..........] - ETA: 24s - loss: 0.3664 - accuracy: 0.8325 - auc: 0.8493 - precision: 0.6736 - recall: 0.3426 258/385 [===================>..........] - ETA: 24s - loss: 0.3662 - accuracy: 0.8327 - auc: 0.8495 - precision: 0.6740 - recall: 0.3436 259/385 [===================>..........] - ETA: 24s - loss: 0.3659 - accuracy: 0.8328 - auc: 0.8498 - precision: 0.6742 - recall: 0.3444 260/385 [===================>..........] - ETA: 23s - loss: 0.3656 - accuracy: 0.8330 - auc: 0.8500 - precision: 0.6745 - recall: 0.3453 261/385 [===================>..........] - ETA: 23s - loss: 0.3655 - accuracy: 0.8331 - auc: 0.8502 - precision: 0.6747 - recall: 0.3461 262/385 [===================>..........] - ETA: 23s - loss: 0.3652 - accuracy: 0.8332 - auc: 0.8504 - precision: 0.6750 - recall: 0.3470 263/385 [===================>..........] - ETA: 23s - loss: 0.3650 - accuracy: 0.8333 - auc: 0.8506 - precision: 0.6751 - recall: 0.3479 264/385 [===================>..........] - ETA: 23s - loss: 0.3647 - accuracy: 0.8335 - auc: 0.8508 - precision: 0.6754 - recall: 0.3488 265/385 [===================>..........] - ETA: 23s - loss: 0.3645 - accuracy: 0.8336 - auc: 0.8511 - precision: 0.6756 - recall: 0.3496 266/385 [===================>..........] - ETA: 22s - loss: 0.3642 - accuracy: 0.8337 - auc: 0.8513 - precision: 0.6759 - recall: 0.3506 267/385 [===================>..........] - ETA: 22s - loss: 0.3640 - accuracy: 0.8339 - auc: 0.8515 - precision: 0.6763 - recall: 0.3514 268/385 [===================>..........] - ETA: 22s - loss: 0.3638 - accuracy: 0.8340 - auc: 0.8518 - precision: 0.6765 - recall: 0.3523 269/385 [===================>..........] - ETA: 22s - loss: 0.3636 - accuracy: 0.8341 - auc: 0.8520 - precision: 0.6768 - recall: 0.3532 270/385 [====================>.........] - ETA: 22s - loss: 0.3633 - accuracy: 0.8343 - auc: 0.8522 - precision: 0.6771 - recall: 0.3540 271/385 [====================>.........] - ETA: 21s - loss: 0.3631 - accuracy: 0.8344 - auc: 0.8524 - precision: 0.6773 - recall: 0.3549 272/385 [====================>.........] - ETA: 21s - loss: 0.3629 - accuracy: 0.8345 - auc: 0.8526 - precision: 0.6775 - recall: 0.3558 273/385 [====================>.........] - ETA: 21s - loss: 0.3627 - accuracy: 0.8347 - auc: 0.8528 - precision: 0.6778 - recall: 0.3567 274/385 [====================>.........] - ETA: 21s - loss: 0.3625 - accuracy: 0.8348 - auc: 0.8530 - precision: 0.6780 - recall: 0.3575 275/385 [====================>.........] - ETA: 21s - loss: 0.3623 - accuracy: 0.8349 - auc: 0.8532 - precision: 0.6782 - recall: 0.3584 276/385 [====================>.........] - ETA: 21s - loss: 0.3620 - accuracy: 0.8350 - auc: 0.8534 - precision: 0.6784 - recall: 0.3592 277/385 [====================>.........] - ETA: 20s - loss: 0.3618 - accuracy: 0.8351 - auc: 0.8536 - precision: 0.6786 - recall: 0.3600 278/385 [====================>.........] - ETA: 20s - loss: 0.3616 - accuracy: 0.8352 - auc: 0.8538 - precision: 0.6788 - recall: 0.3608 279/385 [====================>.........] - ETA: 20s - loss: 0.3614 - accuracy: 0.8353 - auc: 0.8540 - precision: 0.6790 - recall: 0.3616 280/385 [====================>.........] - ETA: 20s - loss: 0.3611 - accuracy: 0.8355 - auc: 0.8542 - precision: 0.6793 - recall: 0.3624 281/385 [====================>.........] - ETA: 20s - loss: 0.3609 - accuracy: 0.8356 - auc: 0.8545 - precision: 0.6796 - recall: 0.3632 282/385 [====================>.........] - ETA: 19s - loss: 0.3607 - accuracy: 0.8357 - auc: 0.8547 - precision: 0.6798 - recall: 0.3640 283/385 [=====================>........] - ETA: 19s - loss: 0.3605 - accuracy: 0.8358 - auc: 0.8549 - precision: 0.6799 - recall: 0.3648 284/385 [=====================>........] - ETA: 19s - loss: 0.3602 - accuracy: 0.8360 - auc: 0.8551 - precision: 0.6802 - recall: 0.3656 285/385 [=====================>........] - ETA: 19s - loss: 0.3600 - accuracy: 0.8361 - auc: 0.8553 - precision: 0.6805 - recall: 0.3664 286/385 [=====================>........] - ETA: 19s - loss: 0.3598 - accuracy: 0.8362 - auc: 0.8555 - precision: 0.6808 - recall: 0.3671 287/385 [=====================>........] - ETA: 18s - loss: 0.3595 - accuracy: 0.8364 - auc: 0.8557 - precision: 0.6810 - recall: 0.3679 288/385 [=====================>........] - ETA: 18s - loss: 0.3593 - accuracy: 0.8365 - auc: 0.8559 - precision: 0.6812 - recall: 0.3686 289/385 [=====================>........] - ETA: 18s - loss: 0.3591 - accuracy: 0.8366 - auc: 0.8561 - precision: 0.6814 - recall: 0.3693 290/385 [=====================>........] - ETA: 18s - loss: 0.3590 - accuracy: 0.8367 - auc: 0.8563 - precision: 0.6816 - recall: 0.3701 291/385 [=====================>........] - ETA: 18s - loss: 0.3587 - accuracy: 0.8368 - auc: 0.8565 - precision: 0.6819 - recall: 0.3709 292/385 [=====================>........] - ETA: 17s - loss: 0.3586 - accuracy: 0.8369 - auc: 0.8567 - precision: 0.6821 - recall: 0.3717 293/385 [=====================>........] - ETA: 17s - loss: 0.3583 - accuracy: 0.8370 - auc: 0.8569 - precision: 0.6823 - recall: 0.3725 294/385 [=====================>........] - ETA: 17s - loss: 0.3581 - accuracy: 0.8372 - auc: 0.8571 - precision: 0.6827 - recall: 0.3733 295/385 [=====================>........] - ETA: 17s - loss: 0.3579 - accuracy: 0.8373 - auc: 0.8572 - precision: 0.6828 - recall: 0.3739 296/385 [======================>.......] - ETA: 17s - loss: 0.3578 - accuracy: 0.8374 - auc: 0.8574 - precision: 0.6830 - recall: 0.3746 297/385 [======================>.......] - ETA: 16s - loss: 0.3576 - accuracy: 0.8375 - auc: 0.8576 - precision: 0.6832 - recall: 0.3753 298/385 [======================>.......] - ETA: 16s - loss: 0.3573 - accuracy: 0.8376 - auc: 0.8578 - precision: 0.6834 - recall: 0.3760 299/385 [======================>.......] - ETA: 16s - loss: 0.3571 - accuracy: 0.8377 - auc: 0.8580 - precision: 0.6835 - recall: 0.3766 300/385 [======================>.......] - ETA: 16s - loss: 0.3569 - accuracy: 0.8378 - auc: 0.8582 - precision: 0.6837 - recall: 0.3773 301/385 [======================>.......] - ETA: 16s - loss: 0.3567 - accuracy: 0.8380 - auc: 0.8583 - precision: 0.6839 - recall: 0.3780 302/385 [======================>.......] - ETA: 16s - loss: 0.3564 - accuracy: 0.8381 - auc: 0.8585 - precision: 0.6842 - recall: 0.3787 303/385 [======================>.......] - ETA: 15s - loss: 0.3562 - accuracy: 0.8382 - auc: 0.8587 - precision: 0.6844 - recall: 0.3793 304/385 [======================>.......] - ETA: 15s - loss: 0.3560 - accuracy: 0.8383 - auc: 0.8589 - precision: 0.6847 - recall: 0.3800 305/385 [======================>.......] - ETA: 15s - loss: 0.3558 - accuracy: 0.8384 - auc: 0.8591 - precision: 0.6847 - recall: 0.3807 306/385 [======================>.......] - ETA: 15s - loss: 0.3556 - accuracy: 0.8385 - auc: 0.8592 - precision: 0.6849 - recall: 0.3814 307/385 [======================>.......] - ETA: 15s - loss: 0.3555 - accuracy: 0.8386 - auc: 0.8594 - precision: 0.6851 - recall: 0.3821 308/385 [=======================>......] - ETA: 14s - loss: 0.3553 - accuracy: 0.8387 - auc: 0.8596 - precision: 0.6853 - recall: 0.3828 309/385 [=======================>......] - ETA: 14s - loss: 0.3551 - accuracy: 0.8388 - auc: 0.8598 - precision: 0.6855 - recall: 0.3836 310/385 [=======================>......] - ETA: 14s - loss: 0.3549 - accuracy: 0.8389 - auc: 0.8600 - precision: 0.6856 - recall: 0.3843 311/385 [=======================>......] - ETA: 14s - loss: 0.3547 - accuracy: 0.8390 - auc: 0.8601 - precision: 0.6857 - recall: 0.3849 312/385 [=======================>......] - ETA: 14s - loss: 0.3545 - accuracy: 0.8391 - auc: 0.8603 - precision: 0.6859 - recall: 0.3856 313/385 [=======================>......] - ETA: 13s - loss: 0.3544 - accuracy: 0.8392 - auc: 0.8604 - precision: 0.6860 - recall: 0.3863 314/385 [=======================>......] - ETA: 13s - loss: 0.3541 - accuracy: 0.8393 - auc: 0.8606 - precision: 0.6862 - recall: 0.3870 315/385 [=======================>......] - ETA: 13s - loss: 0.3540 - accuracy: 0.8394 - auc: 0.8608 - precision: 0.6864 - recall: 0.3876 316/385 [=======================>......] - ETA: 13s - loss: 0.3537 - accuracy: 0.8395 - auc: 0.8610 - precision: 0.6867 - recall: 0.3883 317/385 [=======================>......] - ETA: 13s - loss: 0.3536 - accuracy: 0.8396 - auc: 0.8611 - precision: 0.6869 - recall: 0.3889 318/385 [=======================>......] - ETA: 12s - loss: 0.3534 - accuracy: 0.8397 - auc: 0.8613 - precision: 0.6871 - recall: 0.3895 319/385 [=======================>......] - ETA: 12s - loss: 0.3532 - accuracy: 0.8398 - auc: 0.8615 - precision: 0.6873 - recall: 0.3902 320/385 [=======================>......] - ETA: 12s - loss: 0.3530 - accuracy: 0.8399 - auc: 0.8617 - precision: 0.6875 - recall: 0.3908 321/385 [========================>.....] - ETA: 12s - loss: 0.3528 - accuracy: 0.8400 - auc: 0.8618 - precision: 0.6877 - recall: 0.3914 322/385 [========================>.....] - ETA: 12s - loss: 0.3526 - accuracy: 0.8401 - auc: 0.8620 - precision: 0.6878 - recall: 0.3920 323/385 [========================>.....] - ETA: 11s - loss: 0.3525 - accuracy: 0.8402 - auc: 0.8621 - precision: 0.6880 - recall: 0.3926 324/385 [========================>.....] - ETA: 11s - loss: 0.3523 - accuracy: 0.8403 - auc: 0.8623 - precision: 0.6881 - recall: 0.3932 325/385 [========================>.....] - ETA: 11s - loss: 0.3521 - accuracy: 0.8404 - auc: 0.8625 - precision: 0.6884 - recall: 0.3938 326/385 [========================>.....] - ETA: 11s - loss: 0.3519 - accuracy: 0.8405 - auc: 0.8626 - precision: 0.6885 - recall: 0.3944 327/385 [========================>.....] - ETA: 11s - loss: 0.3517 - accuracy: 0.8406 - auc: 0.8628 - precision: 0.6887 - recall: 0.3951 328/385 [========================>.....] - ETA: 10s - loss: 0.3515 - accuracy: 0.8407 - auc: 0.8630 - precision: 0.6889 - recall: 0.3957 329/385 [========================>.....] - ETA: 10s - loss: 0.3514 - accuracy: 0.8408 - auc: 0.8631 - precision: 0.6891 - recall: 0.3963 330/385 [========================>.....] - ETA: 10s - loss: 0.3512 - accuracy: 0.8409 - auc: 0.8633 - precision: 0.6893 - recall: 0.3969 331/385 [========================>.....] - ETA: 10s - loss: 0.3510 - accuracy: 0.8410 - auc: 0.8634 - precision: 0.6895 - recall: 0.3975 332/385 [========================>.....] - ETA: 10s - loss: 0.3508 - accuracy: 0.8411 - auc: 0.8636 - precision: 0.6896 - recall: 0.3981 333/385 [========================>.....] - ETA: 10s - loss: 0.3507 - accuracy: 0.8412 - auc: 0.8637 - precision: 0.6898 - recall: 0.3988 334/385 [=========================>....] - ETA: 9s - loss: 0.3505 - accuracy: 0.8413 - auc: 0.8639 - precision: 0.6900 - recall: 0.3994 335/385 [=========================>....] - ETA: 9s - loss: 0.3503 - accuracy: 0.8414 - auc: 0.8641 - precision: 0.6902 - recall: 0.4001 336/385 [=========================>....] - ETA: 9s - loss: 0.3502 - accuracy: 0.8415 - auc: 0.8642 - precision: 0.6903 - recall: 0.4006 337/385 [=========================>....] - ETA: 9s - loss: 0.3500 - accuracy: 0.8416 - auc: 0.8644 - precision: 0.6904 - recall: 0.4012 338/385 [=========================>....] - ETA: 9s - loss: 0.3498 - accuracy: 0.8417 - auc: 0.8645 - precision: 0.6906 - recall: 0.4017 339/385 [=========================>....] - ETA: 8s - loss: 0.3497 - accuracy: 0.8417 - auc: 0.8647 - precision: 0.6907 - recall: 0.4023 340/385 [=========================>....] - ETA: 8s - loss: 0.3495 - accuracy: 0.8418 - auc: 0.8648 - precision: 0.6909 - recall: 0.4029 341/385 [=========================>....] - ETA: 8s - loss: 0.3493 - accuracy: 0.8419 - auc: 0.8650 - precision: 0.6912 - recall: 0.4035 342/385 [=========================>....] - ETA: 8s - loss: 0.3491 - accuracy: 0.8420 - auc: 0.8651 - precision: 0.6913 - recall: 0.4041 343/385 [=========================>....] - ETA: 8s - loss: 0.3489 - accuracy: 0.8421 - auc: 0.8653 - precision: 0.6914 - recall: 0.4047 344/385 [=========================>....] - ETA: 7s - loss: 0.3487 - accuracy: 0.8422 - auc: 0.8654 - precision: 0.6916 - recall: 0.4052 345/385 [=========================>....] - ETA: 7s - loss: 0.3486 - accuracy: 0.8423 - auc: 0.8656 - precision: 0.6917 - recall: 0.4058 346/385 [=========================>....] - ETA: 7s - loss: 0.3484 - accuracy: 0.8424 - auc: 0.8657 - precision: 0.6918 - recall: 0.4063 347/385 [==========================>...] - ETA: 7s - loss: 0.3483 - accuracy: 0.8425 - auc: 0.8658 - precision: 0.6919 - recall: 0.4068 348/385 [==========================>...] - ETA: 7s - loss: 0.3481 - accuracy: 0.8426 - auc: 0.8660 - precision: 0.6920 - recall: 0.4073 349/385 [==========================>...] - ETA: 6s - loss: 0.3480 - accuracy: 0.8426 - auc: 0.8661 - precision: 0.6922 - recall: 0.4079 350/385 [==========================>...] - ETA: 6s - loss: 0.3478 - accuracy: 0.8427 - auc: 0.8663 - precision: 0.6923 - recall: 0.4084 351/385 [==========================>...] - ETA: 6s - loss: 0.3476 - accuracy: 0.8428 - auc: 0.8664 - precision: 0.6925 - recall: 0.4089 352/385 [==========================>...] - ETA: 6s - loss: 0.3474 - accuracy: 0.8429 - auc: 0.8665 - precision: 0.6926 - recall: 0.4094 353/385 [==========================>...] - ETA: 6s - loss: 0.3473 - accuracy: 0.8430 - auc: 0.8667 - precision: 0.6928 - recall: 0.4099 354/385 [==========================>...] - ETA: 5s - loss: 0.3471 - accuracy: 0.8431 - auc: 0.8669 - precision: 0.6930 - recall: 0.4104 355/385 [==========================>...] - ETA: 5s - loss: 0.3469 - accuracy: 0.8432 - auc: 0.8670 - precision: 0.6932 - recall: 0.4109 356/385 [==========================>...] - ETA: 5s - loss: 0.3468 - accuracy: 0.8433 - auc: 0.8671 - precision: 0.6933 - recall: 0.4115 357/385 [==========================>...] - ETA: 5s - loss: 0.3466 - accuracy: 0.8433 - auc: 0.8673 - precision: 0.6934 - recall: 0.4120 358/385 [==========================>...] - ETA: 5s - loss: 0.3465 - accuracy: 0.8434 - auc: 0.8674 - precision: 0.6935 - recall: 0.4125 359/385 [==========================>...] - ETA: 5s - loss: 0.3463 - accuracy: 0.8435 - auc: 0.8675 - precision: 0.6936 - recall: 0.4131 360/385 [===========================>..] - ETA: 4s - loss: 0.3462 - accuracy: 0.8436 - auc: 0.8676 - precision: 0.6937 - recall: 0.4136 361/385 [===========================>..] - ETA: 4s - loss: 0.3460 - accuracy: 0.8437 - auc: 0.8678 - precision: 0.6938 - recall: 0.4141 362/385 [===========================>..] - ETA: 4s - loss: 0.3459 - accuracy: 0.8437 - auc: 0.8679 - precision: 0.6940 - recall: 0.4146 363/385 [===========================>..] - ETA: 4s - loss: 0.3457 - accuracy: 0.8438 - auc: 0.8681 - precision: 0.6941 - recall: 0.4152 364/385 [===========================>..] - ETA: 4s - loss: 0.3456 - accuracy: 0.8439 - auc: 0.8682 - precision: 0.6942 - recall: 0.4157 365/385 [===========================>..] - ETA: 3s - loss: 0.3454 - accuracy: 0.8440 - auc: 0.8683 - precision: 0.6944 - recall: 0.4162 366/385 [===========================>..] - ETA: 3s - loss: 0.3453 - accuracy: 0.8440 - auc: 0.8684 - precision: 0.6944 - recall: 0.4167 367/385 [===========================>..] - ETA: 3s - loss: 0.3452 - accuracy: 0.8441 - auc: 0.8685 - precision: 0.6946 - recall: 0.4172 368/385 [===========================>..] - ETA: 3s - loss: 0.3450 - accuracy: 0.8442 - auc: 0.8687 - precision: 0.6947 - recall: 0.4177 369/385 [===========================>..] - ETA: 3s - loss: 0.3448 - accuracy: 0.8443 - auc: 0.8688 - precision: 0.6948 - recall: 0.4182 370/385 [===========================>..] - ETA: 2s - loss: 0.3447 - accuracy: 0.8444 - auc: 0.8690 - precision: 0.6950 - recall: 0.4187 371/385 [===========================>..] - ETA: 2s - loss: 0.3445 - accuracy: 0.8444 - auc: 0.8691 - precision: 0.6952 - recall: 0.4193 372/385 [===========================>..] - ETA: 2s - loss: 0.3444 - accuracy: 0.8445 - auc: 0.8692 - precision: 0.6953 - recall: 0.4197 373/385 [============================>.] - ETA: 2s - loss: 0.3442 - accuracy: 0.8446 - auc: 0.8693 - precision: 0.6954 - recall: 0.4202 374/385 [============================>.] - ETA: 2s - loss: 0.3441 - accuracy: 0.8447 - auc: 0.8695 - precision: 0.6955 - recall: 0.4207 375/385 [============================>.] - ETA: 1s - loss: 0.3439 - accuracy: 0.8448 - auc: 0.8696 - precision: 0.6957 - recall: 0.4212 376/385 [============================>.] - ETA: 1s - loss: 0.3438 - accuracy: 0.8448 - auc: 0.8698 - precision: 0.6958 - recall: 0.4217 377/385 [============================>.] - ETA: 1s - loss: 0.3436 - accuracy: 0.8449 - auc: 0.8699 - precision: 0.6959 - recall: 0.4222 378/385 [============================>.] - ETA: 1s - loss: 0.3435 - accuracy: 0.8450 - auc: 0.8700 - precision: 0.6960 - recall: 0.4226 379/385 [============================>.] - ETA: 1s - loss: 0.3433 - accuracy: 0.8451 - auc: 0.8701 - precision: 0.6961 - recall: 0.4231 380/385 [============================>.] - ETA: 0s - loss: 0.3432 - accuracy: 0.8452 - auc: 0.8703 - precision: 0.6963 - recall: 0.4236 381/385 [============================>.] - ETA: 0s - loss: 0.3430 - accuracy: 0.8452 - auc: 0.8704 - precision: 0.6964 - recall: 0.4240 382/385 [============================>.] - ETA: 0s - loss: 0.3429 - accuracy: 0.8453 - auc: 0.8705 - precision: 0.6966 - recall: 0.4245 383/385 [============================>.] - ETA: 0s - loss: 0.3427 - accuracy: 0.8454 - auc: 0.8707 - precision: 0.6967 - recall: 0.4250 384/385 [============================>.] - ETA: 0s - loss: 0.3425 - accuracy: 0.8455 - auc: 0.8708 - precision: 0.6968 - recall: 0.4254 385/385 [==============================] - ETA: 0s - loss: 0.3424 - accuracy: 0.8455 - auc: 0.8709 - precision: 0.6970 - recall: 0.4258 (Worker pid=1347) 2023-02-13 15:38:04.041709: W tensorflow/core/framework/dataset.cc:769] Input of GeneratorDatasetOp::Dataset will not be optimized because the dataset does not implement the AsGraphDefInternal() method needed to apply optimizations. 385/385 [==============================] - 83s 210ms/step - loss: 0.3424 - accuracy: 0.8455 - auc: 0.8709 - precision: 0.6970 - recall: 0.4258 - val_loss: 0.2736 - val_accuracy: 0.8797 - val_auc: 0.9229 - val_precision: 0.7403 - val_recall: 0.6270 (Worker pid=1347) Epoch 2/2 1/385 [..............................] - ETA: 14s - loss: 0.2812 - accuracy: 0.8746 - auc: 0.9165 - precision: 0.7242 - recall: 0.6010 3/385 [..............................] - ETA: 18s - loss: 0.2888 - accuracy: 0.8727 - auc: 0.9119 - precision: 0.7258 - recall: 0.5951 4/385 [..............................] - ETA: 19s - loss: 0.2865 - accuracy: 0.8733 - auc: 0.9141 - precision: 0.7306 - recall: 0.6004 5/385 [..............................] - ETA: 26s - loss: 0.2863 - accuracy: 0.8742 - auc: 0.9143 - precision: 0.7329 - recall: 0.6056 6/385 [..............................] - ETA: 35s - loss: 0.2860 - accuracy: 0.8742 - auc: 0.9151 - precision: 0.7339 - recall: 0.6082 7/385 [..............................] - ETA: 41s - loss: 0.2849 - accuracy: 0.8747 - auc: 0.9154 - precision: 0.7346 - recall: 0.6069 8/385 [..............................] - ETA: 46s - loss: 0.2838 - accuracy: 0.8755 - auc: 0.9160 - precision: 0.7359 - recall: 0.6103 9/385 [..............................] - ETA: 49s - loss: 0.2836 - accuracy: 0.8758 - auc: 0.9161 - precision: 0.7366 - recall: 0.6109 10/385 [..............................] - ETA: 51s - loss: 0.2838 - accuracy: 0.8758 - auc: 0.9159 - precision: 0.7363 - recall: 0.6100 11/385 [..............................] - ETA: 53s - loss: 0.2841 - accuracy: 0.8754 - auc: 0.9158 - precision: 0.7355 - recall: 0.6092 12/385 [..............................] - ETA: 55s - loss: 0.2837 - accuracy: 0.8752 - auc: 0.9162 - precision: 0.7352 - recall: 0.6094 13/385 [>.............................] - ETA: 56s - loss: 0.2842 - accuracy: 0.8749 - auc: 0.9162 - precision: 0.7353 - recall: 0.6095 14/385 [>.............................] - ETA: 57s - loss: 0.2846 - accuracy: 0.8745 - auc: 0.9159 - precision: 0.7333 - recall: 0.6085 15/385 [>.............................] - ETA: 58s - loss: 0.2842 - accuracy: 0.8749 - auc: 0.9159 - precision: 0.7325 - recall: 0.6098 16/385 [>.............................] - ETA: 58s - loss: 0.2841 - accuracy: 0.8749 - auc: 0.9158 - precision: 0.7321 - recall: 0.6085 17/385 [>.............................] - ETA: 59s - loss: 0.2842 - accuracy: 0.8750 - auc: 0.9156 - precision: 0.7332 - recall: 0.6082 18/385 [>.............................] - ETA: 59s - loss: 0.2843 - accuracy: 0.8751 - auc: 0.9156 - precision: 0.7331 - recall: 0.6080 19/385 [>.............................] - ETA: 1:00 - loss: 0.2842 - accuracy: 0.8752 - auc: 0.9157 - precision: 0.7346 - recall: 0.6078 20/385 [>.............................] - ETA: 1:00 - loss: 0.2838 - accuracy: 0.8754 - auc: 0.9159 - precision: 0.7352 - recall: 0.6075 21/385 [>.............................] - ETA: 1:00 - loss: 0.2838 - accuracy: 0.8756 - auc: 0.9158 - precision: 0.7352 - recall: 0.6080 22/385 [>.............................] - ETA: 1:00 - loss: 0.2839 - accuracy: 0.8755 - auc: 0.9159 - precision: 0.7355 - recall: 0.6085 23/385 [>.............................] - ETA: 1:01 - loss: 0.2838 - accuracy: 0.8757 - auc: 0.9160 - precision: 0.7361 - recall: 0.6093 24/385 [>.............................] - ETA: 1:01 - loss: 0.2834 - accuracy: 0.8760 - auc: 0.9161 - precision: 0.7362 - recall: 0.6103 25/385 [>.............................] - ETA: 1:01 - loss: 0.2836 - accuracy: 0.8760 - auc: 0.9160 - precision: 0.7361 - recall: 0.6106 26/385 [=>............................] - ETA: 1:01 - loss: 0.2834 - accuracy: 0.8762 - auc: 0.9160 - precision: 0.7357 - recall: 0.6111 27/385 [=>............................] - ETA: 1:01 - loss: 0.2832 - accuracy: 0.8762 - auc: 0.9161 - precision: 0.7358 - recall: 0.6112 28/385 [=>............................] - ETA: 1:01 - loss: 0.2829 - accuracy: 0.8763 - auc: 0.9163 - precision: 0.7360 - recall: 0.6116 29/385 [=>............................] - ETA: 1:01 - loss: 0.2828 - accuracy: 0.8764 - auc: 0.9163 - precision: 0.7363 - recall: 0.6110 30/385 [=>............................] - ETA: 1:01 - loss: 0.2828 - accuracy: 0.8764 - auc: 0.9163 - precision: 0.7368 - recall: 0.6113 31/385 [=>............................] - ETA: 1:01 - loss: 0.2828 - accuracy: 0.8764 - auc: 0.9163 - precision: 0.7367 - recall: 0.6107 32/385 [=>............................] - ETA: 1:01 - loss: 0.2823 - accuracy: 0.8766 - auc: 0.9167 - precision: 0.7378 - recall: 0.6113 33/385 [=>............................] - ETA: 1:01 - loss: 0.2825 - accuracy: 0.8764 - auc: 0.9165 - precision: 0.7372 - recall: 0.6109 34/385 [=>............................] - ETA: 1:01 - loss: 0.2822 - accuracy: 0.8766 - auc: 0.9167 - precision: 0.7373 - recall: 0.6117 35/385 [=>............................] - ETA: 1:01 - loss: 0.2822 - accuracy: 0.8766 - auc: 0.9167 - precision: 0.7373 - recall: 0.6117 36/385 [=>............................] - ETA: 1:01 - loss: 0.2821 - accuracy: 0.8766 - auc: 0.9168 - precision: 0.7371 - recall: 0.6121 37/385 [=>............................] - ETA: 1:01 - loss: 0.2820 - accuracy: 0.8766 - auc: 0.9168 - precision: 0.7366 - recall: 0.6121 38/385 [=>............................] - ETA: 1:01 - loss: 0.2818 - accuracy: 0.8767 - auc: 0.9170 - precision: 0.7368 - recall: 0.6131 39/385 [==>...........................] - ETA: 1:01 - loss: 0.2817 - accuracy: 0.8768 - auc: 0.9170 - precision: 0.7368 - recall: 0.6135 40/385 [==>...........................] - ETA: 1:01 - loss: 0.2817 - accuracy: 0.8769 - auc: 0.9171 - precision: 0.7376 - recall: 0.6134 41/385 [==>...........................] - ETA: 1:01 - loss: 0.2816 - accuracy: 0.8770 - auc: 0.9172 - precision: 0.7379 - recall: 0.6138 42/385 [==>...........................] - ETA: 1:01 - loss: 0.2815 - accuracy: 0.8771 - auc: 0.9172 - precision: 0.7380 - recall: 0.6139 43/385 [==>...........................] - ETA: 1:01 - loss: 0.2814 - accuracy: 0.8771 - auc: 0.9173 - precision: 0.7382 - recall: 0.6136 44/385 [==>...........................] - ETA: 1:01 - loss: 0.2815 - accuracy: 0.8771 - auc: 0.9172 - precision: 0.7381 - recall: 0.6137 45/385 [==>...........................] - ETA: 1:01 - loss: 0.2815 - accuracy: 0.8771 - auc: 0.9172 - precision: 0.7380 - recall: 0.6135 46/385 [==>...........................] - ETA: 1:01 - loss: 0.2817 - accuracy: 0.8769 - auc: 0.9171 - precision: 0.7377 - recall: 0.6132 47/385 [==>...........................] - ETA: 1:00 - loss: 0.2817 - accuracy: 0.8770 - auc: 0.9171 - precision: 0.7378 - recall: 0.6132 48/385 [==>...........................] - ETA: 1:00 - loss: 0.2816 - accuracy: 0.8770 - auc: 0.9171 - precision: 0.7379 - recall: 0.6130 49/385 [==>...........................] - ETA: 1:00 - loss: 0.2817 - accuracy: 0.8770 - auc: 0.9172 - precision: 0.7381 - recall: 0.6131 50/385 [==>...........................] - ETA: 1:00 - loss: 0.2816 - accuracy: 0.8769 - auc: 0.9172 - precision: 0.7378 - recall: 0.6129 51/385 [==>...........................] - ETA: 1:00 - loss: 0.2815 - accuracy: 0.8770 - auc: 0.9173 - precision: 0.7379 - recall: 0.6133 52/385 [===>..........................] - ETA: 1:00 - loss: 0.2813 - accuracy: 0.8770 - auc: 0.9173 - precision: 0.7379 - recall: 0.6134 53/385 [===>..........................] - ETA: 1:00 - loss: 0.2812 - accuracy: 0.8771 - auc: 0.9174 - precision: 0.7382 - recall: 0.6136 54/385 [===>..........................] - ETA: 1:00 - loss: 0.2810 - accuracy: 0.8772 - auc: 0.9175 - precision: 0.7382 - recall: 0.6137 55/385 [===>..........................] - ETA: 59s - loss: 0.2811 - accuracy: 0.8772 - auc: 0.9175 - precision: 0.7382 - recall: 0.6137 56/385 [===>..........................] - ETA: 59s - loss: 0.2811 - accuracy: 0.8771 - auc: 0.9174 - precision: 0.7379 - recall: 0.6136 57/385 [===>..........................] - ETA: 59s - loss: 0.2810 - accuracy: 0.8771 - auc: 0.9175 - precision: 0.7376 - recall: 0.6137 58/385 [===>..........................] - ETA: 59s - loss: 0.2809 - accuracy: 0.8772 - auc: 0.9176 - precision: 0.7377 - recall: 0.6141 59/385 [===>..........................] - ETA: 59s - loss: 0.2809 - accuracy: 0.8772 - auc: 0.9175 - precision: 0.7374 - recall: 0.6142 60/385 [===>..........................] - ETA: 59s - loss: 0.2809 - accuracy: 0.8772 - auc: 0.9176 - precision: 0.7375 - recall: 0.6143 61/385 [===>..........................] - ETA: 59s - loss: 0.2809 - accuracy: 0.8772 - auc: 0.9176 - precision: 0.7377 - recall: 0.6143 62/385 [===>..........................] - ETA: 59s - loss: 0.2807 - accuracy: 0.8773 - auc: 0.9176 - precision: 0.7378 - recall: 0.6141 63/385 [===>..........................] - ETA: 58s - loss: 0.2805 - accuracy: 0.8775 - auc: 0.9178 - precision: 0.7383 - recall: 0.6146 64/385 [===>..........................] - ETA: 58s - loss: 0.2805 - accuracy: 0.8775 - auc: 0.9177 - precision: 0.7383 - recall: 0.6145 65/385 [====>.........................] - ETA: 58s - loss: 0.2805 - accuracy: 0.8774 - auc: 0.9177 - precision: 0.7379 - recall: 0.6143 66/385 [====>.........................] - ETA: 58s - loss: 0.2804 - accuracy: 0.8775 - auc: 0.9178 - precision: 0.7379 - recall: 0.6145 67/385 [====>.........................] - ETA: 58s - loss: 0.2804 - accuracy: 0.8775 - auc: 0.9178 - precision: 0.7381 - recall: 0.6146 68/385 [====>.........................] - ETA: 58s - loss: 0.2803 - accuracy: 0.8775 - auc: 0.9179 - precision: 0.7383 - recall: 0.6145 69/385 [====>.........................] - ETA: 57s - loss: 0.2802 - accuracy: 0.8775 - auc: 0.9179 - precision: 0.7383 - recall: 0.6146 70/385 [====>.........................] - ETA: 57s - loss: 0.2803 - accuracy: 0.8775 - auc: 0.9179 - precision: 0.7383 - recall: 0.6146 71/385 [====>.........................] - ETA: 57s - loss: 0.2803 - accuracy: 0.8775 - auc: 0.9179 - precision: 0.7384 - recall: 0.6146 72/385 [====>.........................] - ETA: 57s - loss: 0.2803 - accuracy: 0.8775 - auc: 0.9180 - precision: 0.7387 - recall: 0.6146 73/385 [====>.........................] - ETA: 57s - loss: 0.2802 - accuracy: 0.8775 - auc: 0.9180 - precision: 0.7386 - recall: 0.6146 74/385 [====>.........................] - ETA: 57s - loss: 0.2800 - accuracy: 0.8776 - auc: 0.9181 - precision: 0.7389 - recall: 0.6148 75/385 [====>.........................] - ETA: 57s - loss: 0.2800 - accuracy: 0.8777 - auc: 0.9181 - precision: 0.7391 - recall: 0.6147 76/385 [====>.........................] - ETA: 56s - loss: 0.2798 - accuracy: 0.8777 - auc: 0.9182 - precision: 0.7392 - recall: 0.6149 77/385 [=====>........................] - ETA: 56s - loss: 0.2798 - accuracy: 0.8777 - auc: 0.9183 - precision: 0.7394 - recall: 0.6149 78/385 [=====>........................] - ETA: 56s - loss: 0.2797 - accuracy: 0.8777 - auc: 0.9183 - precision: 0.7396 - recall: 0.6149 79/385 [=====>........................] - ETA: 56s - loss: 0.2796 - accuracy: 0.8778 - auc: 0.9184 - precision: 0.7396 - recall: 0.6150 80/385 [=====>........................] - ETA: 56s - loss: 0.2796 - accuracy: 0.8778 - auc: 0.9184 - precision: 0.7398 - recall: 0.6150 81/385 [=====>........................] - ETA: 56s - loss: 0.2796 - accuracy: 0.8778 - auc: 0.9184 - precision: 0.7396 - recall: 0.6152 82/385 [=====>........................] - ETA: 56s - loss: 0.2797 - accuracy: 0.8777 - auc: 0.9184 - precision: 0.7395 - recall: 0.6153 83/385 [=====>........................] - ETA: 55s - loss: 0.2796 - accuracy: 0.8778 - auc: 0.9184 - precision: 0.7395 - recall: 0.6155 84/385 [=====>........................] - ETA: 55s - loss: 0.2795 - accuracy: 0.8778 - auc: 0.9185 - precision: 0.7397 - recall: 0.6158 85/385 [=====>........................] - ETA: 55s - loss: 0.2794 - accuracy: 0.8779 - auc: 0.9186 - precision: 0.7397 - recall: 0.6162 86/385 [=====>........................] - ETA: 55s - loss: 0.2794 - accuracy: 0.8780 - auc: 0.9186 - precision: 0.7400 - recall: 0.6163 87/385 [=====>........................] - ETA: 55s - loss: 0.2793 - accuracy: 0.8780 - auc: 0.9187 - precision: 0.7401 - recall: 0.6163 88/385 [=====>........................] - ETA: 55s - loss: 0.2793 - accuracy: 0.8780 - auc: 0.9187 - precision: 0.7400 - recall: 0.6162 89/385 [=====>........................] - ETA: 54s - loss: 0.2792 - accuracy: 0.8780 - auc: 0.9187 - precision: 0.7400 - recall: 0.6161 90/385 [======>.......................] - ETA: 54s - loss: 0.2792 - accuracy: 0.8780 - auc: 0.9187 - precision: 0.7399 - recall: 0.6161 91/385 [======>.......................] - ETA: 54s - loss: 0.2791 - accuracy: 0.8781 - auc: 0.9187 - precision: 0.7399 - recall: 0.6165 92/385 [======>.......................] - ETA: 54s - loss: 0.2791 - accuracy: 0.8781 - auc: 0.9187 - precision: 0.7399 - recall: 0.6167 93/385 [======>.......................] - ETA: 54s - loss: 0.2790 - accuracy: 0.8781 - auc: 0.9188 - precision: 0.7400 - recall: 0.6168 94/385 [======>.......................] - ETA: 54s - loss: 0.2790 - accuracy: 0.8782 - auc: 0.9188 - precision: 0.7402 - recall: 0.6172 95/385 [======>.......................] - ETA: 53s - loss: 0.2789 - accuracy: 0.8782 - auc: 0.9189 - precision: 0.7403 - recall: 0.6174 96/385 [======>.......................] - ETA: 53s - loss: 0.2788 - accuracy: 0.8782 - auc: 0.9189 - precision: 0.7403 - recall: 0.6175 97/385 [======>.......................] - ETA: 53s - loss: 0.2788 - accuracy: 0.8782 - auc: 0.9189 - precision: 0.7400 - recall: 0.6177 98/385 [======>.......................] - ETA: 53s - loss: 0.2788 - accuracy: 0.8783 - auc: 0.9190 - precision: 0.7400 - recall: 0.6180 99/385 [======>.......................] - ETA: 53s - loss: 0.2787 - accuracy: 0.8783 - auc: 0.9190 - precision: 0.7402 - recall: 0.6180 100/385 [======>.......................] - ETA: 53s - loss: 0.2787 - accuracy: 0.8783 - auc: 0.9191 - precision: 0.7402 - recall: 0.6181 101/385 [======>.......................] - ETA: 52s - loss: 0.2786 - accuracy: 0.8783 - auc: 0.9191 - precision: 0.7403 - recall: 0.6181 102/385 [======>.......................] - ETA: 52s - loss: 0.2786 - accuracy: 0.8783 - auc: 0.9191 - precision: 0.7403 - recall: 0.6182 103/385 [=======>......................] - ETA: 52s - loss: 0.2785 - accuracy: 0.8784 - auc: 0.9192 - precision: 0.7404 - recall: 0.6184 104/385 [=======>......................] - ETA: 52s - loss: 0.2784 - accuracy: 0.8784 - auc: 0.9192 - precision: 0.7403 - recall: 0.6184 105/385 [=======>......................] - ETA: 52s - loss: 0.2783 - accuracy: 0.8785 - auc: 0.9193 - precision: 0.7405 - recall: 0.6187 106/385 [=======>......................] - ETA: 52s - loss: 0.2783 - accuracy: 0.8785 - auc: 0.9193 - precision: 0.7407 - recall: 0.6189 107/385 [=======>......................] - ETA: 51s - loss: 0.2783 - accuracy: 0.8785 - auc: 0.9193 - precision: 0.7407 - recall: 0.6188 108/385 [=======>......................] - ETA: 51s - loss: 0.2782 - accuracy: 0.8785 - auc: 0.9194 - precision: 0.7407 - recall: 0.6187 109/385 [=======>......................] - ETA: 51s - loss: 0.2781 - accuracy: 0.8785 - auc: 0.9194 - precision: 0.7408 - recall: 0.6189 110/385 [=======>......................] - ETA: 51s - loss: 0.2780 - accuracy: 0.8786 - auc: 0.9195 - precision: 0.7408 - recall: 0.6188 111/385 [=======>......................] - ETA: 51s - loss: 0.2779 - accuracy: 0.8786 - auc: 0.9195 - precision: 0.7409 - recall: 0.6189 112/385 [=======>......................] - ETA: 50s - loss: 0.2779 - accuracy: 0.8786 - auc: 0.9195 - precision: 0.7411 - recall: 0.6190 113/385 [=======>......................] - ETA: 50s - loss: 0.2779 - accuracy: 0.8786 - auc: 0.9195 - precision: 0.7411 - recall: 0.6191 114/385 [=======>......................] - ETA: 50s - loss: 0.2778 - accuracy: 0.8787 - auc: 0.9196 - precision: 0.7410 - recall: 0.6191 115/385 [=======>......................] - ETA: 50s - loss: 0.2779 - accuracy: 0.8787 - auc: 0.9196 - precision: 0.7411 - recall: 0.6191 116/385 [========>.....................] - ETA: 50s - loss: 0.2778 - accuracy: 0.8787 - auc: 0.9196 - precision: 0.7411 - recall: 0.6191 117/385 [========>.....................] - ETA: 50s - loss: 0.2777 - accuracy: 0.8787 - auc: 0.9197 - precision: 0.7413 - recall: 0.6193 118/385 [========>.....................] - ETA: 49s - loss: 0.2777 - accuracy: 0.8787 - auc: 0.9197 - precision: 0.7413 - recall: 0.6195 119/385 [========>.....................] - ETA: 49s - loss: 0.2776 - accuracy: 0.8788 - auc: 0.9198 - precision: 0.7415 - recall: 0.6195 120/385 [========>.....................] - ETA: 49s - loss: 0.2776 - accuracy: 0.8788 - auc: 0.9198 - precision: 0.7415 - recall: 0.6196 121/385 [========>.....................] - ETA: 49s - loss: 0.2775 - accuracy: 0.8788 - auc: 0.9198 - precision: 0.7415 - recall: 0.6196 122/385 [========>.....................] - ETA: 49s - loss: 0.2774 - accuracy: 0.8788 - auc: 0.9199 - precision: 0.7414 - recall: 0.6198 123/385 [========>.....................] - ETA: 49s - loss: 0.2774 - accuracy: 0.8789 - auc: 0.9199 - precision: 0.7415 - recall: 0.6198 124/385 [========>.....................] - ETA: 48s - loss: 0.2772 - accuracy: 0.8789 - auc: 0.9200 - precision: 0.7417 - recall: 0.6200 125/385 [========>.....................] - ETA: 48s - loss: 0.2771 - accuracy: 0.8790 - auc: 0.9201 - precision: 0.7418 - recall: 0.6202 126/385 [========>.....................] - ETA: 48s - loss: 0.2770 - accuracy: 0.8790 - auc: 0.9201 - precision: 0.7419 - recall: 0.6205 127/385 [========>.....................] - ETA: 48s - loss: 0.2770 - accuracy: 0.8790 - auc: 0.9201 - precision: 0.7417 - recall: 0.6205 128/385 [========>.....................] - ETA: 48s - loss: 0.2769 - accuracy: 0.8791 - auc: 0.9202 - precision: 0.7419 - recall: 0.6207 129/385 [=========>....................] - ETA: 48s - loss: 0.2769 - accuracy: 0.8791 - auc: 0.9202 - precision: 0.7421 - recall: 0.6209 130/385 [=========>....................] - ETA: 47s - loss: 0.2768 - accuracy: 0.8791 - auc: 0.9203 - precision: 0.7421 - recall: 0.6210 131/385 [=========>....................] - ETA: 47s - loss: 0.2767 - accuracy: 0.8792 - auc: 0.9203 - precision: 0.7423 - recall: 0.6211 132/385 [=========>....................] - ETA: 47s - loss: 0.2767 - accuracy: 0.8792 - auc: 0.9203 - precision: 0.7425 - recall: 0.6212 133/385 [=========>....................] - ETA: 47s - loss: 0.2766 - accuracy: 0.8793 - auc: 0.9204 - precision: 0.7425 - recall: 0.6213 134/385 [=========>....................] - ETA: 47s - loss: 0.2766 - accuracy: 0.8793 - auc: 0.9204 - precision: 0.7426 - recall: 0.6214 135/385 [=========>....................] - ETA: 46s - loss: 0.2765 - accuracy: 0.8793 - auc: 0.9205 - precision: 0.7426 - recall: 0.6215 136/385 [=========>....................] - ETA: 46s - loss: 0.2764 - accuracy: 0.8793 - auc: 0.9205 - precision: 0.7427 - recall: 0.6216 137/385 [=========>....................] - ETA: 46s - loss: 0.2764 - accuracy: 0.8793 - auc: 0.9205 - precision: 0.7426 - recall: 0.6216 138/385 [=========>....................] - ETA: 46s - loss: 0.2764 - accuracy: 0.8793 - auc: 0.9205 - precision: 0.7426 - recall: 0.6218 139/385 [=========>....................] - ETA: 46s - loss: 0.2764 - accuracy: 0.8793 - auc: 0.9205 - precision: 0.7426 - recall: 0.6218 140/385 [=========>....................] - ETA: 46s - loss: 0.2764 - accuracy: 0.8793 - auc: 0.9206 - precision: 0.7426 - recall: 0.6219 141/385 [=========>....................] - ETA: 45s - loss: 0.2763 - accuracy: 0.8794 - auc: 0.9206 - precision: 0.7426 - recall: 0.6221 142/385 [==========>...................] - ETA: 45s - loss: 0.2762 - accuracy: 0.8794 - auc: 0.9207 - precision: 0.7428 - recall: 0.6222 143/385 [==========>...................] - ETA: 45s - loss: 0.2761 - accuracy: 0.8794 - auc: 0.9207 - precision: 0.7429 - recall: 0.6222 144/385 [==========>...................] - ETA: 45s - loss: 0.2761 - accuracy: 0.8795 - auc: 0.9207 - precision: 0.7430 - recall: 0.6223 145/385 [==========>...................] - ETA: 45s - loss: 0.2760 - accuracy: 0.8795 - auc: 0.9208 - precision: 0.7431 - recall: 0.6224 146/385 [==========>...................] - ETA: 44s - loss: 0.2760 - accuracy: 0.8795 - auc: 0.9208 - precision: 0.7431 - recall: 0.6226 147/385 [==========>...................] - ETA: 44s - loss: 0.2759 - accuracy: 0.8796 - auc: 0.9208 - precision: 0.7431 - recall: 0.6226 148/385 [==========>...................] - ETA: 44s - loss: 0.2759 - accuracy: 0.8795 - auc: 0.9209 - precision: 0.7431 - recall: 0.6228 149/385 [==========>...................] - ETA: 44s - loss: 0.2759 - accuracy: 0.8795 - auc: 0.9208 - precision: 0.7430 - recall: 0.6227 150/385 [==========>...................] - ETA: 44s - loss: 0.2759 - accuracy: 0.8795 - auc: 0.9209 - precision: 0.7430 - recall: 0.6227 151/385 [==========>...................] - ETA: 43s - loss: 0.2759 - accuracy: 0.8796 - auc: 0.9209 - precision: 0.7431 - recall: 0.6229 152/385 [==========>...................] - ETA: 43s - loss: 0.2758 - accuracy: 0.8796 - auc: 0.9209 - precision: 0.7431 - recall: 0.6231 153/385 [==========>...................] - ETA: 43s - loss: 0.2758 - accuracy: 0.8796 - auc: 0.9209 - precision: 0.7432 - recall: 0.6233 154/385 [===========>..................] - ETA: 43s - loss: 0.2757 - accuracy: 0.8797 - auc: 0.9210 - precision: 0.7433 - recall: 0.6235 155/385 [===========>..................] - ETA: 43s - loss: 0.2757 - accuracy: 0.8797 - auc: 0.9210 - precision: 0.7433 - recall: 0.6235 156/385 [===========>..................] - ETA: 43s - loss: 0.2757 - accuracy: 0.8797 - auc: 0.9210 - precision: 0.7434 - recall: 0.6235 157/385 [===========>..................] - ETA: 42s - loss: 0.2756 - accuracy: 0.8797 - auc: 0.9210 - precision: 0.7434 - recall: 0.6237 158/385 [===========>..................] - ETA: 42s - loss: 0.2756 - accuracy: 0.8797 - auc: 0.9211 - precision: 0.7435 - recall: 0.6238 159/385 [===========>..................] - ETA: 42s - loss: 0.2756 - accuracy: 0.8798 - auc: 0.9211 - precision: 0.7435 - recall: 0.6239 160/385 [===========>..................] - ETA: 42s - loss: 0.2755 - accuracy: 0.8798 - auc: 0.9211 - precision: 0.7436 - recall: 0.6240 161/385 [===========>..................] - ETA: 42s - loss: 0.2755 - accuracy: 0.8798 - auc: 0.9211 - precision: 0.7436 - recall: 0.6240 162/385 [===========>..................] - ETA: 41s - loss: 0.2754 - accuracy: 0.8798 - auc: 0.9211 - precision: 0.7436 - recall: 0.6240 163/385 [===========>..................] - ETA: 41s - loss: 0.2754 - accuracy: 0.8799 - auc: 0.9212 - precision: 0.7437 - recall: 0.6241 164/385 [===========>..................] - ETA: 41s - loss: 0.2753 - accuracy: 0.8799 - auc: 0.9212 - precision: 0.7439 - recall: 0.6241 165/385 [===========>..................] - ETA: 41s - loss: 0.2753 - accuracy: 0.8799 - auc: 0.9213 - precision: 0.7439 - recall: 0.6242 166/385 [===========>..................] - ETA: 41s - loss: 0.2753 - accuracy: 0.8799 - auc: 0.9213 - precision: 0.7439 - recall: 0.6242 167/385 [============>.................] - ETA: 41s - loss: 0.2752 - accuracy: 0.8799 - auc: 0.9213 - precision: 0.7439 - recall: 0.6242 168/385 [============>.................] - ETA: 40s - loss: 0.2752 - accuracy: 0.8799 - auc: 0.9214 - precision: 0.7441 - recall: 0.6244 169/385 [============>.................] - ETA: 40s - loss: 0.2751 - accuracy: 0.8800 - auc: 0.9214 - precision: 0.7441 - recall: 0.6245 170/385 [============>.................] - ETA: 40s - loss: 0.2751 - accuracy: 0.8800 - auc: 0.9214 - precision: 0.7441 - recall: 0.6246 171/385 [============>.................] - ETA: 40s - loss: 0.2751 - accuracy: 0.8800 - auc: 0.9214 - precision: 0.7441 - recall: 0.6247 172/385 [============>.................] - ETA: 40s - loss: 0.2751 - accuracy: 0.8800 - auc: 0.9214 - precision: 0.7441 - recall: 0.6248 173/385 [============>.................] - ETA: 39s - loss: 0.2750 - accuracy: 0.8800 - auc: 0.9215 - precision: 0.7442 - recall: 0.6249 174/385 [============>.................] - ETA: 39s - loss: 0.2749 - accuracy: 0.8801 - auc: 0.9215 - precision: 0.7443 - recall: 0.6249 175/385 [============>.................] - ETA: 39s - loss: 0.2748 - accuracy: 0.8801 - auc: 0.9215 - precision: 0.7443 - recall: 0.6249 176/385 [============>.................] - ETA: 39s - loss: 0.2748 - accuracy: 0.8801 - auc: 0.9216 - precision: 0.7444 - recall: 0.6250 177/385 [============>.................] - ETA: 39s - loss: 0.2747 - accuracy: 0.8801 - auc: 0.9216 - precision: 0.7444 - recall: 0.6250 178/385 [============>.................] - ETA: 39s - loss: 0.2746 - accuracy: 0.8802 - auc: 0.9217 - precision: 0.7446 - recall: 0.6252 179/385 [============>.................] - ETA: 38s - loss: 0.2746 - accuracy: 0.8802 - auc: 0.9217 - precision: 0.7447 - recall: 0.6252 180/385 [=============>................] - ETA: 38s - loss: 0.2745 - accuracy: 0.8802 - auc: 0.9217 - precision: 0.7447 - recall: 0.6253 181/385 [=============>................] - ETA: 38s - loss: 0.2745 - accuracy: 0.8802 - auc: 0.9217 - precision: 0.7447 - recall: 0.6253 182/385 [=============>................] - ETA: 38s - loss: 0.2744 - accuracy: 0.8803 - auc: 0.9218 - precision: 0.7448 - recall: 0.6254 183/385 [=============>................] - ETA: 38s - loss: 0.2743 - accuracy: 0.8803 - auc: 0.9218 - precision: 0.7449 - recall: 0.6256 184/385 [=============>................] - ETA: 37s - loss: 0.2743 - accuracy: 0.8803 - auc: 0.9218 - precision: 0.7449 - recall: 0.6257 185/385 [=============>................] - ETA: 37s - loss: 0.2743 - accuracy: 0.8803 - auc: 0.9218 - precision: 0.7449 - recall: 0.6257 186/385 [=============>................] - ETA: 37s - loss: 0.2743 - accuracy: 0.8804 - auc: 0.9218 - precision: 0.7449 - recall: 0.6257 187/385 [=============>................] - ETA: 37s - loss: 0.2742 - accuracy: 0.8804 - auc: 0.9219 - precision: 0.7450 - recall: 0.6258 188/385 [=============>................] - ETA: 37s - loss: 0.2741 - accuracy: 0.8804 - auc: 0.9219 - precision: 0.7450 - recall: 0.6259 189/385 [=============>................] - ETA: 37s - loss: 0.2741 - accuracy: 0.8804 - auc: 0.9219 - precision: 0.7450 - recall: 0.6258 190/385 [=============>................] - ETA: 36s - loss: 0.2741 - accuracy: 0.8804 - auc: 0.9219 - precision: 0.7450 - recall: 0.6259 191/385 [=============>................] - ETA: 36s - loss: 0.2741 - accuracy: 0.8804 - auc: 0.9219 - precision: 0.7451 - recall: 0.6259 192/385 [=============>................] - ETA: 36s - loss: 0.2740 - accuracy: 0.8805 - auc: 0.9220 - precision: 0.7451 - recall: 0.6260 193/385 [==============>...............] - ETA: 36s - loss: 0.2740 - accuracy: 0.8805 - auc: 0.9220 - precision: 0.7452 - recall: 0.6261 194/385 [==============>...............] - ETA: 36s - loss: 0.2739 - accuracy: 0.8805 - auc: 0.9221 - precision: 0.7452 - recall: 0.6261 195/385 [==============>...............] - ETA: 35s - loss: 0.2738 - accuracy: 0.8805 - auc: 0.9221 - precision: 0.7453 - recall: 0.6262 196/385 [==============>...............] - ETA: 35s - loss: 0.2738 - accuracy: 0.8805 - auc: 0.9221 - precision: 0.7452 - recall: 0.6262 197/385 [==============>...............] - ETA: 35s - loss: 0.2738 - accuracy: 0.8805 - auc: 0.9221 - precision: 0.7453 - recall: 0.6262 198/385 [==============>...............] - ETA: 35s - loss: 0.2738 - accuracy: 0.8806 - auc: 0.9221 - precision: 0.7454 - recall: 0.6263 199/385 [==============>...............] - ETA: 35s - loss: 0.2738 - accuracy: 0.8806 - auc: 0.9222 - precision: 0.7455 - recall: 0.6264 200/385 [==============>...............] - ETA: 34s - loss: 0.2738 - accuracy: 0.8806 - auc: 0.9222 - precision: 0.7455 - recall: 0.6264 201/385 [==============>...............] - ETA: 34s - loss: 0.2737 - accuracy: 0.8806 - auc: 0.9222 - precision: 0.7456 - recall: 0.6266 202/385 [==============>...............] - ETA: 34s - loss: 0.2737 - accuracy: 0.8806 - auc: 0.9222 - precision: 0.7456 - recall: 0.6267 203/385 [==============>...............] - ETA: 34s - loss: 0.2736 - accuracy: 0.8806 - auc: 0.9222 - precision: 0.7456 - recall: 0.6268 204/385 [==============>...............] - ETA: 34s - loss: 0.2736 - accuracy: 0.8806 - auc: 0.9223 - precision: 0.7456 - recall: 0.6269 205/385 [==============>...............] - ETA: 34s - loss: 0.2735 - accuracy: 0.8807 - auc: 0.9223 - precision: 0.7456 - recall: 0.6270 206/385 [===============>..............] - ETA: 33s - loss: 0.2735 - accuracy: 0.8807 - auc: 0.9224 - precision: 0.7457 - recall: 0.6271 207/385 [===============>..............] - ETA: 33s - loss: 0.2734 - accuracy: 0.8807 - auc: 0.9224 - precision: 0.7458 - recall: 0.6272 208/385 [===============>..............] - ETA: 33s - loss: 0.2734 - accuracy: 0.8807 - auc: 0.9224 - precision: 0.7458 - recall: 0.6272 209/385 [===============>..............] - ETA: 33s - loss: 0.2733 - accuracy: 0.8808 - auc: 0.9225 - precision: 0.7459 - recall: 0.6272 210/385 [===============>..............] - ETA: 33s - loss: 0.2733 - accuracy: 0.8808 - auc: 0.9225 - precision: 0.7460 - recall: 0.6272 211/385 [===============>..............] - ETA: 32s - loss: 0.2733 - accuracy: 0.8808 - auc: 0.9225 - precision: 0.7460 - recall: 0.6272 212/385 [===============>..............] - ETA: 32s - loss: 0.2732 - accuracy: 0.8808 - auc: 0.9225 - precision: 0.7460 - recall: 0.6273 213/385 [===============>..............] - ETA: 32s - loss: 0.2732 - accuracy: 0.8808 - auc: 0.9225 - precision: 0.7460 - recall: 0.6274 214/385 [===============>..............] - ETA: 32s - loss: 0.2731 - accuracy: 0.8808 - auc: 0.9226 - precision: 0.7461 - recall: 0.6275 215/385 [===============>..............] - ETA: 32s - loss: 0.2731 - accuracy: 0.8808 - auc: 0.9226 - precision: 0.7460 - recall: 0.6275 216/385 [===============>..............] - ETA: 31s - loss: 0.2731 - accuracy: 0.8809 - auc: 0.9226 - precision: 0.7461 - recall: 0.6276 217/385 [===============>..............] - ETA: 31s - loss: 0.2731 - accuracy: 0.8809 - auc: 0.9226 - precision: 0.7461 - recall: 0.6277 218/385 [===============>..............] - ETA: 31s - loss: 0.2730 - accuracy: 0.8809 - auc: 0.9226 - precision: 0.7461 - recall: 0.6277 219/385 [================>.............] - ETA: 31s - loss: 0.2730 - accuracy: 0.8809 - auc: 0.9226 - precision: 0.7461 - recall: 0.6278 220/385 [================>.............] - ETA: 31s - loss: 0.2729 - accuracy: 0.8809 - auc: 0.9227 - precision: 0.7461 - recall: 0.6278 221/385 [================>.............] - ETA: 30s - loss: 0.2729 - accuracy: 0.8809 - auc: 0.9227 - precision: 0.7461 - recall: 0.6279 222/385 [================>.............] - ETA: 30s - loss: 0.2729 - accuracy: 0.8809 - auc: 0.9227 - precision: 0.7461 - recall: 0.6279 223/385 [================>.............] - ETA: 30s - loss: 0.2729 - accuracy: 0.8809 - auc: 0.9227 - precision: 0.7462 - recall: 0.6279 224/385 [================>.............] - ETA: 30s - loss: 0.2728 - accuracy: 0.8809 - auc: 0.9227 - precision: 0.7461 - recall: 0.6279 225/385 [================>.............] - ETA: 30s - loss: 0.2728 - accuracy: 0.8809 - auc: 0.9227 - precision: 0.7462 - recall: 0.6279 226/385 [================>.............] - ETA: 30s - loss: 0.2728 - accuracy: 0.8809 - auc: 0.9227 - precision: 0.7461 - recall: 0.6279 227/385 [================>.............] - ETA: 29s - loss: 0.2728 - accuracy: 0.8809 - auc: 0.9227 - precision: 0.7461 - recall: 0.6279 228/385 [================>.............] - ETA: 29s - loss: 0.2728 - accuracy: 0.8810 - auc: 0.9228 - precision: 0.7461 - recall: 0.6279 229/385 [================>.............] - ETA: 29s - loss: 0.2728 - accuracy: 0.8810 - auc: 0.9228 - precision: 0.7461 - recall: 0.6279 230/385 [================>.............] - ETA: 29s - loss: 0.2727 - accuracy: 0.8810 - auc: 0.9228 - precision: 0.7462 - recall: 0.6280 231/385 [=================>............] - ETA: 29s - loss: 0.2727 - accuracy: 0.8810 - auc: 0.9228 - precision: 0.7463 - recall: 0.6280 232/385 [=================>............] - ETA: 28s - loss: 0.2727 - accuracy: 0.8810 - auc: 0.9228 - precision: 0.7464 - recall: 0.6280 233/385 [=================>............] - ETA: 28s - loss: 0.2726 - accuracy: 0.8810 - auc: 0.9228 - precision: 0.7463 - recall: 0.6279 234/385 [=================>............] - ETA: 28s - loss: 0.2726 - accuracy: 0.8810 - auc: 0.9229 - precision: 0.7464 - recall: 0.6279 235/385 [=================>............] - ETA: 28s - loss: 0.2726 - accuracy: 0.8810 - auc: 0.9229 - precision: 0.7464 - recall: 0.6280 236/385 [=================>............] - ETA: 28s - loss: 0.2725 - accuracy: 0.8810 - auc: 0.9229 - precision: 0.7464 - recall: 0.6281 237/385 [=================>............] - ETA: 27s - loss: 0.2725 - accuracy: 0.8810 - auc: 0.9229 - precision: 0.7464 - recall: 0.6282 238/385 [=================>............] - ETA: 27s - loss: 0.2725 - accuracy: 0.8811 - auc: 0.9229 - precision: 0.7463 - recall: 0.6283 239/385 [=================>............] - ETA: 27s - loss: 0.2725 - accuracy: 0.8811 - auc: 0.9229 - precision: 0.7463 - recall: 0.6283 240/385 [=================>............] - ETA: 27s - loss: 0.2725 - accuracy: 0.8811 - auc: 0.9229 - precision: 0.7463 - recall: 0.6284 241/385 [=================>............] - ETA: 27s - loss: 0.2724 - accuracy: 0.8811 - auc: 0.9230 - precision: 0.7464 - recall: 0.6285 242/385 [=================>............] - ETA: 27s - loss: 0.2725 - accuracy: 0.8811 - auc: 0.9229 - precision: 0.7464 - recall: 0.6284 243/385 [=================>............] - ETA: 26s - loss: 0.2725 - accuracy: 0.8811 - auc: 0.9229 - precision: 0.7464 - recall: 0.6284 244/385 [==================>...........] - ETA: 26s - loss: 0.2724 - accuracy: 0.8811 - auc: 0.9230 - precision: 0.7465 - recall: 0.6284 245/385 [==================>...........] - ETA: 26s - loss: 0.2724 - accuracy: 0.8811 - auc: 0.9230 - precision: 0.7465 - recall: 0.6285 246/385 [==================>...........] - ETA: 26s - loss: 0.2724 - accuracy: 0.8811 - auc: 0.9230 - precision: 0.7465 - recall: 0.6285 247/385 [==================>...........] - ETA: 26s - loss: 0.2724 - accuracy: 0.8811 - auc: 0.9230 - precision: 0.7465 - recall: 0.6286 248/385 [==================>...........] - ETA: 25s - loss: 0.2724 - accuracy: 0.8811 - auc: 0.9230 - precision: 0.7465 - recall: 0.6287 249/385 [==================>...........] - ETA: 25s - loss: 0.2724 - accuracy: 0.8811 - auc: 0.9230 - precision: 0.7465 - recall: 0.6287 250/385 [==================>...........] - ETA: 25s - loss: 0.2723 - accuracy: 0.8811 - auc: 0.9230 - precision: 0.7465 - recall: 0.6287 251/385 [==================>...........] - ETA: 25s - loss: 0.2723 - accuracy: 0.8811 - auc: 0.9231 - precision: 0.7466 - recall: 0.6288 252/385 [==================>...........] - ETA: 25s - loss: 0.2723 - accuracy: 0.8812 - auc: 0.9231 - precision: 0.7466 - recall: 0.6289 253/385 [==================>...........] - ETA: 25s - loss: 0.2722 - accuracy: 0.8812 - auc: 0.9231 - precision: 0.7466 - recall: 0.6290 254/385 [==================>...........] - ETA: 24s - loss: 0.2722 - accuracy: 0.8812 - auc: 0.9231 - precision: 0.7467 - recall: 0.6290 255/385 [==================>...........] - ETA: 24s - loss: 0.2722 - accuracy: 0.8812 - auc: 0.9231 - precision: 0.7467 - recall: 0.6290 256/385 [==================>...........] - ETA: 24s - loss: 0.2722 - accuracy: 0.8812 - auc: 0.9231 - precision: 0.7466 - recall: 0.6290 257/385 [===================>..........] - ETA: 24s - loss: 0.2722 - accuracy: 0.8812 - auc: 0.9231 - precision: 0.7466 - recall: 0.6290 258/385 [===================>..........] - ETA: 24s - loss: 0.2722 - accuracy: 0.8812 - auc: 0.9231 - precision: 0.7467 - recall: 0.6291 259/385 [===================>..........] - ETA: 23s - loss: 0.2721 - accuracy: 0.8812 - auc: 0.9232 - precision: 0.7467 - recall: 0.6292 260/385 [===================>..........] - ETA: 23s - loss: 0.2721 - accuracy: 0.8812 - auc: 0.9232 - precision: 0.7467 - recall: 0.6293 261/385 [===================>..........] - ETA: 23s - loss: 0.2721 - accuracy: 0.8813 - auc: 0.9232 - precision: 0.7468 - recall: 0.6293 262/385 [===================>..........] - ETA: 23s - loss: 0.2721 - accuracy: 0.8813 - auc: 0.9232 - precision: 0.7468 - recall: 0.6293 263/385 [===================>..........] - ETA: 23s - loss: 0.2721 - accuracy: 0.8813 - auc: 0.9232 - precision: 0.7469 - recall: 0.6293 264/385 [===================>..........] - ETA: 22s - loss: 0.2721 - accuracy: 0.8813 - auc: 0.9232 - precision: 0.7469 - recall: 0.6294 265/385 [===================>..........] - ETA: 22s - loss: 0.2720 - accuracy: 0.8813 - auc: 0.9232 - precision: 0.7469 - recall: 0.6294 266/385 [===================>..........] - ETA: 22s - loss: 0.2720 - accuracy: 0.8813 - auc: 0.9232 - precision: 0.7470 - recall: 0.6295 267/385 [===================>..........] - ETA: 22s - loss: 0.2720 - accuracy: 0.8813 - auc: 0.9233 - precision: 0.7470 - recall: 0.6295 268/385 [===================>..........] - ETA: 22s - loss: 0.2719 - accuracy: 0.8813 - auc: 0.9233 - precision: 0.7470 - recall: 0.6295 269/385 [===================>..........] - ETA: 22s - loss: 0.2719 - accuracy: 0.8813 - auc: 0.9233 - precision: 0.7470 - recall: 0.6296 270/385 [====================>.........] - ETA: 21s - loss: 0.2719 - accuracy: 0.8813 - auc: 0.9233 - precision: 0.7469 - recall: 0.6296 271/385 [====================>.........] - ETA: 21s - loss: 0.2719 - accuracy: 0.8813 - auc: 0.9233 - precision: 0.7470 - recall: 0.6296 272/385 [====================>.........] - ETA: 21s - loss: 0.2718 - accuracy: 0.8813 - auc: 0.9233 - precision: 0.7471 - recall: 0.6296 273/385 [====================>.........] - ETA: 21s - loss: 0.2718 - accuracy: 0.8813 - auc: 0.9234 - precision: 0.7471 - recall: 0.6296 274/385 [====================>.........] - ETA: 21s - loss: 0.2718 - accuracy: 0.8814 - auc: 0.9234 - precision: 0.7471 - recall: 0.6296 275/385 [====================>.........] - ETA: 20s - loss: 0.2718 - accuracy: 0.8814 - auc: 0.9234 - precision: 0.7471 - recall: 0.6296 276/385 [====================>.........] - ETA: 20s - loss: 0.2718 - accuracy: 0.8814 - auc: 0.9234 - precision: 0.7472 - recall: 0.6297 277/385 [====================>.........] - ETA: 20s - loss: 0.2718 - accuracy: 0.8814 - auc: 0.9234 - precision: 0.7472 - recall: 0.6297 278/385 [====================>.........] - ETA: 20s - loss: 0.2718 - accuracy: 0.8814 - auc: 0.9234 - precision: 0.7471 - recall: 0.6297 279/385 [====================>.........] - ETA: 20s - loss: 0.2717 - accuracy: 0.8814 - auc: 0.9234 - precision: 0.7472 - recall: 0.6298 280/385 [====================>.........] - ETA: 20s - loss: 0.2717 - accuracy: 0.8814 - auc: 0.9234 - precision: 0.7472 - recall: 0.6299 281/385 [====================>.........] - ETA: 19s - loss: 0.2717 - accuracy: 0.8814 - auc: 0.9235 - precision: 0.7472 - recall: 0.6299 282/385 [====================>.........] - ETA: 19s - loss: 0.2717 - accuracy: 0.8814 - auc: 0.9235 - precision: 0.7472 - recall: 0.6299 283/385 [=====================>........] - ETA: 19s - loss: 0.2717 - accuracy: 0.8814 - auc: 0.9235 - precision: 0.7472 - recall: 0.6300 284/385 [=====================>........] - ETA: 19s - loss: 0.2716 - accuracy: 0.8814 - auc: 0.9235 - precision: 0.7473 - recall: 0.6301 285/385 [=====================>........] - ETA: 19s - loss: 0.2716 - accuracy: 0.8814 - auc: 0.9235 - precision: 0.7473 - recall: 0.6301 286/385 [=====================>........] - ETA: 18s - loss: 0.2715 - accuracy: 0.8815 - auc: 0.9236 - precision: 0.7474 - recall: 0.6302 287/385 [=====================>........] - ETA: 18s - loss: 0.2715 - accuracy: 0.8815 - auc: 0.9236 - precision: 0.7474 - recall: 0.6302 288/385 [=====================>........] - ETA: 18s - loss: 0.2715 - accuracy: 0.8815 - auc: 0.9236 - precision: 0.7475 - recall: 0.6303 289/385 [=====================>........] - ETA: 18s - loss: 0.2714 - accuracy: 0.8815 - auc: 0.9236 - precision: 0.7476 - recall: 0.6302 290/385 [=====================>........] - ETA: 18s - loss: 0.2714 - accuracy: 0.8815 - auc: 0.9236 - precision: 0.7475 - recall: 0.6302 291/385 [=====================>........] - ETA: 18s - loss: 0.2714 - accuracy: 0.8815 - auc: 0.9236 - precision: 0.7476 - recall: 0.6303 292/385 [=====================>........] - ETA: 17s - loss: 0.2714 - accuracy: 0.8815 - auc: 0.9236 - precision: 0.7476 - recall: 0.6303 293/385 [=====================>........] - ETA: 17s - loss: 0.2714 - accuracy: 0.8815 - auc: 0.9237 - precision: 0.7477 - recall: 0.6303 294/385 [=====================>........] - ETA: 17s - loss: 0.2714 - accuracy: 0.8815 - auc: 0.9237 - precision: 0.7476 - recall: 0.6304 295/385 [=====================>........] - ETA: 17s - loss: 0.2714 - accuracy: 0.8815 - auc: 0.9237 - precision: 0.7476 - recall: 0.6304 296/385 [======================>.......] - ETA: 17s - loss: 0.2713 - accuracy: 0.8816 - auc: 0.9237 - precision: 0.7477 - recall: 0.6305 297/385 [======================>.......] - ETA: 16s - loss: 0.2713 - accuracy: 0.8816 - auc: 0.9237 - precision: 0.7477 - recall: 0.6306 298/385 [======================>.......] - ETA: 16s - loss: 0.2713 - accuracy: 0.8816 - auc: 0.9237 - precision: 0.7478 - recall: 0.6306 299/385 [======================>.......] - ETA: 16s - loss: 0.2713 - accuracy: 0.8816 - auc: 0.9237 - precision: 0.7477 - recall: 0.6305 300/385 [======================>.......] - ETA: 16s - loss: 0.2712 - accuracy: 0.8816 - auc: 0.9238 - precision: 0.7477 - recall: 0.6306 301/385 [======================>.......] - ETA: 16s - loss: 0.2712 - accuracy: 0.8816 - auc: 0.9238 - precision: 0.7478 - recall: 0.6305 302/385 [======================>.......] - ETA: 15s - loss: 0.2712 - accuracy: 0.8816 - auc: 0.9238 - precision: 0.7479 - recall: 0.6306 303/385 [======================>.......] - ETA: 15s - loss: 0.2711 - accuracy: 0.8816 - auc: 0.9238 - precision: 0.7479 - recall: 0.6306 304/385 [======================>.......] - ETA: 15s - loss: 0.2711 - accuracy: 0.8816 - auc: 0.9238 - precision: 0.7479 - recall: 0.6307 305/385 [======================>.......] - ETA: 15s - loss: 0.2711 - accuracy: 0.8816 - auc: 0.9238 - precision: 0.7479 - recall: 0.6307 306/385 [======================>.......] - ETA: 15s - loss: 0.2711 - accuracy: 0.8816 - auc: 0.9238 - precision: 0.7479 - recall: 0.6307 307/385 [======================>.......] - ETA: 15s - loss: 0.2711 - accuracy: 0.8816 - auc: 0.9238 - precision: 0.7479 - recall: 0.6307 308/385 [=======================>......] - ETA: 14s - loss: 0.2710 - accuracy: 0.8817 - auc: 0.9239 - precision: 0.7479 - recall: 0.6309 309/385 [=======================>......] - ETA: 14s - loss: 0.2710 - accuracy: 0.8817 - auc: 0.9239 - precision: 0.7480 - recall: 0.6309 310/385 [=======================>......] - ETA: 14s - loss: 0.2710 - accuracy: 0.8817 - auc: 0.9239 - precision: 0.7480 - recall: 0.6310 311/385 [=======================>......] - ETA: 14s - loss: 0.2710 - accuracy: 0.8817 - auc: 0.9239 - precision: 0.7480 - recall: 0.6310 312/385 [=======================>......] - ETA: 14s - loss: 0.2709 - accuracy: 0.8817 - auc: 0.9239 - precision: 0.7480 - recall: 0.6311 313/385 [=======================>......] - ETA: 13s - loss: 0.2709 - accuracy: 0.8817 - auc: 0.9240 - precision: 0.7481 - recall: 0.6311 314/385 [=======================>......] - ETA: 13s - loss: 0.2709 - accuracy: 0.8817 - auc: 0.9240 - precision: 0.7481 - recall: 0.6311 315/385 [=======================>......] - ETA: 13s - loss: 0.2709 - accuracy: 0.8817 - auc: 0.9240 - precision: 0.7481 - recall: 0.6312 316/385 [=======================>......] - ETA: 13s - loss: 0.2709 - accuracy: 0.8818 - auc: 0.9240 - precision: 0.7481 - recall: 0.6313 317/385 [=======================>......] - ETA: 13s - loss: 0.2708 - accuracy: 0.8818 - auc: 0.9240 - precision: 0.7481 - recall: 0.6313 318/385 [=======================>......] - ETA: 12s - loss: 0.2708 - accuracy: 0.8818 - auc: 0.9240 - precision: 0.7482 - recall: 0.6314 319/385 [=======================>......] - ETA: 12s - loss: 0.2708 - accuracy: 0.8818 - auc: 0.9240 - precision: 0.7482 - recall: 0.6314 320/385 [=======================>......] - ETA: 12s - loss: 0.2707 - accuracy: 0.8818 - auc: 0.9241 - precision: 0.7482 - recall: 0.6315 321/385 [========================>.....] - ETA: 12s - loss: 0.2707 - accuracy: 0.8818 - auc: 0.9241 - precision: 0.7483 - recall: 0.6315 322/385 [========================>.....] - ETA: 12s - loss: 0.2707 - accuracy: 0.8818 - auc: 0.9241 - precision: 0.7483 - recall: 0.6315 323/385 [========================>.....] - ETA: 11s - loss: 0.2707 - accuracy: 0.8818 - auc: 0.9241 - precision: 0.7483 - recall: 0.6314 324/385 [========================>.....] - ETA: 11s - loss: 0.2707 - accuracy: 0.8818 - auc: 0.9241 - precision: 0.7484 - recall: 0.6314 325/385 [========================>.....] - ETA: 11s - loss: 0.2707 - accuracy: 0.8818 - auc: 0.9241 - precision: 0.7484 - recall: 0.6314 326/385 [========================>.....] - ETA: 11s - loss: 0.2707 - accuracy: 0.8818 - auc: 0.9241 - precision: 0.7484 - recall: 0.6315 327/385 [========================>.....] - ETA: 11s - loss: 0.2707 - accuracy: 0.8819 - auc: 0.9241 - precision: 0.7484 - recall: 0.6316 328/385 [========================>.....] - ETA: 11s - loss: 0.2706 - accuracy: 0.8819 - auc: 0.9241 - precision: 0.7485 - recall: 0.6317 329/385 [========================>.....] - ETA: 10s - loss: 0.2706 - accuracy: 0.8819 - auc: 0.9242 - precision: 0.7485 - recall: 0.6317 330/385 [========================>.....] - ETA: 10s - loss: 0.2706 - accuracy: 0.8819 - auc: 0.9242 - precision: 0.7485 - recall: 0.6317 331/385 [========================>.....] - ETA: 10s - loss: 0.2705 - accuracy: 0.8819 - auc: 0.9242 - precision: 0.7485 - recall: 0.6318 332/385 [========================>.....] - ETA: 10s - loss: 0.2705 - accuracy: 0.8819 - auc: 0.9242 - precision: 0.7485 - recall: 0.6318 333/385 [========================>.....] - ETA: 10s - loss: 0.2705 - accuracy: 0.8820 - auc: 0.9242 - precision: 0.7485 - recall: 0.6318 334/385 [=========================>....] - ETA: 9s - loss: 0.2704 - accuracy: 0.8820 - auc: 0.9242 - precision: 0.7485 - recall: 0.6318 335/385 [=========================>....] - ETA: 9s - loss: 0.2704 - accuracy: 0.8820 - auc: 0.9242 - precision: 0.7486 - recall: 0.6318 336/385 [=========================>....] - ETA: 9s - loss: 0.2704 - accuracy: 0.8820 - auc: 0.9242 - precision: 0.7486 - recall: 0.6318 337/385 [=========================>....] - ETA: 9s - loss: 0.2704 - accuracy: 0.8820 - auc: 0.9242 - precision: 0.7486 - recall: 0.6318 338/385 [=========================>....] - ETA: 9s - loss: 0.2704 - accuracy: 0.8820 - auc: 0.9243 - precision: 0.7486 - recall: 0.6318 339/385 [=========================>....] - ETA: 8s - loss: 0.2703 - accuracy: 0.8820 - auc: 0.9243 - precision: 0.7487 - recall: 0.6318 340/385 [=========================>....] - ETA: 8s - loss: 0.2704 - accuracy: 0.8820 - auc: 0.9243 - precision: 0.7487 - recall: 0.6318 341/385 [=========================>....] - ETA: 8s - loss: 0.2704 - accuracy: 0.8820 - auc: 0.9243 - precision: 0.7487 - recall: 0.6317 342/385 [=========================>....] - ETA: 8s - loss: 0.2703 - accuracy: 0.8820 - auc: 0.9243 - precision: 0.7488 - recall: 0.6318 343/385 [=========================>....] - ETA: 8s - loss: 0.2703 - accuracy: 0.8820 - auc: 0.9243 - precision: 0.7488 - recall: 0.6318 344/385 [=========================>....] - ETA: 7s - loss: 0.2703 - accuracy: 0.8820 - auc: 0.9243 - precision: 0.7488 - recall: 0.6318 345/385 [=========================>....] - ETA: 7s - loss: 0.2703 - accuracy: 0.8820 - auc: 0.9243 - precision: 0.7488 - recall: 0.6319 346/385 [=========================>....] - ETA: 7s - loss: 0.2703 - accuracy: 0.8820 - auc: 0.9243 - precision: 0.7488 - recall: 0.6319 347/385 [==========================>...] - ETA: 7s - loss: 0.2702 - accuracy: 0.8820 - auc: 0.9244 - precision: 0.7488 - recall: 0.6319 348/385 [==========================>...] - ETA: 7s - loss: 0.2702 - accuracy: 0.8821 - auc: 0.9244 - precision: 0.7489 - recall: 0.6320 349/385 [==========================>...] - ETA: 6s - loss: 0.2702 - accuracy: 0.8821 - auc: 0.9244 - precision: 0.7489 - recall: 0.6320 350/385 [==========================>...] - ETA: 6s - loss: 0.2702 - accuracy: 0.8821 - auc: 0.9244 - precision: 0.7489 - recall: 0.6320 351/385 [==========================>...] - ETA: 6s - loss: 0.2701 - accuracy: 0.8821 - auc: 0.9244 - precision: 0.7489 - recall: 0.6321 352/385 [==========================>...] - ETA: 6s - loss: 0.2701 - accuracy: 0.8821 - auc: 0.9244 - precision: 0.7489 - recall: 0.6321 353/385 [==========================>...] - ETA: 6s - loss: 0.2701 - accuracy: 0.8821 - auc: 0.9244 - precision: 0.7489 - recall: 0.6322 354/385 [==========================>...] - ETA: 5s - loss: 0.2701 - accuracy: 0.8821 - auc: 0.9244 - precision: 0.7489 - recall: 0.6322 355/385 [==========================>...] - ETA: 5s - loss: 0.2701 - accuracy: 0.8821 - auc: 0.9244 - precision: 0.7489 - recall: 0.6323 356/385 [==========================>...] - ETA: 5s - loss: 0.2701 - accuracy: 0.8821 - auc: 0.9245 - precision: 0.7490 - recall: 0.6323 357/385 [==========================>...] - ETA: 5s - loss: 0.2700 - accuracy: 0.8821 - auc: 0.9245 - precision: 0.7490 - recall: 0.6324 358/385 [==========================>...] - ETA: 5s - loss: 0.2700 - accuracy: 0.8821 - auc: 0.9245 - precision: 0.7489 - recall: 0.6324 359/385 [==========================>...] - ETA: 5s - loss: 0.2700 - accuracy: 0.8821 - auc: 0.9245 - precision: 0.7490 - recall: 0.6324 360/385 [===========================>..] - ETA: 4s - loss: 0.2700 - accuracy: 0.8821 - auc: 0.9245 - precision: 0.7490 - recall: 0.6324 361/385 [===========================>..] - ETA: 4s - loss: 0.2700 - accuracy: 0.8821 - auc: 0.9245 - precision: 0.7491 - recall: 0.6325 362/385 [===========================>..] - ETA: 4s - loss: 0.2700 - accuracy: 0.8821 - auc: 0.9245 - precision: 0.7491 - recall: 0.6325 363/385 [===========================>..] - ETA: 4s - loss: 0.2699 - accuracy: 0.8822 - auc: 0.9245 - precision: 0.7491 - recall: 0.6325 364/385 [===========================>..] - ETA: 4s - loss: 0.2699 - accuracy: 0.8822 - auc: 0.9246 - precision: 0.7491 - recall: 0.6326 365/385 [===========================>..] - ETA: 3s - loss: 0.2699 - accuracy: 0.8822 - auc: 0.9246 - precision: 0.7491 - recall: 0.6326 366/385 [===========================>..] - ETA: 3s - loss: 0.2699 - accuracy: 0.8822 - auc: 0.9246 - precision: 0.7491 - recall: 0.6326 367/385 [===========================>..] - ETA: 3s - loss: 0.2699 - accuracy: 0.8822 - auc: 0.9246 - precision: 0.7492 - recall: 0.6327 368/385 [===========================>..] - ETA: 3s - loss: 0.2699 - accuracy: 0.8822 - auc: 0.9246 - precision: 0.7492 - recall: 0.6327 369/385 [===========================>..] - ETA: 3s - loss: 0.2698 - accuracy: 0.8822 - auc: 0.9246 - precision: 0.7492 - recall: 0.6327 370/385 [===========================>..] - ETA: 2s - loss: 0.2698 - accuracy: 0.8822 - auc: 0.9246 - precision: 0.7491 - recall: 0.6327 371/385 [===========================>..] - ETA: 2s - loss: 0.2698 - accuracy: 0.8822 - auc: 0.9246 - precision: 0.7492 - recall: 0.6328 372/385 [===========================>..] - ETA: 2s - loss: 0.2698 - accuracy: 0.8822 - auc: 0.9246 - precision: 0.7492 - recall: 0.6328 373/385 [============================>.] - ETA: 2s - loss: 0.2698 - accuracy: 0.8822 - auc: 0.9246 - precision: 0.7492 - recall: 0.6328 374/385 [============================>.] - ETA: 2s - loss: 0.2698 - accuracy: 0.8822 - auc: 0.9247 - precision: 0.7492 - recall: 0.6328 375/385 [============================>.] - ETA: 1s - loss: 0.2697 - accuracy: 0.8822 - auc: 0.9247 - precision: 0.7492 - recall: 0.6328 376/385 [============================>.] - ETA: 1s - loss: 0.2697 - accuracy: 0.8822 - auc: 0.9247 - precision: 0.7492 - recall: 0.6328 377/385 [============================>.] - ETA: 1s - loss: 0.2697 - accuracy: 0.8822 - auc: 0.9247 - precision: 0.7493 - recall: 0.6329 378/385 [============================>.] - ETA: 1s - loss: 0.2697 - accuracy: 0.8822 - auc: 0.9247 - precision: 0.7492 - recall: 0.6329 379/385 [============================>.] - ETA: 1s - loss: 0.2697 - accuracy: 0.8822 - auc: 0.9247 - precision: 0.7492 - recall: 0.6329 380/385 [============================>.] - ETA: 0s - loss: 0.2697 - accuracy: 0.8823 - auc: 0.9247 - precision: 0.7493 - recall: 0.6329 381/385 [============================>.] - ETA: 0s - loss: 0.2696 - accuracy: 0.8823 - auc: 0.9247 - precision: 0.7494 - recall: 0.6330 382/385 [============================>.] - ETA: 0s - loss: 0.2696 - accuracy: 0.8823 - auc: 0.9247 - precision: 0.7494 - recall: 0.6330 383/385 [============================>.] - ETA: 0s - loss: 0.2696 - accuracy: 0.8823 - auc: 0.9248 - precision: 0.7494 - recall: 0.6330 384/385 [============================>.] - ETA: 0s - loss: 0.2696 - accuracy: 0.8823 - auc: 0.9248 - precision: 0.7495 - recall: 0.6330 385/385 [==============================] - ETA: 0s - loss: 0.2696 - accuracy: 0.8823 - auc: 0.9248 - precision: 0.7495 - recall: 0.6331 (Worker pid=1347) 2023-02-13 15:39:24.075849: W tensorflow/core/framework/dataset.cc:769] Input of GeneratorDatasetOp::Dataset will not be optimized because the dataset does not implement the AsGraphDefInternal() method needed to apply optimizations. Train results: verbose: 1 epochs: 2 steps: 385 loss: [0.342410147190094, 0.2695556879043579] accuracy: [0.845525324344635, 0.8823168873786926] auc: [0.8709021806716919, 0.9247820973396301] precision: [0.6969649195671082, 0.7495014071464539] recall: [0.42583805322647095, 0.6330686807632446] val_loss: [0.273588627576828, 0.25649839639663696] val_accuracy: [0.8797479271888733, 0.8873348832130432] val_auc: [0.9229004979133606, 0.9333168864250183] val_precision: [0.7403081059455872, 0.7492455840110779] val_recall: [0.6269906759262085, 0.6680351495742798] 385/385 [==============================] - 79s 207ms/step - loss: 0.2696 - accuracy: 0.8823 - auc: 0.9248 - precision: 0.7495 - recall: 0.6331 - val_loss: 0.2565 - val_accuracy: 0.8873 - val_auc: 0.9333 - val_precision: 0.7492 - val_recall: 0.6680 2023-02-13 15:39:49,471 INFO worker.py:843 -- Connecting to existing Ray cluster at address: 172.26.34.55:6379 2023-02-13 15:39:49,526 INFO worker.py:843 -- Connecting to existing Ray cluster at address: 172.26.34.55:6379 2023-02-13 15:39:49,535 INFO worker.py:843 -- Connecting to existing Ray cluster at address: 172.26.34.55:6379 2023-02-13 15:39:49,583 INFO worker.py:843 -- Connecting to existing Ray cluster at address: 172.26.34.55:6379 2023-02-13 15:39:49,584 INFO worker.py:843 -- Connecting to existing Ray cluster at address: 172.26.34.55:6379 2023-02-13 15:39:49,586 INFO worker.py:843 -- Connecting to existing Ray cluster at address: 172.26.34.55:6379 2023-02-13 15:39:49,591 INFO worker.py:843 -- Connecting to existing Ray cluster at address: 172.26.34.55:6379 2023-02-13 15:39:49,646 INFO worker.py:843 -- Connecting to existing Ray cluster at address: 172.26.34.55:6379 (Worker pid=1347) 2023-02-13 15:40:20.536980: W tensorflow/core/framework/dataset.cc:769] Input of GeneratorDatasetOp::Dataset will not be optimized because the dataset does not implement the AsGraphDefInternal() method needed to apply optimizations. 1/97 [..............................] - ETA: 17s - loss: 0.2711 - accuracy: 0.8789 - auc: 0.9243 - precision: 0.7336 - recall: 0.6373 2/97 [..............................] - ETA: 7s - loss: 0.2648 - accuracy: 0.8827 - auc: 0.9278 - precision: 0.7425 - recall: 0.6433 3/97 [..............................] - ETA: 6s - loss: 0.2633 - accuracy: 0.8843 - auc: 0.9292 - precision: 0.7471 - recall: 0.6547 4/97 [>.............................] - ETA: 6s - loss: 0.2624 - accuracy: 0.8838 - auc: 0.9294 - precision: 0.7440 - recall: 0.6509 5/97 [>.............................] - ETA: 6s - loss: 0.2617 - accuracy: 0.8838 - auc: 0.9303 - precision: 0.7444 - recall: 0.6535 6/97 [>.............................] - ETA: 6s - loss: 0.2618 - accuracy: 0.8844 - auc: 0.9299 - precision: 0.7450 - r 7/97 [=>............................] - ETA: 6s - loss: 0.2622 - accuracy: 0.8843 - auc: 0.9294 - precision: 0.7439 - recall: 0.6509 8/97 [=>............................] - ETA: 6s - loss: 0.2627 - accuracy: 0.8845 - auc: 0.9292 - precision: 0.7452 - recall: 0.6515 9/97 [=>............................] - ETA: 6s - loss: 0.2624 - accuracy: 0.8848 - auc: 0.9292 - precision: 0.7458 - r10/97 [==>...........................] - ETA: 6s - loss: 0.2636 - accuracy: 0.8839 - auc: 0.9288 - precision: 0.7440 - recall: 0.6523 11/97 [==>...........................] - ETA: 6s - loss: 0.2626 - accuracy: 0.8845 - auc: 0.9292 - precision: 0.7445 - recall: 0.6538 12/97 [==>...........................] - ETA: 5s - loss: 0.2637 - accuracy: 0.8841 - auc: 0.9285 - precision: 0.7435 - r13/97 [===>..........................] - ETA: 5s - loss: 0.2639 - accuracy: 0.8840 - auc: 0.9283 - precision: 0.7437 - recall: 0.6514 14/97 [===>..........................] - ETA: 5s - loss: 0.2644 - accuracy: 0.8837 - auc: 0.9281 - precision: 0.7427 - recall: 0.6508 15/97 [===>..........................] - ETA: 5s - loss: 0.2642 - accuracy: 0.8836 - auc: 0.9282 - precision: 0.7427 - r16/97 [===>..........................] - ETA: 5s - loss: 0.2641 - accuracy: 0.8836 - auc: 0.9284 - precision: 0.7439 - recall: 0.6508 17/97 [====>.........................] - ETA: 5s - loss: 0.2639 - accuracy: 0.8839 - auc: 0.9287 - precision: 0.7447 - recall: 0.6521 18/97 [====>.........................] - ETA: 5s - loss: 0.2644 - accuracy: 0.8836 - auc: 0.9285 - precision: 0.7446 - r19/97 [====>.........................] - ETA: 5s - loss: 0.2641 - accuracy: 0.8839 - auc: 0.9286 - precision: 0.7450 - recall: 0.6517 20/97 [=====>........................] - ETA: 5s - loss: 0.2636 - accuracy: 0.8842 - auc: 0.9288 - precision: 0.7459 - recall: 0.6521 21/97 [=====>........................] - ETA: 5s - loss: 0.2636 - accuracy: 0.8841 - auc: 0.9288 - precision: 0.7458 - r22/97 [=====>........................] - ETA: 5s - loss: 0.2631 - accuracy: 0.8845 - auc: 0.9290 - precision: 0.7459 - recall: 0.6529 23/97 [======>.......................] - ETA: 5s - loss: 0.2631 - accuracy: 0.8845 - auc: 0.9290 - precision: 0.7461 - recall: 0.6529 24/97 [======>.......................] - ETA: 5s - loss: 0.2629 - accuracy: 0.8846 - auc: 0.9292 - precision: 0.7468 - r25/97 [======>.......................] - ETA: 4s - loss: 0.2629 - accuracy: 0.8847 - auc: 0.9292 - precision: 0.7467 - recall: 0.6538 26/97 [=======>......................] - ETA: 4s - loss: 0.2627 - accuracy: 0.8848 - auc: 0.9294 - precision: 0.7475 - recall: 0.6546 27/97 [=======>......................] - ETA: 4s - loss: 0.2622 - accuracy: 0.8850 - auc: 0.9295 - precision: 0.7476 - r28/97 [=======>......................] - ETA: 4s - loss: 0.2621 - accuracy: 0.8851 - auc: 0.9296 - precision: 0.7476 - recall: 0.6550 29/97 [=======>......................] - ETA: 4s - loss: 0.2619 - accuracy: 0.8853 - auc: 0.9297 - precision: 0.7483 - recall: 0.6554 30/97 [========>.....................] - ETA: 4s - loss: 0.2621 - accuracy: 0.8852 - auc: 0.9296 - precision: 0.7482 - r31/97 [========>.....................] - ETA: 4s - loss: 0.2622 - accuracy: 0.8852 - auc: 0.9297 - precision: 0.7484 - recall: 0.6559 32/97 [========>.....................] - ETA: 4s - loss: 0.2620 - accuracy: 0.8852 - auc: 0.9298 - precision: 0.7481 - recall: 0.6565 33/97 [=========>....................] - ETA: 4s - loss: 0.2617 - accuracy: 0.8853 - auc: 0.9299 - precision: 0.7481 - r34/97 [=========>....................] - ETA: 4s - loss: 0.2617 - accuracy: 0.8853 - auc: 0.9299 - precision: 0.7478 - recall: 0.6569 35/97 [=========>....................] - ETA: 4s - loss: 0.2619 - accuracy: 0.8851 - auc: 0.9298 - precision: 0.7475 - recall: 0.6563 36/97 [==========>...................] - ETA: 4s - loss: 0.2620 - accuracy: 0.8851 - auc: 0.9297 - precision: 0.7472 - r37/97 [==========>...................] - ETA: 4s - loss: 0.2617 - accuracy: 0.8853 - auc: 0.9299 - precision: 0.7476 - recall: 0.6570 38/97 [==========>...................] - ETA: 4s - loss: 0.2615 - accuracy: 0.8852 - auc: 0.9300 - precision: 0.7475 - recall: 0.6572 39/97 [===========>..................] - ETA: 3s - loss: 0.2616 - accuracy: 0.8852 - auc: 0.9300 - precision: 0.7474 - r40/97 [===========>..................] - ETA: 3s - loss: 0.2614 - accuracy: 0.8852 - auc: 0.9301 - precision: 0.7477 - recall: 0.6572 41/97 [===========>..................] - ETA: 3s - loss: 0.2610 - accuracy: 0.8855 - auc: 0.9303 - precision: 0.7481 - r42/97 [===========>..................] - ETA: 3s - loss: 0.2610 - accuracy: 0.8855 - auc: 0.9304 - precision: 0.7480 - recall: 0.6579 43/97 [============>.................] - ETA: 3s - loss: 0.2612 - accuracy: 0.8853 - auc: 0.9303 - precision: 0.7480 - recall: 0.6576 44/97 [============>.................] - ETA: 3s - loss: 0.2611 - accuracy: 0.8853 - auc: 0.9303 - precision: 0.7480 - r45/97 [============>.................] - ETA: 3s - loss: 0.2610 - accuracy: 0.8854 - auc: 0.9304 - precision: 0.7483 - recall: 0.6573 46/97 [=============>................] - ETA: 3s - loss: 0.2610 - accuracy: 0.8854 - auc: 0.9304 - precision: 0.7482 - recall: 0.6575 47/97 [=============>................] - ETA: 3s - loss: 0.2610 - accuracy: 0.8854 - auc: 0.9305 - precision: 0.7484 - r48/97 [=============>................] - ETA: 3s - loss: 0.2610 - accuracy: 0.8854 - auc: 0.9305 - precision: 0.7486 - recall: 0.6582 49/97 [==============>...............] - ETA: 3s - loss: 0.2609 - accuracy: 0.8855 - auc: 0.9306 - precision: 0.7486 - recall: 0.6582 50/97 [==============>...............] - ETA: 3s - loss: 0.2608 - accuracy: 0.8854 - auc: 0.9306 - precision: 0.7482 - r51/97 [==============>...............] - ETA: 3s - loss: 0.2608 - accuracy: 0.8853 - auc: 0.9307 - precision: 0.7481 - recall: 0.6582 52/97 [===============>..............] - ETA: 3s - loss: 0.2606 - accuracy: 0.8854 - auc: 0.9308 - precision: 0.7481 - recall: 0.6584 53/97 [===============>..............] - ETA: 2s - loss: 0.2605 - accuracy: 0.8855 - auc: 0.9308 - precision: 0.7481 - r54/97 [===============>..............] - ETA: 2s - loss: 0.2604 - accuracy: 0.8855 - auc: 0.9309 - precision: 0.7480 - recall: 0.6584 55/97 [================>.............] - ETA: 2s - loss: 0.2602 - accuracy: 0.8855 - auc: 0.9310 - precision: 0.7485 - recall: 0.6582 56/97 [================>.............] - ETA: 2s - loss: 0.2600 - accuracy: 0.8856 - auc: 0.9311 - precision: 0.7489 - r57/97 [================>.............] - ETA: 2s - loss: 0.2600 - accuracy: 0.8857 - auc: 0.9312 - precision: 0.7491 - recall: 0.6587 58/97 [================>.............] - ETA: 2s - loss: 0.2600 - accuracy: 0.8856 - auc: 0.9312 - precision: 0.7490 - recall: 0.6588 59/97 [=================>............] - ETA: 2s - loss: 0.2598 - accuracy: 0.8857 - auc: 0.9312 - precision: 0.7491 - r60/97 [=================>............] - ETA: 2s - loss: 0.2597 - accuracy: 0.8858 - auc: 0.9313 - precision: 0.7493 - recall: 0.6590 61/97 [=================>............] - ETA: 2s - loss: 0.2597 - accuracy: 0.8858 - auc: 0.9314 - precision: 0.7492 - r62/97 [==================>...........] - ETA: 2s - loss: 0.2597 - accuracy: 0.8857 - auc: 0.9314 - precision: 0.7493 - recall: 0.6590 63/97 [==================>...........] - ETA: 2s - loss: 0.2595 - accuracy: 0.8858 - auc: 0.9315 - precision: 0.7493 - recall: 0.6593 64/97 [==================>...........] - ETA: 2s - loss: 0.2594 - accuracy: 0.8859 - auc: 0.9315 - precision: 0.7494 - r65/97 [===================>..........] - ETA: 2s - loss: 0.2592 - accuracy: 0.8860 - auc: 0.9317 - precision: 0.7495 - recall: 0.6605 66/97 [===================>..........] - ETA: 2s - loss: 0.2591 - accuracy: 0.8860 - auc: 0.9317 - precision: 0.7496 - recall: 0.6609 67/97 [===================>..........] - ETA: 2s - loss: 0.2590 - accuracy: 0.8861 - auc: 0.9318 - precision: 0.7495 - r68/97 [====================>.........] - ETA: 1s - loss: 0.2588 - accuracy: 0.8862 - auc: 0.9319 - precision: 0.7498 - recall: 0.6615 69/97 [====================>.........] - ETA: 1s - loss: 0.2589 - accuracy: 0.8862 - auc: 0.9318 - precision: 0.7495 - recall: 0.6614 70/97 [====================>.........] - ETA: 1s - loss: 0.2588 - accuracy: 0.8862 - auc: 0.9319 - precision: 0.7494 - r71/97 [====================>.........] - ETA: 1s - loss: 0.2587 - accuracy: 0.8862 - auc: 0.9319 - precision: 0.7494 - recall: 0.6618 72/97 [=====================>........] - ETA: 1s - loss: 0.2586 - accuracy: 0.8863 - auc: 0.9320 - precision: 0.7496 - recall: 0.6623 73/97 [=====================>........] - ETA: 1s - loss: 0.2585 - accuracy: 0.8864 - auc: 0.9321 - precision: 0.7499 - r74/97 [=====================>........] - ETA: 1s - loss: 0.2584 - accuracy: 0.8865 - auc: 0.9322 - precision: 0.7499 - recall: 0.6628 75/97 [======================>.......] - ETA: 1s - loss: 0.2583 - accuracy: 0.8865 - auc: 0.9322 - precision: 0.7499 - recall: 0.6631 76/97 [======================>.......] - ETA: 1s - loss: 0.2582 - accuracy: 0.8866 - auc: 0.9323 - precision: 0.7499 - r77/97 [======================>.......] - ETA: 1s - loss: 0.2581 - accuracy: 0.8866 - auc: 0.9324 - precision: 0.7500 - recall: 0.6638 78/97 [=======================>......] - ETA: 1s - loss: 0.2580 - accuracy: 0.8867 - auc: 0.9324 - precision: 0.7501 - recall: 0.6642 79/97 [=======================>......] - ETA: 1s - loss: 0.2578 - accuracy: 0.8868 - auc: 0.9325 - precision: 0.7502 - r80/97 [=======================>......] - ETA: 1s - loss: 0.2576 - accuracy: 0.8869 - auc: 0.9326 - precision: 0.7502 - recall: 0.6649 81/97 [========================>.....] - ETA: 1s - loss: 0.2574 - accuracy: 0.8870 - auc: 0.9327 - precision: 0.7504 - recall: 0.6651 82/97 [========================>.....] - ETA: 1s - loss: 0.2574 - accuracy: 0.8871 - auc: 0.9328 - precision: 0.7505 - r83/97 [========================>.....] - ETA: 0s - loss: 0.2573 - accuracy: 0.8871 - auc: 0.9328 - precision: 0.7505 - recall: 0.6658 84/97 [========================>.....] - ETA: 0s - loss: 0.2571 - accuracy: 0.8872 - auc: 0.9329 - precision: 0.7504 - recall: 0.6659 85/97 [=========================>....] - ETA: 0s - loss: 0.2570 - accuracy: 0.8872 - auc: 0.9330 - precision: 0.7505 - r86/97 [=========================>....] - ETA: 0s - loss: 0.2570 - accuracy: 0.8872 - auc: 0.9330 - precision: 0.7503 - recall: 0.6666 87/97 [=========================>....] - ETA: 0s - loss: 0.2568 - accuracy: 0.8873 - auc: 0.9331 - precision: 0.7505 - recall: 0.6669 88/97 [==========================>...] - ETA: 0s - loss: 0.2567 - accuracy: 0.8873 - auc: 0.9332 - precision: 0.7505 - r89/97 [==========================>...] - ETA: 0s - loss: 0.2566 - accuracy: 0.8873 - auc: 0.9333 - precision: 0.7506 - recall: 0.6675 90/97 [==========================>...] - ETA: 0s - loss: 0.2564 - accuracy: 0.8874 - auc: 0.9334 - precision: 0.7507 - recall: 0.6679 91/97 [===========================>..] - ETA: 0s - loss: 0.2563 - accuracy: 0.8875 - auc: 0.9335 - precision: 0.7508 - r92/97 [===========================>..] - ETA: 0s - loss: 0.2562 - accuracy: 0.8875 - auc: 0.9335 - precision: 0.7508 - recall: 0.6683 93/97 [===========================>..] - ETA: 0s - loss: 0.2562 - accuracy: 0.8875 - auc: 0.9336 - precision: 0.7508 - recall: 0.6686 94/97 [============================>.] - ETA: 0s - loss: 0.2561 - accuracy: 0.8875 - auc: 0.9336 - precision: 0.7508 - r95/97 [============================>.] - ETA: 0s - loss: 0.2561 - accuracy: 0.8875 - auc: 0.9337 - precision: 0.7508 - recall: 0.6690 96/97 [============================>.] - ETA: 0s - loss: 0.2561 - accuracy: 0.8875 - auc: 0.9337 - precision: 0.7507 - recall: 0.6691 Evaluation results: validation_loss: 0.25619223713874817 validation_accuracy: 0.8874637484550476 validation_auc: 0.9335925579071045 validation_precision: 0.7505918145179749 validation_recall: 0.6687561273574829 97/97 [==============================] - ETA: 0s - loss: 0.2562 - accuracy: 0.8875 - auc: 0.9336 - precision: 0.7506 - r97/97 [==============================] - 7s 67ms/step - loss: 0.2562 - accuracy: 0.8875 - auc: 0.9336 - precision: 0.7506 - recall: 0.6688 (Worker pid=1347) WARNING:absl:Found untraced functions such as _update_step_xla while saving (showing 1 of 1). These functions will not be directly callable after loading. Stopping orca context Could it be an issue caused by redis version or version of other packages?
lalalapotter
commented
1 year ago Will check it.
hamham223
commented
1 year ago Can't reproduce the ray error at est.shutdown(), no exceptions occured.
Seemly errors are caused by ray.shutdown() in https://github.com/intel-analytics/BigDL/blob/main/python/orca/src/bigdl/orca/ray/raycontext.py#L93
Exception is due to redis connection:
Exception in thread ray_import_thread:
Traceback (most recent call last):
File "/home/yinchen/anaconda3/envs/yinchen/lib/python3.7/threading.py", line 926, in _bootstrap_inner
self.run()
File "/home/yinchen/anaconda3/envs/yinchen/lib/python3.7/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
File "/home/yinchen/anaconda3/envs/yinchen/lib/python3.7/site-packages/ray/_private/import_thread.py", line 74, in _run
msg = import_pubsub_client.get_message()
File "/home/yinchen/anaconda3/envs/yinchen/lib/python3.7/site-packages/redis/client.py", line 1586, in get_message
response = self.parse_response(block=False, timeout=timeout)
File "/home/yinchen/anaconda3/envs/yinchen/lib/python3.7/site-packages/redis/client.py", line 1436, in parse_response
if not block and not self._execute(conn, conn.can_read, timeout=timeout):
File "/home/yinchen/anaconda3/envs/yinchen/lib/python3.7/site-packages/redis/client.py", line 1422, in _execute
lambda error: self._disconnect_raise_connect(conn, error),
File "/home/yinchen/anaconda3/envs/yinchen/lib/python3.7/site-packages/redis/retry.py", line 48, in call_with_retry
fail(error)
File "/home/yinchen/anaconda3/envs/yinchen/lib/python3.7/site-packages/redis/client.py", line 1422, in <lambda>
lambda error: self._disconnect_raise_connect(conn, error),
File "/home/yinchen/anaconda3/envs/yinchen/lib/python3.7/site-packages/redis/client.py", line 1409, in _disconnect_raise_connect
raise error
File "/home/yinchen/anaconda3/envs/yinchen/lib/python3.7/site-packages/redis/retry.py", line 45, in call_with_retry
return do()
File "/home/yinchen/anaconda3/envs/yinchen/lib/python3.7/site-packages/redis/client.py", line 1421, in <lambda>
lambda: command(*args, **kwargs),
File "/home/yinchen/anaconda3/envs/yinchen/lib/python3.7/site-packages/redis/connection.py", line 804, in can_read
return self._parser.can_read(timeout)
File "/home/yinchen/anaconda3/envs/yinchen/lib/python3.7/site-packages/redis/connection.py", line 416, in can_read
return self.read_from_socket(timeout=timeout, raise_on_timeout=False)
File "/home/yinchen/anaconda3/envs/yinchen/lib/python3.7/site-packages/redis/connection.py", line 428, in read_from_socket
raise ConnectionError(SERVER_CLOSED_CONNECTION_ERROR)
redis.exceptions.ConnectionError: Connection closed by server.
Run TF2 estimator ray backend, encountered following error:
Issue observed for ray backend only. Differ from issue #6838, it appears before
stop_orca_context. It should be caused by theestimator.shutdown(), refer to link. But it doesn't impact the running. Need to investigate.