"W tensorflow/compiler/jit/mark_for_compilation_pass.cc:1412] (One-time warning): Not using XLA:CPU for cluster because envvar TF_XLA_FLAGS=--tf_xla_cpu_global_jit was not set. If you want XLA:CPU, either set that envvar, or use experimental_jit_scope to enable XLA:CPU. To confirm that XLA is active, pass --vmodule=xla_compilation_cache=1 (as a proper command-line flag, not via TF_XLA_FLAGS) or set the envvar XLA_FLAGS=--xla_hlo_profile. " & "tensorflow.python.framework.errors_impl.NotFoundError: /home/paperspace/stylegan2/dnnlib/tflib/_cudacache/fused_bias_act_afa4ff47e864043461655b92eafcdf5f.so: undefined symbol: _ZN10tensorflow12OpDefBuilder5InputESs" #101
Hello, does anyone know how to fix these problems? I reinstall "pip install tensorflow-gpu", but still not work. I train a custom dataset through Jupyter Notebook on paperspace.
paperspace@psesbu2bo:~/stylegan2$ python run_training.py --num-gpus=1 --data-dir=./datasets --config=config-f --dataset=mix --mirror-augment=true --metrics=none
Local submit - run_dir: results/00004-stylegan2-mix-1gpu-config-f
dnnlib: Running training.training_loop.training_loop() on localhost...
Streaming data using training.dataset.TFRecordDataset...
Dataset shape = [3, 1024, 1024]
Dynamic range = [0, 255]
Label size = 0
2021-07-11 15:46:54.278075: W tensorflow/compiler/jit/mark_for_compilation_pass.cc:1412] (One-time warning): Not using XLA:CPU for cluster because envvar TF_XLA_FLAGS=--tf_xla_cpu_global_jit was not set. If you want XLA:CPU, either set that envvar, or use experimental_jit_scope to enable XLA:CPU. To confirm that XLA is active, pass --vmodule=xla_compilation_cache=1 (as a proper command-line flag, not via TF_XLA_FLAGS) or set the envvar XLA_FLAGS=--xla_hlo_profile.
Constructing networks...
Setting up TensorFlow plugin "fused_bias_act.cu": Preprocessing... Loading... Failed!
Traceback (most recent call last):
File "run_training.py", line 192, in
main()
File "run_training.py", line 187, in main
run(vars(args))
File "run_training.py", line 120, in run
dnnlib.submit_run(kwargs)
File "/home/paperspace/stylegan2/dnnlib/submission/submit.py", line 343, in submit_run
return farm.submit(submit_config, host_run_dir)
File "/home/paperspace/stylegan2/dnnlib/submission/internal/local.py", line 22, in submit
return run_wrapper(submit_config)
File "/home/paperspace/stylegan2/dnnlib/submission/submit.py", line 280, in run_wrapper
run_func_obj(submit_config.run_func_kwargs)
File "/home/paperspace/stylegan2/training/training_loop.py", line 149, in training_loop
G = tflib.Network('G', num_channels=training_set.shape[0], resolution=training_set.shape[1], label_size=training_set.label_size, G_args)
File "/home/paperspace/stylegan2/dnnlib/tflib/network.py", line 97, in init
self._init_graph()
File "/home/paperspace/stylegan2/dnnlib/tflib/network.py", line 154, in _init_graph
out_expr = self._build_func(self.input_templates, build_kwargs)
File "/home/paperspace/stylegan2/training/networks_stylegan2.py", line 186, in G_main
components.synthesis = tflib.Network('G_synthesis', func_name=globals()[synthesis_func], kwargs)
File "/home/paperspace/stylegan2/dnnlib/tflib/network.py", line 97, in init
self._init_graph()
File "/home/paperspace/stylegan2/dnnlib/tflib/network.py", line 154, in _init_graph
out_expr = self._build_func(self.input_templates, **build_kwargs)
File "/home/paperspace/stylegan2/training/networks_stylegan2.py", line 491, in G_synthesis_stylegan2
x = layer(x, layer_idx=0, fmaps=nf(1), kernel=3)
File "/home/paperspace/stylegan2/training/networks_stylegan2.py", line 455, in layer
x = modulated_conv2d_layer(x, dlatents_in[:, layer_idx], fmaps=fmaps, kernel=kernel, up=up, resample_kernel=resample_kernel, fused_modconv=fused_modconv)
File "/home/paperspace/stylegan2/training/networks_stylegan2.py", line 99, in modulated_conv2d_layer
s = apply_bias_act(s, bias_var=mod_bias_var) + 1 # [BI] Add bias (initially 1).
File "/home/paperspace/stylegan2/training/networks_stylegan2.py", line 68, in apply_bias_act
return fused_bias_act(x, b=tf.cast(b, x.dtype), act=act, alpha=alpha, gain=gain)
File "/home/paperspace/stylegan2/dnnlib/tflib/ops/fused_bias_act.py", line 68, in fused_bias_act
return impl_dict[impl](x=x, b=b, axis=axis, act=act, alpha=alpha, gain=gain)
File "/home/paperspace/stylegan2/dnnlib/tflib/ops/fused_bias_act.py", line 122, in _fused_bias_act_cuda
cuda_kernel = _get_plugin().fused_bias_act
File "/home/paperspace/stylegan2/dnnlib/tflib/ops/fused_bias_act.py", line 16, in _get_plugin
return custom_ops.get_plugin(os.path.splitext(file)[0] + '.cu')
File "/home/paperspace/stylegan2/dnnlib/tflib/custom_ops.py", line 156, in get_plugin
plugin = tf.load_op_library(bin_file)
File "/home/paperspace/.local/lib/python3.7/site-packages/tensorflow/python/framework/load_library.py", line 61, in load_op_library
lib_handle = py_tf.TF_LoadLibrary(library_filename)
tensorflow.python.framework.errors_impl.NotFoundError: /home/paperspace/stylegan2/dnnlib/tflib/_cudacache/fused_bias_act_afa4ff47e864043461655b92eafcdf5f.so: undefined symbol: _ZN10tensorflow12OpDefBuilder5InputESs
Hello, does anyone know how to fix these problems? I reinstall "pip install tensorflow-gpu", but still not work. I train a custom dataset through Jupyter Notebook on paperspace.
paperspace@psesbu2bo:~/stylegan2$ python run_training.py --num-gpus=1 --data-dir=./datasets --config=config-f --dataset=mix --mirror-augment=true --metrics=none Local submit - run_dir: results/00004-stylegan2-mix-1gpu-config-f dnnlib: Running training.training_loop.training_loop() on localhost... Streaming data using training.dataset.TFRecordDataset... Dataset shape = [3, 1024, 1024] Dynamic range = [0, 255] Label size = 0 2021-07-11 15:46:54.278075: W tensorflow/compiler/jit/mark_for_compilation_pass.cc:1412] (One-time warning): Not using XLA:CPU for cluster because envvar TF_XLA_FLAGS=--tf_xla_cpu_global_jit was not set. If you want XLA:CPU, either set that envvar, or use experimental_jit_scope to enable XLA:CPU. To confirm that XLA is active, pass --vmodule=xla_compilation_cache=1 (as a proper command-line flag, not via TF_XLA_FLAGS) or set the envvar XLA_FLAGS=--xla_hlo_profile. Constructing networks... Setting up TensorFlow plugin "fused_bias_act.cu": Preprocessing... Loading... Failed! Traceback (most recent call last): File "run_training.py", line 192, in
main()
File "run_training.py", line 187, in main
run(vars(args))
File "run_training.py", line 120, in run
dnnlib.submit_run(kwargs)
File "/home/paperspace/stylegan2/dnnlib/submission/submit.py", line 343, in submit_run
return farm.submit(submit_config, host_run_dir)
File "/home/paperspace/stylegan2/dnnlib/submission/internal/local.py", line 22, in submit
return run_wrapper(submit_config)
File "/home/paperspace/stylegan2/dnnlib/submission/submit.py", line 280, in run_wrapper
run_func_obj(submit_config.run_func_kwargs)
File "/home/paperspace/stylegan2/training/training_loop.py", line 149, in training_loop
G = tflib.Network('G', num_channels=training_set.shape[0], resolution=training_set.shape[1], label_size=training_set.label_size, G_args)
File "/home/paperspace/stylegan2/dnnlib/tflib/network.py", line 97, in init
self._init_graph()
File "/home/paperspace/stylegan2/dnnlib/tflib/network.py", line 154, in _init_graph
out_expr = self._build_func(self.input_templates, build_kwargs)
File "/home/paperspace/stylegan2/training/networks_stylegan2.py", line 186, in G_main
components.synthesis = tflib.Network('G_synthesis', func_name=globals()[synthesis_func], kwargs)
File "/home/paperspace/stylegan2/dnnlib/tflib/network.py", line 97, in init
self._init_graph()
File "/home/paperspace/stylegan2/dnnlib/tflib/network.py", line 154, in _init_graph
out_expr = self._build_func(self.input_templates, **build_kwargs)
File "/home/paperspace/stylegan2/training/networks_stylegan2.py", line 491, in G_synthesis_stylegan2
x = layer(x, layer_idx=0, fmaps=nf(1), kernel=3)
File "/home/paperspace/stylegan2/training/networks_stylegan2.py", line 455, in layer
x = modulated_conv2d_layer(x, dlatents_in[:, layer_idx], fmaps=fmaps, kernel=kernel, up=up, resample_kernel=resample_kernel, fused_modconv=fused_modconv)
File "/home/paperspace/stylegan2/training/networks_stylegan2.py", line 99, in modulated_conv2d_layer
s = apply_bias_act(s, bias_var=mod_bias_var) + 1 # [BI] Add bias (initially 1).
File "/home/paperspace/stylegan2/training/networks_stylegan2.py", line 68, in apply_bias_act
return fused_bias_act(x, b=tf.cast(b, x.dtype), act=act, alpha=alpha, gain=gain)
File "/home/paperspace/stylegan2/dnnlib/tflib/ops/fused_bias_act.py", line 68, in fused_bias_act
return impl_dict[impl](x=x, b=b, axis=axis, act=act, alpha=alpha, gain=gain)
File "/home/paperspace/stylegan2/dnnlib/tflib/ops/fused_bias_act.py", line 122, in _fused_bias_act_cuda
cuda_kernel = _get_plugin().fused_bias_act
File "/home/paperspace/stylegan2/dnnlib/tflib/ops/fused_bias_act.py", line 16, in _get_plugin
return custom_ops.get_plugin(os.path.splitext(file)[0] + '.cu')
File "/home/paperspace/stylegan2/dnnlib/tflib/custom_ops.py", line 156, in get_plugin
plugin = tf.load_op_library(bin_file)
File "/home/paperspace/.local/lib/python3.7/site-packages/tensorflow/python/framework/load_library.py", line 61, in load_op_library
lib_handle = py_tf.TF_LoadLibrary(library_filename)
tensorflow.python.framework.errors_impl.NotFoundError: /home/paperspace/stylegan2/dnnlib/tflib/_cudacache/fused_bias_act_afa4ff47e864043461655b92eafcdf5f.so: undefined symbol: _ZN10tensorflow12OpDefBuilder5InputESs