google / automl

Google Brain AutoML
Apache License 2.0
6.26k stars 1.46k forks source link

Fatal Python error: Aborted #1214

Open yihchu opened 2 weeks ago

yihchu commented 2 weeks ago

Please help, my command is python main.py --mode=train --model_name=efficientnetv2-s --dataset_cfg=imagenet --model_dir=D:\Code\Pycharm\automl-master\efficientnetv2\data\

And it gives me the error below, how to fix it?

Fatal Python error: Aborted

Thread 0x000063f8 (most recent call first): File "E:\Anaconda3\Lib\threading.py", line 320 in wait File "E:\Anaconda3\Lib\site-packages\tensorflow\python\summary\writer\event_file_writer.py", line 261 in get File "E:\Anaconda3\Lib\site-packages\tensorflow\python\summary\writer\event_file_writer.py", line 204 in run File "E:\Anaconda3\Lib\threading.py", line 1038 in _bootstrap_inner File "E:\Anaconda3\Lib\threading.py", line 995 in _bootstrap

Thread 0x00007dc8 (most recent call first): File "E:\Anaconda3\Lib\site-packages\tensorflow\python\client\session.py", line 1478 in _call_tf_sessionrun File "E:\Anaconda3\Lib\site-packages\tensorflow\python\client\session.py", line 1385 in _run_fn File "E:\Anaconda3\Lib\site-packages\tensorflow\python\client\session.py", line 1402 in _do_call File "E:\Anaconda3\Lib\site-packages\tensorflow\python\client\session.py", line 1395 in _do_run File "E:\Anaconda3\Lib\site-packages\tensorflow\python\client\session.py", line 1215 in _run File "E:\Anaconda3\Lib\site-packages\tensorflow\python\client\session.py", line 972 in run File "E:\Anaconda3\Lib\site-packages\tensorflow\python\training\monitored_session.py", line 1228 in run File "E:\Anaconda3\Lib\site-packages\tensorflow\python\training\monitored_session.py", line 1464 in run File "E:\Anaconda3\Lib\site-packages\tensorflow\python\training\monitored_session.py", line 1397 in run File "E:\Anaconda3\Lib\site-packages\tensorflow\python\training\monitored_session.py", line 1307 in run File "E:\Anaconda3\Lib\site-packages\tensorflow\python\training\monitored_session.py", line 778 in run File "E:\Anaconda3\Lib\site-packages\tensorflow_estimator\python\estimator\estimator.py", line 1535 in _train_with_estimator_spec File "E:\Anaconda3\Lib\site-packages\tensorflow_estimator\python\estimator\estimator.py", line 1219 in _train_model_default File "E:\Anaconda3\Lib\site-packages\tensorflow_estimator\python\estimator\estimator.py", line 1188 in _train_model File "E:\Anaconda3\Lib\site-packages\tensorflow_estimator\python\estimator\estimator.py", line 360 in train File "E:\Anaconda3\Lib\site-packages\tensorflow_estimator\python\estimator\tpu\tpu_estimator.py", line 3096 in train File "D:\Code\Pycharm\automl-master\efficientnetv2\main.py", line 496 in main File "E:\Anaconda3\Lib\site-packages\absl\app.py", line 254 in _run_main File "E:\Anaconda3\Lib\site-packages\absl\app.py", line 308 in run File "D:\Code\Pycharm\automl-master\efficientnetv2\main.py", line 504 in

Extension modules: numpy.core._multiarray_umath, numpy.core._multiarray_tests, numpy.linalg._umath_linalg, numpy.fft._pocketfft_internal, numpy.random._common, numpy.random.bit_gen erator, numpy.random._bounded_integers, numpy.random._mt19937, numpy.random.mtrand, numpy.random._philox, numpy.random._pcg64, numpy.random._sfc64, numpy.random._generator, google. _upb._message, _cffi_backend, simplejson._speedups, h5py._errors, h5py.defs, h5py._objects, h5py.h5, h5py.utils, h5py.h5t, h5py.h5s, h5py.h5ac, h5py.h5p, h5py.h5r, h5py._proxy, h5p y._conv, h5py.h5z, h5py.h5a, h5py.h5d, h5py.h5ds, h5py.h5g, h5py.h5i, h5py.h5o, h5py.h5f, h5py.h5fd, h5py.h5pl, h5py.h5l, h5py._selector, scipy._lib._ccallback_c, scipy.sparse._spa rsetools, scipy.sparse._csparsetools, scipy.sparse.linalg._isolve._iterative, scipy.linalg._fblas, scipy.linalg._flapack, scipy.linalg.cython_lapack, scipy.linalg._cythonized_array _utils, scipy.linalg._solve_toeplitz, scipy.linalg._decomp_lu_cython, scipy.linalg._matfuncs_sqrtm_triu, scipy.linalg.cython_blas, scipy.linalg._matfuncs_expm, scipy.linalg._decomp _update, scipy.linalg._flinalg, scipy.sparse.linalg._dsolve._superlu, scipy.sparse.linalg._eigen.arpack._arpack, scipy.sparse.csgraph._tools, scipy.sparse.csgraph._shortest_path, s cipy.sparse.csgraph._traversal, scipy.sparse.csgraph._min_spanning_tree, scipy.sparse.csgraph._flow, scipy.sparse.csgraph._matching, scipy.sparse.csgraph._reordering, snappy._snapp y, lz4._version, lz4.frame._frame, zstandard.backend_c, PIL._imaging, pandas._libs.tslibs.np_datetime, pandas._libs.tslibs.dtypes, pandas._libs.tslibs.base, pandas._libs.tslibs.nat type, pandas._libs.tslibs.timezones, pandas._libs.tslibs.ccalendar, pandas._libs.tslibs.fields, pandas._libs.tslibs.timedeltas, pandas._libs.tslibs.tzconversion, pandas._libs.tslib s.timestamps, pandas._libs.properties, pandas._libs.tslibs.offsets, pandas._libs.tslibs.strptime, pandas._libs.tslibs.parsing, pandas._libs.tslibs.conversion, pandas._libs.tslibs.p eriod, pandas._libs.tslibs.vectorized, pandas._libs.ops_dispatch, pandas._libs.missing, pandas._libs.hashtable, pandas._libs.algos, pandas._libs.interval, pandas._libs.lib, pandas. _libs.hashing, pyarrow.lib, pyarrow._hdfsio, pandas._libs.tslib, pandas._libs.ops, numexpr.interpreter, pyarrow._compute, bottleneck.move, bottleneck.nonreduce, bottleneck.nonreduc e_axis, bottleneck.reduce, pandas._libs.arrays, pandas._libs.sparse, pandas._libs.reduction, pandas._libs.indexing, pandas._libs.index, pandas._libs.internals, pandas._libs.join, p andas._libs.writers, pandas._libs.window.aggregations, pandas._libs.window.indexers, pandas._libs.reshape, pandas._libs.groupby, pandas._libs.testing, pandas._libs.parsers, pandas. _libs.json, scipy.ndimage._nd_image, scipy.special._ufuncs_cxx, scipy.special._ufuncs, scipy.special._specfun, scipy.special._comb, scipy.special._ellip_harm_2, _ni_label, scipy.ndimage._ni_label, yaml._yaml (total: 127)

yihchu commented 2 weeks ago

Python 3.11, tensorflow 2.15.0

yihchu commented 2 weeks ago

If I use main_tf2.py, the error is: Windows fatal exception: access violation

Thread 0x00006514 (most recent call first): File "E:\Anaconda3\Lib\site-packages\tensorflow\python\eager\execute.py", line 53 in quick_execute File "E:\Anaconda3\Lib\site-packages\tensorflow\python\eager\context.py", line 1486 in call_function File "E:\Anaconda3\Lib\site-packages\tensorflow\python\eager\polymorphic_function\atomic_function.py", line 251 in call_flat File "E:\Anaconda3\Lib\site-packages\tensorflow\python\eager\polymorphic_function\atomic_function.py", line 216 in call_preflattened File "E:\Anaconda3\Lib\site-packages\tensorflow\python\eager\polymorphic_function\concrete_function.py", line 1323 in _call_flat File "E:\Anaconda3\Lib\site-packages\tensorflow\python\eager\polymorphic_function\tracing_compilation.py", line 139 in call_function File "E:\Anaconda3\Lib\site-packages\tensorflow\python\eager\polymorphic_function\polymorphic_function.py", line 905 in _call File "E:\Anaconda3\Lib\site-packages\tensorflow\python\eager\polymorphic_function\polymorphic_function.py", line 832 in call File "E:\Anaconda3\Lib\site-packages\tensorflow\python\util\traceback_utils.py", line 150 in error_handler File "E:\Anaconda3\Lib\site-packages\keras\src\engine\training.py", line 1807 in fit File "E:\Anaconda3\Lib\site-packages\keras\src\utils\traceback_utils.py", line 65 in error_handler File "D:\Code\Pycharm\automl-master\efficientnetv2\main_tf2.py", line 277 in main File "E:\Anaconda3\Lib\site-packages\absl\app.py", line 254 in _run_main File "E:\Anaconda3\Lib\site-packages\absl\app.py", line 308 in run File "D:\Code\Pycharm\automl-master\efficientnetv2\main_tf2.py", line 312 in