Open qrdlgit opened 11 months ago
Getting strange, non deterministic error when doing fit. This is on the kaggle environment, just using !pip install autokeras
--------------------------------------------------------------------------- TypeError Traceback (most recent call last) Cell In[3], line 92 87 with CustomObjectScope({'custom_metric': custom_metric}): 88 # Initialize and train the AutoKeras structured data classifier 89 clf = StructuredDataClassifier(max_trials=mts,metrics=[custom_metric], 90 objective = kerastuner.Objective("val_custom_metric", direction="min"), 91 project_name = f'sdc30h_{split}') ---> 92 clf.fit(X_train, y_train, validation_data = [X_test, y_test], epochs = 1000, 93 verbose=0, callbacks=[CustomCallback(), 94 tf.keras.callbacks.EarlyStopping( 95 monitor="val_custom_metric", 96 min_delta=0, 97 patience=100, 98 verbose=1, 99 mode="min", 100 baseline=None, 101 restore_best_weights=True, 102 start_from_epoch=0, 103 )]) # Set verbose to 1 or 2 to see the training progress 104 print("y_test", list(y_test), list(test_index)) 105 metric = clf.evaluate(X_test, y_test, verbose=0) File /opt/conda/lib/python3.10/site-packages/autokeras/tasks/structured_data.py:326, in StructuredDataClassifier.fit(self, x, y, epochs, callbacks, validation_split, validation_data, **kwargs) 279 def fit( 280 self, 281 x=None, (...) 287 **kwargs 288 ): 289 """Search for the best model and hyperparameters for the AutoModel. 290 291 # Arguments (...) 324 validation loss values and validation metrics values (if applicable). 325 """ --> 326 history = super().fit( 327 x=x, 328 y=y, 329 epochs=epochs, 330 callbacks=callbacks, 331 validation_split=validation_split, 332 validation_data=validation_data, 333 **kwargs 334 ) 335 return history File /opt/conda/lib/python3.10/site-packages/autokeras/tasks/structured_data.py:139, in BaseStructuredDataPipeline.fit(self, x, y, epochs, callbacks, validation_split, validation_data, **kwargs) 135 validation_data = self._read_from_csv(x_val, y_val) 137 self.check_in_fit(x) --> 139 history = super().fit( 140 x=x, 141 y=y, 142 epochs=epochs, 143 callbacks=callbacks, 144 validation_split=validation_split, 145 validation_data=validation_data, 146 **kwargs 147 ) 148 return history File /opt/conda/lib/python3.10/site-packages/autokeras/auto_model.py:292, in AutoModel.fit(self, x, y, batch_size, epochs, callbacks, validation_split, validation_data, verbose, **kwargs) 287 if validation_data is None and validation_split: 288 dataset, validation_data = data_utils.split_dataset( 289 dataset, validation_split 290 ) --> 292 history = self.tuner.search( 293 x=dataset, 294 epochs=epochs, 295 callbacks=callbacks, 296 validation_data=validation_data, 297 validation_split=validation_split, 298 verbose=verbose, 299 **kwargs 300 ) 302 return history File /opt/conda/lib/python3.10/site-packages/autokeras/engine/tuner.py:193, in AutoTuner.search(self, epochs, callbacks, validation_split, verbose, **fit_kwargs) 191 self._try_build(hp) 192 self.oracle.update_space(hp) --> 193 super().search( 194 epochs=epochs, callbacks=new_callbacks, verbose=verbose, **fit_kwargs 195 ) 197 # Train the best model use validation data. 198 # Train the best model with enough number of epochs. 199 if validation_split > 0 or early_stopping_inserted: File /opt/conda/lib/python3.10/site-packages/keras_tuner/engine/base_tuner.py:231, in BaseTuner.search(self, *fit_args, **fit_kwargs) 229 self.on_trial_begin(trial) 230 self._try_run_and_update_trial(trial, *fit_args, **fit_kwargs) --> 231 self.on_trial_end(trial) 232 self.on_search_end() File /opt/conda/lib/python3.10/site-packages/keras_tuner/engine/base_tuner.py:335, in BaseTuner.on_trial_end(self, trial) 329 def on_trial_end(self, trial): 330 """Called at the end of a trial. 331 332 Args: 333 trial: A `Trial` instance. 334 """ --> 335 self.oracle.end_trial(trial) 336 # Display needs the updated trial scored by the Oracle. 337 self._display.on_trial_end(self.oracle.get_trial(trial.trial_id)) File /opt/conda/lib/python3.10/site-packages/keras_tuner/engine/oracle.py:107, in synchronized.<locals>.wrapped_func(*args, **kwargs) 105 LOCKS[oracle].acquire() 106 THREADS[oracle] = thread_name --> 107 ret_val = func(*args, **kwargs) 108 if need_acquire: 109 THREADS[oracle] = None File /opt/conda/lib/python3.10/site-packages/keras_tuner/engine/oracle.py:434, in Oracle.end_trial(self, trial) 432 if not self._retry(trial): 433 self.end_order.append(trial.trial_id) --> 434 self._check_consecutive_failures() 436 self._save_trial(trial) 437 self.save() File /opt/conda/lib/python3.10/site-packages/keras_tuner/engine/oracle.py:387, in Oracle._check_consecutive_failures(self) 384 consecutive_failures = 0 385 if consecutive_failures == self.max_consecutive_failed_trials: 386 raise RuntimeError( --> 387 "Number of consecutive failures excceeded the limit " 388 f"of {self.max_consecutive_failed_trials}.\n" 389 + trial.message 390 ) TypeError: can only concatenate str (not "NoneType") to str
I am also encountering this issue.
Getting strange, non deterministic error when doing fit. This is on the kaggle environment, just using !pip install autokeras