Open moghadas76 opened 3 months ago
Evaluation multiple windows took too long(25Min!!!). I am tiered from this package. It is real garbage!
from gluonts.dataset.common import DataEntry, Dataset from gluonts.dataset.field_names import FieldName from gluonts.dataset.split import split from gluonts.dataset.util import period_index from gluonts.evaluation import Evaluator def _to_dataframe(input_label) -> pd.DataFrame: start = input_label[0][FieldName.START] targets = [entry[FieldName.TARGET] for entry in input_label] full_target = np.concatenate(targets, axis=-1) index = period_index( {FieldName.START: start, FieldName.TARGET: full_target} ) return pd.DataFrame(full_target.transpose(), index=index) def _make_evaluation_predictions( dataset: Dataset, predictor, num_samples: int = 100, ): window_length = predictor.prediction_length + predictor.lead_time N_WINDOWS = 50 _, test_template = split(dataset, offset=-window_length*N_WINDOWS) test_data = test_template.generate_instances(window_length, windows=N_WINDOWS) return ( predictor.predict(test_data.input, num_samples=num_samples), map(_to_dataframe, test_data), ) forecast_it, ts_it = _make_evaluation_predictions( dataset=dataset_04.test, # test dataset predictor=predictor, # predictor num_samples=200, # number of sample paths we want for evaluation ) evaluator = Evaluator( custom_eval_fn={ "mae_sota": [metrics.mae, "mean", "median"], "mape_sota": [metrics.mape, "mean", "median"], "rmse_sota": [metrics.rmse, "mean", "median"] } ) agg_m, item_m = evaluator( ts_iterator=ts_it, fcst_iterator=forecast_it, num_series=len(dataset_04.test), ) print(json.dumps(agg_m, indent=4)) print(agg_m["mae_sota"], agg_m["mape_sota"], agg_m["rmse_sota"])
(Add as much information about your environment as possible, e.g. dependencies versions.)
Hi @moghadas76, can you share details about the test data that you're using and the model? Without that, it's hard to tell if there's an actual issue here
Description
Evaluation multiple windows took too long(25Min!!!). I am tiered from this package. It is real garbage!
To Reproduce
Environment
(Add as much information about your environment as possible, e.g. dependencies versions.)