search

EmuKit / emukit

A Python-based toolbox of various methods in decision making, uncertainty quantification and statistical emulation: multi-fidelity, experimental design, Bayesian optimisation, Bayesian quadrature, etc.
https://emukit.github.io/emukit/
Apache License 2.0
605 stars 128 forks source link

>2 level NonLinearMultiFidelityModel incorrect prediction on levels above 2nd #321

Closed rns294 closed 4 years ago

rns294 commented 4 years ago

Running a test with NonLinearMultiFidelityModel using 3 levels of code. The first two level predictions are accurate but the 3rd level does not interpolate the training points and has a large error/variance. I setup a 3 level model using similar code from paraklas that the emukit code was based on and was able to predict all 3 levels with high accuracy/low variance.

import numpy as np

from emukit.multi_fidelity.convert_lists_to_array import convert_x_list_to_array, convert_xy_lists_to_arrays

from emukit.multi_fidelity.models.non_linear_multi_fidelity_model import make_non_linear_kernels, NonLinearMultiFidelityModel

import GPy

from matplotlib import pyplot as plt

np.random.seed(12345)

def fh(x): return (6*x-2)*2np.sin(12*x-4)

def fm(x): return 0.4*fh(x)-x-1

def fl(x): return 0.5fh(x)+10(x-0.5)-(5)

Nts = 200 x_plot = np.linspace(0, 1, Nts)[:, None] y_plot_l = fl(x_plot) y_plot_m = fm(x_plot) y_plot_h = fh(x_plot)

n0, n1, n2 = 6, 15, 40 Xtrain = np.random.rand(100, 1) idx = np.random.permutation(100)

x_train_h = Xtrain[idx[0:n0], :] x_train_m = Xtrain[idx[0:n1], :] x_train_l = Xtrain[idx[0:n2], :]

y_train_l = fl(x_train_l) y_train_m = fm(x_train_m) y_train_h = fh(x_train_h)

X_train, Y_train = convert_xy_lists_to_arrays([x_train_l, x_train_m, x_train_h], [y_train_l, y_train_m, y_train_h])

X_plot = convert_x_list_to_array([x_plot, x_plot, x_plot]) X_plot_l = X_plot[:len(x_plot)] X_plot_m = X_plot[len(x_plot):-len(x_plot)] X_plot_h = X_plot[-len(x_plot):]

plt.figure(figsize=(12, 8)) plt.plot(x_plot, y_plot_l, 'b') plt.plot(x_plot, y_plot_m, 'r') plt.plot(x_plot, y_plot_h, 'k') plt.scatter(x_train_l, y_train_l, color='b', s=40) plt.scatter(x_train_m, y_train_m, color='r', s=40) plt.scatter(x_train_h, y_train_h, color='k', s=40) plt.ylabel('f (x)') plt.xlabel('x') plt.legend(['Low fidelity', 'Mid fidelity', 'High fidelity'])

base_kernel = GPy.kern.RBF kernels = make_non_linear_kernels(base_kernel, 3, X_train.shape[1] - 1) nonlin_mf_model = NonLinearMultiFidelityModel(X_train, Y_train, n_fidelities=3, kernels=kernels, verbose=True, optimization_restarts=10) for m in nonlin_mf_model.models: m.Gaussian_noise.variance.fix(1e-6)

nonlin_mf_model.optimize()

hf_mean_nonlin_mf_model, hf_var_nonlin_mf_model = nonlin_mf_model.predict(X_plot_h) hf_std_nonlin_mf_model = np.sqrt(hf_var_nonlin_mf_model)

mf_mean_nonlin_mf_model, mf_var_nonlin_mf_model = nonlin_mf_model.predict(X_plot_m) mf_std_nonlin_mf_model = np.sqrt(mf_var_nonlin_mf_model)

lf_mean_nonlin_mf_model, lf_var_nonlin_mf_model = nonlin_mf_model.predict(X_plot_l) lf_std_nonlin_mf_model = np.sqrt(lf_var_nonlin_mf_model)

error = np.linalg.norm(y_plot_l - lf_mean_nonlin_mf_model) / np.linalg.norm(y_plot_l) print(error)

error = np.linalg.norm(y_plot_m - mf_mean_nonlin_mf_model) / np.linalg.norm(y_plot_m) print(error)

error = np.linalg.norm(y_plot_h - hf_mean_nonlin_mf_model) / np.linalg.norm(y_plot_h) print(error)

plt.plot(x_plot, hf_mean_nonlin_mf_model, 'g--', linewidth=3) plt.plot(x_plot, mf_mean_nonlin_mf_model, 'm--', linewidth=3) plt.plot(x_plot, lf_mean_nonlin_mf_model, 'y--', linewidth=3)