Not a Number (nan) Loss Values for RAR option

[mkutuk]

I tried to test the convection-diffusion equation with the RAR option. But when the RAR process begins the loss values return as nan. What can be the reason for this situation? How can I fix this problem?

[lululxvi]

There should be something wrong when you use RAR. You may check the added locations, index, etc.

[mkutuk]

I used the same codes which are written for Burger's Equation with RAR option. Here you can see the codes and the result.

from future import absolute_import from future import division from future import print_function from mpl_toolkits import mplot3d

import numpy as np import tensorflow as tf import math as mathe

import deepxde as dde import matplotlib.pyplot as plt import scipy.io

def main(): eps_fun = tf.Variable(0.0001) beta1 = tf.Variable(2.0) beta2 = tf.Variable(3.0) alpha = tf.Variable(1.0)

def pde(x, y):
    x1, x2 = x[:, 0:1], x[:, 1:]
    dy_x = tf.gradients(y, x)[0]                           # du/dx
    dy_x, dy_y = dy_x[:, 0:1], dy_x[:, 1:]                 # du/dx & du/dy
    dy_xx = tf.gradients(dy_x, x)[0][:, 0:1]               # d2u/dx2
    dy_yy = tf.gradients(dy_y, x)[0][:, 1:]                # d2u/dy2
    u_exact = (2 / np.pi)*tf.atan((1/tf.sqrt(eps_fun))*(-0.5 * x1 + x2 - 0.25))
    u_x_exact = -1 / (tf.sqrt(eps_fun)*np.pi*(((0.5 * x1 - x2 + 0.25)**2 / eps_fun) + 1))
    u_y_exact = 2 / (tf.sqrt(eps_fun)*np.pi*(((0.5 * x1 - x2 + 0.25)**2 / eps_fun) + 1))
    u_xx_exact = (0.5 * x1 - x2 + 0.25) / (
                eps_fun*tf.sqrt(eps_fun)*np.pi*(((0.5 * x1 - x2 + 0.25)**2 / eps_fun) + 1)**2)
    u_yy_exact = (2 * (x1 - 2*x2 + 0.5)) / (
                eps_fun*tf.sqrt(eps_fun)*np.pi*(((0.5 * x1 - x2 + 0.25)**2 / eps_fun) + 1)**2)
    f = -eps_fun * (u_xx_exact + u_yy_exact) + beta1*u_x_exact + beta2*u_y_exact + alpha*u_exact
    return -eps_fun * (dy_xx + dy_yy) + beta1*dy_x + beta2*dy_y + alpha*y - f  # residual

def boundary(x, on_boundary):
    return on_boundary

def func(x):
    eps_fun = 0.0001  # epsilon value
    u_bound = np.zeros([len(x), 1])
    for i in range(len(x)):
        u_bound[i] = (2 / np.pi) * mathe.atan((1 / mathe.sqrt(eps_fun))*(-0.5 * x[i, 0] + x[i, 1] - 0.25))
    return u_bound   # Boundary Values

geom = dde.geometry.Polygon([[0, 0], [1, 0], [1, 1], [0, 1]])    # square geometry
bc = dde.DirichletBC(geom, func, boundary)                                           # Dirichlet Boundary Condition

data = dde.data.PDE(
    geom,
    pde,
    bc,
    num_domain=1200,
    num_boundary=120,
    solution=func,
    num_test=1500,
)
scipy.io.savemat('x_train.mat', {"x_train": data.train_x})
scipy.io.savemat('x_test.mat', {"x_test": data.test_x})
net = dde.maps.FNN([2] + [50] * 4 + [1], "tanh", "Glorot uniform")        # NN Construction with 4 hidden layers
                                                         # (each layer has 50 neurons), input_size:2, output_size:1,
                                                                                    # tanh (activation function)

modeladam = dde.Model(data, net)

modeladam.compile("adam", lr=0.001)
modeladam.train(epochs=15000)
modeladam.compile("L-BFGS-B")
modeladam.train()

X = geom.random_points(1000)
err = 1
while err > (5 * 1e-3):
    f = modeladam.predict(X, operator=pde)
    err_eq = np.absolute(f)
    err = np.mean(err_eq)
    print("Mean residual: %.3e" % (err))
    x_id = np.argmax(err_eq)
    print("Adding new point:", X[x_id], "\n")
    data.add_anchors(X[x_id])
    early_stopping = dde.callbacks.EarlyStopping(min_delta=1e-4, patience=2000)
    modeladam.compile("adam", lr=1e-3)
    modeladam.train(
        epochs=10000, disregard_previous_best=True, callbacks=[early_stopping]
    )
    modeladam.compile("L-BFGS-B")
    losshistory, train_state = modeladam.train()
dde.saveplot(losshistory, train_state, issave=True, isplot=True)
adamloss_train = [i[0] for i in losshistory.loss_train]
adamloss_test = [i[0] for i in losshistory.loss_test]

plt.xlabel('# steps')
plt.ylabel('training loss')
plt.title('Training and test losses of Convection-Diffusion equation with RAR algorithm')
plt.yscale('log')

plt.plot(adamloss_train, 'c', linestyle='-.', label='adam_train')
plt.plot(adamloss_test, 'r', linestyle='--', label='adam_test')
leg = plt.legend()
plt.show()
dde.saveplot(losshistory, train_state, issave=False, isplot=False)

tmp = geom.uniform_boundary_points(100)
xx = geom.uniform_points(1000, True)
X = np.vstack((tmp, xx))
y_pred_adam = modeladam.predict(X)

y_true = (2 / np.pi) * np.arctan((1 / np.sqrt(0.0001)) * (-0.5 * X[:, 0:1] + X[:, 1:] - 0.25))

scipy.io.savemat('x.mat', {"x": X})
scipy.io.savemat('y_true.mat', {"y_true": y_true})
scipy.io.savemat('y_pred_adam.mat', {"y_pred_adam": y_pred_adam})

if name == "main": main()

[lululxvi]

It is strange. At step 15030 it is correct, which means the code should be OK. My guess is that SGD just blows up. Try a smaller learning rate to see what happens.