Open CAridorc opened 1 year ago
Ok, so I did a quick experiment, with ReLU the net does not train, here is the code to implement the residual connections:
"""
@author: Computational Domain
"""
import torch
import torch.nn as nn
import numpy as np
import scipy.io
from matplotlib import pyplot as plt
import matplotlib.animation as animation
nu = 0.01
class ResNet(torch.nn.Module):
def __init__(self, module):
super().__init__()
self.module = module
def forward(self, inputs):
return self.module(inputs) + inputs
class NavierStokes():
def __init__(self, X, Y, T, u, v):
self.x = torch.tensor(X, dtype=torch.float32, requires_grad=True)
self.y = torch.tensor(Y, dtype=torch.float32, requires_grad=True)
self.t = torch.tensor(T, dtype=torch.float32, requires_grad=True)
self.u = torch.tensor(u, dtype=torch.float32)
self.v = torch.tensor(v, dtype=torch.float32)
#null vector to test against f and g:
self.null = torch.zeros((self.x.shape[0], 1))
# initialize network:
self.network()
self.optimizer = torch.optim.LBFGS(self.net.parameters(), lr=1, max_iter=200000, max_eval=50000,
history_size=50, tolerance_grad=1e-05, tolerance_change=0.5 * np.finfo(float).eps,
line_search_fn="strong_wolfe")
self.mse = nn.MSELoss()
#loss
self.ls = 0
#iteration number
self.iter = 0
def network(self):
self.net = nn.Sequential(
nn.Linear(3, 20), nn.Tanh(),
nn.Linear(20, 20), nn.Tanh(),
ResNet(nn.Linear(20, 20)), nn.Tanh(),
ResNet(nn.Linear(20, 20)), nn.Tanh(),
ResNet(nn.Linear(20, 20)), nn.Tanh(),
ResNet(nn.Linear(20, 20)), nn.Tanh(),
ResNet(nn.Linear(20, 20)), nn.Tanh(),
ResNet(nn.Linear(20, 20)), nn.Tanh(),
ResNet(nn.Linear(20, 20)), nn.Tanh(),
nn.Linear(20, 2))
# define the net as before but with residual learning
def function(self, x, y, t):
res = self.net(torch.hstack((x, y, t)))
psi, p = res[:, 0:1], res[:, 1:2]
u = torch.autograd.grad(psi, y, grad_outputs=torch.ones_like(psi), create_graph=True)[0] #retain_graph=True,
v = -1.*torch.autograd.grad(psi, x, grad_outputs=torch.ones_like(psi), create_graph=True)[0]
u_x = torch.autograd.grad(u, x, grad_outputs=torch.ones_like(u), create_graph=True)[0]
u_xx = torch.autograd.grad(u_x, x, grad_outputs=torch.ones_like(u_x), create_graph=True)[0]
u_y = torch.autograd.grad(u, y, grad_outputs=torch.ones_like(u), create_graph=True)[0]
u_yy = torch.autograd.grad(u_y, y, grad_outputs=torch.ones_like(u_y), create_graph=True)[0]
u_t = torch.autograd.grad(u, t, grad_outputs=torch.ones_like(u), create_graph=True)[0]
v_x = torch.autograd.grad(v, x, grad_outputs=torch.ones_like(v), create_graph=True)[0]
v_xx = torch.autograd.grad(v_x, x, grad_outputs=torch.ones_like(v_x), create_graph=True)[0]
v_y = torch.autograd.grad(v, y, grad_outputs=torch.ones_like(v), create_graph=True)[0]
v_yy = torch.autograd.grad(v_y, y, grad_outputs=torch.ones_like(v_y), create_graph=True)[0]
v_t = torch.autograd.grad(v, t, grad_outputs=torch.ones_like(v), create_graph=True)[0]
p_x = torch.autograd.grad(p, x, grad_outputs=torch.ones_like(p), create_graph=True)[0]
p_y = torch.autograd.grad(p, y, grad_outputs=torch.ones_like(p), create_graph=True)[0]
f = u_t + u * u_x + v * u_y + p_x - nu * (u_xx + u_yy)
g = v_t + u * v_x + v * v_y + p_y - nu * (v_xx + v_yy)
return u, v, p, f, g
def closure(self):
# reset gradients to zero:
self.optimizer.zero_grad()
# u, v, p, g and f predictions:
u_prediction, v_prediction, p_prediction, f_prediction, g_prediction = self.function(self.x, self.y, self.t)
# calculate losses
u_loss = self.mse(u_prediction, self.u)
v_loss = self.mse(v_prediction, self.v)
f_loss = self.mse(f_prediction, self.null)
g_loss = self.mse(g_prediction, self.null)
self.ls = u_loss + v_loss + f_loss +g_loss
# derivative with respect to net's weights:
self.ls.backward()
self.iter += 1
if not self.iter % 1:
print('Iteration: {:}, Loss: {:0.6f}'.format(self.iter, self.ls))
return self.ls
def train(self):
# training loop
self.net.train()
self.optimizer.step(self.closure)
N_train = 5000
data = scipy.io.loadmat('cylinder_nektar_wake.mat')
U_star = data['U_star'] # N x 2 x T
P_star = data['p_star'] # N x T
t_star = data['t'] # T x 1
X_star = data['X_star'] # N x 2
N = X_star.shape[0]
T = t_star.shape[0]
x_test = X_star[:, 0:1]
y_test = X_star[:, 1:2]
p_test = P_star[:, 0:1]
u_test = U_star[:, 0:1, 0]
t_test = np.ones((x_test.shape[0], x_test.shape[1]))
# Rearrange Data
XX = np.tile(X_star[:, 0:1], (1, T)) # N x T
YY = np.tile(X_star[:, 1:2], (1, T)) # N x T
TT = np.tile(t_star, (1, N)).T # N x T
UU = U_star[:, 0, :] # N x T
VV = U_star[:, 1, :] # N x T
PP = P_star # N x T
x = XX.flatten()[:, None] # NT x 1
y = YY.flatten()[:, None] # NT x 1
t = TT.flatten()[:, None] # NT x 1
u = UU.flatten()[:, None] # NT x 1
v = VV.flatten()[:, None] # NT x 1
p = PP.flatten()[:, None] # NT x 1
# Training Data
idx = np.random.choice(N * T, N_train, replace=False)
x_train = x[idx, :]
y_train = y[idx, :]
t_train = t[idx, :]
u_train = u[idx, :]
v_train = v[idx, :]
pinn = NavierStokes(x_train, y_train, t_train, u_train, v_train)
pinn.train()
torch.save(pinn.net.state_dict(), 'model.pt')
pinn = NavierStokes(x_train, y_train, t_train, u_train, v_train)
pinn.net.load_state_dict(torch.load('model.pt'))
pinn.net.eval()
x_test = torch.tensor(x_test, dtype=torch.float32, requires_grad=True)
y_test = torch.tensor(y_test, dtype=torch.float32, requires_grad=True)
t_test = torch.tensor(t_test, dtype=torch.float32, requires_grad=True)
u_out, v_out, p_out, f_out, g_out = pinn.function(x_test, y_test, t_test)
u_plot = p_out.data.cpu().numpy()
u_plot = np.reshape(u_plot, (50, 100))
fig, ax = plt.subplots()
plt.contourf(u_plot, levels=30, cmap='jet')
plt.colorbar()
#plt.show()
def animate(i):
ax.clear()
u_out, v_out, p_out, f_out, g_out = pinn.function(x_test, y_test, i*t_test)
u_plot = p_out.data.cpu().numpy()
u_plot = np.reshape(u_plot, (50, 100))
cax = ax.contourf(u_plot, levels=20, cmap='jet')
plt.xlabel(r'$x$')
plt.xlabel(r'$y$')
plt.title(r'$p(x,\; y, \; t)$')
# Call animate method
ani = animation.FuncAnimation(fig, animate, 20, interval=1, blit=False)
#ani.save('p_field_lbfgs.gif')
#plt.close()
# Display the plot
plt.show()
I cannot run too many experiments with my weak computer, but training for a few iterations seems promising, so far I got Iteration: 1000, Loss: 0.023212
Iteration: 1500, Loss: 0.016457
after some time Iteration: 11000, Loss: 0.001247
and Iteration 13500 Loss: 0.000957
and it looks like the loss is still going down, try running this on your supercomputer and tell me what you get!
Ok here is a better version, using residuals correctly and avoiding the initial tanh as it obfuscates the inputs:
"""
@author: Computational Domain
"""
import torch
import torch.nn as nn
import numpy as np
import scipy.io
from matplotlib import pyplot as plt
import matplotlib.animation as animation
nu = 0.01
class ResNet(torch.nn.Module):
def __init__(self, module):
super().__init__()
self.module = module
self.tanh = nn.Tanh()
def forward(self, inputs):
return self.tanh(self.module(inputs)) + inputs
class NavierStokes():
def __init__(self, X, Y, T, u, v):
self.x = torch.tensor(X, dtype=torch.float32, requires_grad=True)
self.y = torch.tensor(Y, dtype=torch.float32, requires_grad=True)
self.t = torch.tensor(T, dtype=torch.float32, requires_grad=True)
self.u = torch.tensor(u, dtype=torch.float32)
self.v = torch.tensor(v, dtype=torch.float32)
#null vector to test against f and g:
self.null = torch.zeros((self.x.shape[0], 1))
# initialize network:
self.network()
self.optimizer = torch.optim.LBFGS(self.net.parameters(), lr=1, max_iter=200000, max_eval=50000,
history_size=50, tolerance_grad=1e-05, tolerance_change=0.5 * np.finfo(float).eps,
line_search_fn="strong_wolfe")
self.mse = nn.MSELoss()
#loss
self.ls = 0
#iteration number
self.iter = 0
def network(self):
self.net = nn.Sequential(
nn.Linear(3, 20),
ResNet(nn.Linear(20, 20)),
ResNet(nn.Linear(20, 20)),
ResNet(nn.Linear(20, 20)),
ResNet(nn.Linear(20, 20)),
ResNet(nn.Linear(20, 20)),
ResNet(nn.Linear(20, 20)),
ResNet(nn.Linear(20, 20)),
ResNet(nn.Linear(20, 20)),
nn.Linear(20, 2))
# define the net as before but with residual learning
def function(self, x, y, t):
res = self.net(torch.hstack((x, y, t)))
psi, p = res[:, 0:1], res[:, 1:2]
u = torch.autograd.grad(psi, y, grad_outputs=torch.ones_like(psi), create_graph=True)[0] #retain_graph=True,
v = -1.*torch.autograd.grad(psi, x, grad_outputs=torch.ones_like(psi), create_graph=True)[0]
u_x = torch.autograd.grad(u, x, grad_outputs=torch.ones_like(u), create_graph=True)[0]
u_xx = torch.autograd.grad(u_x, x, grad_outputs=torch.ones_like(u_x), create_graph=True)[0]
u_y = torch.autograd.grad(u, y, grad_outputs=torch.ones_like(u), create_graph=True)[0]
u_yy = torch.autograd.grad(u_y, y, grad_outputs=torch.ones_like(u_y), create_graph=True)[0]
u_t = torch.autograd.grad(u, t, grad_outputs=torch.ones_like(u), create_graph=True)[0]
v_x = torch.autograd.grad(v, x, grad_outputs=torch.ones_like(v), create_graph=True)[0]
v_xx = torch.autograd.grad(v_x, x, grad_outputs=torch.ones_like(v_x), create_graph=True)[0]
v_y = torch.autograd.grad(v, y, grad_outputs=torch.ones_like(v), create_graph=True)[0]
v_yy = torch.autograd.grad(v_y, y, grad_outputs=torch.ones_like(v_y), create_graph=True)[0]
v_t = torch.autograd.grad(v, t, grad_outputs=torch.ones_like(v), create_graph=True)[0]
p_x = torch.autograd.grad(p, x, grad_outputs=torch.ones_like(p), create_graph=True)[0]
p_y = torch.autograd.grad(p, y, grad_outputs=torch.ones_like(p), create_graph=True)[0]
f = u_t + u * u_x + v * u_y + p_x - nu * (u_xx + u_yy)
g = v_t + u * v_x + v * v_y + p_y - nu * (v_xx + v_yy)
return u, v, p, f, g
def closure(self):
# reset gradients to zero:
self.optimizer.zero_grad()
# u, v, p, g and f predictions:
u_prediction, v_prediction, p_prediction, f_prediction, g_prediction = self.function(self.x, self.y, self.t)
# calculate losses
u_loss = self.mse(u_prediction, self.u)
v_loss = self.mse(v_prediction, self.v)
f_loss = self.mse(f_prediction, self.null)
g_loss = self.mse(g_prediction, self.null)
self.ls = u_loss + v_loss + f_loss +g_loss
# derivative with respect to net's weights:
self.ls.backward()
self.iter += 1
if not self.iter % 1:
print('Iteration: {:}, Loss: {:0.6f}'.format(self.iter, self.ls))
return self.ls
def train(self):
# training loop
self.net.train()
self.optimizer.step(self.closure)
N_train = 5000
data = scipy.io.loadmat('cylinder_nektar_wake.mat')
U_star = data['U_star'] # N x 2 x T
P_star = data['p_star'] # N x T
t_star = data['t'] # T x 1
X_star = data['X_star'] # N x 2
N = X_star.shape[0]
T = t_star.shape[0]
x_test = X_star[:, 0:1]
y_test = X_star[:, 1:2]
p_test = P_star[:, 0:1]
u_test = U_star[:, 0:1, 0]
t_test = np.ones((x_test.shape[0], x_test.shape[1]))
# Rearrange Data
XX = np.tile(X_star[:, 0:1], (1, T)) # N x T
YY = np.tile(X_star[:, 1:2], (1, T)) # N x T
TT = np.tile(t_star, (1, N)).T # N x T
UU = U_star[:, 0, :] # N x T
VV = U_star[:, 1, :] # N x T
PP = P_star # N x T
x = XX.flatten()[:, None] # NT x 1
y = YY.flatten()[:, None] # NT x 1
t = TT.flatten()[:, None] # NT x 1
u = UU.flatten()[:, None] # NT x 1
v = VV.flatten()[:, None] # NT x 1
p = PP.flatten()[:, None] # NT x 1
# Training Data
idx = np.random.choice(N * T, N_train, replace=False)
x_train = x[idx, :]
y_train = y[idx, :]
t_train = t[idx, :]
u_train = u[idx, :]
v_train = v[idx, :]
pinn = NavierStokes(x_train, y_train, t_train, u_train, v_train)
pinn.train()
torch.save(pinn.net.state_dict(), 'model.pt')
pinn = NavierStokes(x_train, y_train, t_train, u_train, v_train)
pinn.net.load_state_dict(torch.load('model.pt'))
pinn.net.eval()
x_test = torch.tensor(x_test, dtype=torch.float32, requires_grad=True)
y_test = torch.tensor(y_test, dtype=torch.float32, requires_grad=True)
t_test = torch.tensor(t_test, dtype=torch.float32, requires_grad=True)
u_out, v_out, p_out, f_out, g_out = pinn.function(x_test, y_test, t_test)
u_plot = p_out.data.cpu().numpy()
u_plot = np.reshape(u_plot, (50, 100))
fig, ax = plt.subplots()
plt.contourf(u_plot, levels=30, cmap='jet')
plt.colorbar()
#plt.show()
def animate(i):
ax.clear()
u_out, v_out, p_out, f_out, g_out = pinn.function(x_test, y_test, i*t_test)
u_plot = p_out.data.cpu().numpy()
u_plot = np.reshape(u_plot, (50, 100))
cax = ax.contourf(u_plot, levels=20, cmap='jet')
plt.xlabel(r'$x$')
plt.xlabel(r'$y$')
plt.title(r'$p(x,\; y, \; t)$')
# Call animate method
ani = animation.FuncAnimation(fig, animate, 20, interval=1, blit=False)
#ani.save('p_field_lbfgs.gif')
#plt.close()
# Display the plot
plt.show()
With this version the loss goes down wonderfully fast with a loss of Iteration: 1000, Loss: 0.003410
Iteration: 1500, Loss: 0.002177
Iteration: 3500, Loss: 0.000796
and still going down.
By the way, I used this https://github.com/maziarraissi/PINNs/blob/master/main/Data/cylinder_nektar_wake.mat data file as there is no data file in your repository,
Have you tried using ReLU activations and residual connections in your network? They might make it easier to train, reduce the time needed to optimize it and maybe even improve the final results. 8 layers is a lot to be using a tanh function with no residuals. I am really curious about this possibility.