dkasuga
commented
4 years ago As a result of some trials and errors. I found that the number of meshes, pathtracing samples, or batch size have large influence on whether rendering stops or not. In my cases, I tried to render 8(batch size) scenes each of which has 7920 vertices at 256 rendering samples many iterations. This placed high load on CPUs.
So now I've just tried changing parameters such as batch size or rendering samples, but again the rendering stops at the middle of training, although there was some improvement (bedore: at 1-2 iteration, after:15-16 iteration). Do I have to do some kind of memory release processing?
My codes:
'''
It is a simple neural network just like Auto Encoder, which inputs an image of sphere with material and outputs material parameters. In this code, I focus only on diffuse_reflectance parameters.
'''
import torch
import pyredner
import numpy as np
from torch import nn
import torch.nn.functional as F
import torch.optim as optim
from torch.utils.data import Dataset, DataLoader
import torchvision.utils as vutils
from torchvision import datasets, models, transforms
from PIL import Image
import sys
from tqdm import tqdm
import argparse
import os.path
import random
import time
from torch.utils.tensorboard import SummaryWriter
from my_models import MyResnet, Model
def make_obj_and_cam(d_reflectance):
vertices, indices, uvs, normals = pyredner.generate_sphere(theta_steps=64,
phi_steps=128)
diffuse_reflectance = d_reflectance
m = pyredner.Material(diffuse_reflectance=d_reflectance,
specular_reflectance=torch.tensor(
(0.2, 0.2, 0.2), device=pyredner.get_device()),
roughness=torch.tensor([0.001],
device=pyredner.get_device()))
obj = pyredner.Object(vertices=vertices,
indices=indices,
uvs=uvs,
normals=normals,
material=m)
cam = pyredner.automatic_camera_placement([obj],
resolution=args.resolution)
return obj, cam
def train(args, envmap, model, optimizer):
for iteration in tqdm(range(args.total_iter)):
input_scenes = []
for batch in range(args.batch_size):
diffuse_reflectance = torch.rand(3, device=pyredner.get_device())
obj, cam = make_obj_and_cam(diffuse_reflectance)
scene = pyredner.Scene(objects=[obj], camera=cam, envmap=envmap)
input_scenes.append(scene)
input_imgs = pyredner.render_pathtracing(scene=input_scenes,
num_samples=args.samples)
input_imgs = torch.pow(input_imgs, 1.0 / 2.2) # gamma correction
input_imgs = input_imgs.permute(0, 3, 1, 2) #NxCxHxW
# Estimation
output_params = model(input_imgs)
output_scenes = []
for batch in range(args.batch_size):
diffuse_reflectance = output_params[batch]
obj, cam = make_obj_and_cam(diffuse_reflectance)
scene = pyredner.Scene(objects=[obj], camera=cam, envmap=envmap)
output_scenes.append(scene)
try:
output_imgs = pyredner.render_pathtracing(scene=output_scenes,
num_samples=args.samples)
except:
print("error")
print("output_params:{}".format(output_params))
output_imgs = torch.pow(output_imgs, 1.0 / 2.2) # gamma correction
output_imgs = output_imgs.permute(0, 3, 1, 2) #NxCxHxW
#loss = nn.MSELoss()(output_imgs, input_imgs)
loss = (output_imgs - input_imgs).pow(2).sum()
optimizer.zero_grad()
loss.backward()
optimizer.step()
if iteration % 10 == 0:
saved_imgs = torch.cat(
[input_imgs.detach().cpu(),
output_imgs.detach().cpu()], dim=2)
vutils.save_image(saved_imgs,
os.path.join(args.output_dir,
f'logs/saved_imgs_{iteration}.jpg'),
nrow=(args.batch_size))
writer.add_images('saved_imgs', saved_imgs, iteration)
print(f"loss:{loss}")
writer.add_scalar("loss", loss, iteration)
if __name__ == '__main__':
from datetime import datetime
date_time = datetime.now()
log_folder = 'trial_%s_%d_%d_%d' % (date_time.date(), date_time.hour,
date_time.minute, date_time.second)
os.mkdir(log_folder)
os.mkdir(log_folder + '/logs')
parser = argparse.ArgumentParser(
description='diffuse_reflectance learning')
parser.add_argument('--trial_name',
type=str,
default="diffuse_learning_test1",
help='a brief description of the training trial')
parser.add_argument('--output_dir',
default=log_folder,
help='output directory')
parser.add_argument('--gpu_id',
type=int,
default=0,
help='0 is the first gpu, 1 is the second gpu, etc.')
parser.add_argument('--resolution',
type=int,
default=224,
help='image resolution')
parser.add_argument('--samples',
type=int,
default=64,
help='rendering samples')
parser.add_argument(
'--lr',
type=float,
default=0.001,
help=
'learning rate, default is 1e-3, usually dont need to change it, you can try make it bigger, such as 2e-3'
)
parser.add_argument(
'--batch_size',
type=int,
default=8,
help='how many images to train together at one iteration')
parser.add_argument(
'--total_iter',
type=int,
default=200,
help=
'how many iterations to train in total, the value is in assumption that init step is 1'
)
args = parser.parse_args()
print(str(args))
args.resolution = (args.resolution, args.resolution)
trial_name = args.trial_name
device = torch.device("cuda:%d" % (args.gpu_id))
pyredner.set_use_gpu(torch.cuda.is_available())
pyredner.set_device(device)
writer = SummaryWriter(log_dir=f"{args.output_dir}/tensorboard")
envmap_img = pyredner.imread('./grace-new.exr')
envmap_img.to(device)
envmap = pyredner.EnvironmentMap(envmap_img * 10.0)
model = MyResnet(dim_out=3)
model = model.to(device)
model.train()
optimizer = torch.optim.Adam(model.parameters(),
lr=args.lr,
betas=(0.0, 0.99))
train(args, envmap, model, optimizer)
BachiLi
I "pip install"ed redner-gpu to my python environment. There's basically no problem, but render_pathtracing operation often stops in the middle, especially when I'm using the function iteratively (using pyredner as neural network module). This problem didn't appear when I used pyredner on the provided docker-image.
Does anybody have the same problem? My system is 18.04.3 LTS (Bionic Beaver) 4.15.0-60-generic, NVIDIA Driver Version: 410.129.