Open HosseinRZ95 opened 2 years ago
are you using tf or torch? can you attach a minimum repro script as a file? the repro script has a weird format, and I couldn't see the problem when I tried a simple example..
bugtest.zip I'am using torch. I don't know why my repro code look like this here. but I attached the file. I got same error with my original env and dummy env. it works fine when I give True to use_critic btw.
What happened + What you expected to happen
I tried a ppo agent with these configs: model_config['batch_mode'] = 'complete_episodes' , model_config['use_gae'] = 'False',model_config['use_critic'] = 'False' then I got this error:" /usr/local/lib/python3.7/dist-packages/ray/rllib/policy/torch_policy.py in(s)
701 data.append(
702 tree.map_structure(
--> 703 lambda s: s.to(self.device), tower.tower_stats[stats_name]
704 )
705 )
AttributeError: 'float' object has no attribute 'to'" so to be sure I wrote a dummy env and tested it again and I got same error. why I'am doing this config? well, I've manipulated postprocess so I can't go truncated episode and thus I can't use gae.
Versions / Dependencies
ray 1.12.1 google colab python 3.7.13
Reproduction script
import gym from gym import spaces import numpy as np import torch from torch import nn from ray.rllib.models.torch.torch_modelv2 import TorchModelV2 from ray.rllib.agents.ppo import ppo import ray from ray.rllib.models import ModelCatalog class dummyvisoinenv(gym.Env): def init(self,config): self.length = length self.width = width self.channels = channels self.action_space = spaces.Tuple([spaces.Discrete(4),spaces.Discrete(4),spaces.Discrete(3)]) self.observation_space = spaces.Box(low=-1 * 10 10, high=10 10, shape=(self.length,self.width,self.channels), dtype=np.float32) self.episode_len = episode_len
def reset(self): self.timestamp = 0 self.done = False observation = np.random.rand(self.length,self.width,self.channels) return observation def step(self,action): a1, a2, a3 = action self.timestamp = self.timestamp + 1 if self.timestamp == self.episode_len: self.done = True reward = np.random.randint(0,5) observation = np.random.rand(self.length,self.width,self.channels) info = {} print(self.timestamp) return observation, reward, self.done, info
resnet model based on pytorch official
def conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1): """3x3 convolution with padding""" return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, padding=dilation, groups=groups, bias=False, dilation=dilation)
def conv1x1(in_planes, out_planes, stride=1): """1x1 convolution""" return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False)
import torch
from torch import nn
from ray.rllib.models.torch.torch_modelv2 import TorchModelV2
class BasicBlock(nn.Module): expansion = 1
class Bottleneck(nn.Module):
Bottleneck in torchvision places the stride for downsampling at 3x3 convolution(self.conv2)
class ResNet(TorchModelV2,nn.Module):
configs
length = 20 width = 20 channels = 3 episode_len = 10 env_config = {"length" : length , 'width': width , 'channels': channels,'episode_len' : episode_len} env_instance = dummyvisoinenv(env_config) model = ppo model_config = model.DEFAULT_CONFIG.copy() model_config["env"] = env_instance model_config['num_workers'] = 0 model_config['train_batch_size'] = 5 model_config["framework"]="torch" model_config['sgd_minibatch_size'] = 5 model_config["model"] = { "custom_model": 'resnet18torch', 'custom_model_config': { 'block':BasicBlock , 'layers':[2, 2, 2, 2], 'zero_init_residual': False , 'groups' : 1, 'width_per_group' : 64 , 'replace_stride_with_dilation' : None , 'norm_layer' : None } } model_config['use_gae'] = False model_config["use_critic"] = False model_config['batch_mode'] = 'complete_episodes' # ray.shutdown() ray.init(ignore_reinit_error=True) ModelCatalog.register_custom_model('resnet18torch', ResNet) total_episodes = 100 trainer = model.PPOTrainer(env= dummyvisoinenv, config=model_config) for i in range(total_episodes): result = trainer.train() print('done: ',i)
Issue Severity
Medium: It is a significant difficulty but I can work around it.