search

rlworkgroup / garage

A toolkit for reproducible reinforcement learning research.
MIT License
1.86k stars 310 forks source link

AttributeError: 'NoneType' object has no attribute 'close' #2313

Closed sadegh16 closed 2 years ago

sadegh16 commented 2 years ago

Hi,

I have a problem running the experiment trpo_pendulum_ray_sampler

` 

  #!/usr/bin/env python3
   """This is an example to train a task with TRPO algorithm (PyTorch).
  Uses Ray sampler instead of MultiprocessingSampler.
  Here it runs InvertedDoublePendulum-v2 environment with 100 iterations.
  """
  import numpy as np
  import ray
  import torch

  from garage import wrap_experiment
  from garage.envs import GymEnv
  from garage.experiment import deterministic
  from garage.sampler import RaySampler
  from garage.torch.algos import TRPO
  from garage.torch.policies import GaussianMLPPolicy
  from garage.torch.value_functions import GaussianMLPValueFunction
  from garage.trainer import Trainer

  @wrap_experiment(snapshot_mode='none')
  def trpo_pendulum_ray_sampler(ctxt=None, seed=1):
      """Set up environment and algorithm and run the task.

      Args:
          ctxt (garage.experiment.ExperimentContext): The experiment
              configuration used by Trainer to create the snapshotter.
          seed (int): Used to seed the random number generator to produce
              determinism.

      """
      # Since this is an example, we are running ray in a reduced state.
      # One can comment this line out in order to run ray at full capacity
      ray.init(
               object_store_memory=78643200,
               ignore_reinit_error=True,
               log_to_driver=False,
      )
      deterministic.set_seed(seed)
      env = GymEnv('InvertedDoublePendulum-v2')

      trainer = Trainer(ctxt)

      policy = GaussianMLPPolicy(env.spec,
                                 hidden_sizes=[32, 32],
                                 hidden_nonlinearity=torch.tanh,
                                 output_nonlinearity=None)

      value_function = GaussianMLPValueFunction(env_spec=env.spec,
                                                hidden_sizes=(32, 32),
                                                hidden_nonlinearity=torch.tanh,
                                                output_nonlinearity=None)

      sampler = RaySampler(agents=policy,
                           envs=env,
                           max_episode_length=env.spec.max_episode_length)

      algo = TRPO(env_spec=env.spec,
                  policy=policy,
                  value_function=value_function,
                  sampler=sampler,
                  discount=0.99,
                  center_adv=False)

      trainer.setup(algo, env)
      trainer.train(n_epochs=100, batch_size=1024)

  s = np.random.randint(0, 1000)
  trpo_pendulum_ray_sampler(seed=s)

` after running this experiment I get the error below:

` Traceback (most recent call last): File "/root/varReductionRl/rl_env/lib/python3.8/site-packages/gym/envs/mujoco/mujoco_env.py", line 12, in import mujoco_py ModuleNotFoundError: No module named 'mujoco_py'

    During handling of the above exception, another exception occurred:

    Traceback (most recent call last):
      File "first_exp.py", line 70, in <module>
        trpo_pendulum_ray_sampler(seed=s)
      File "/root/varReductionRl/rl_env/lib/python3.8/site-packages/garage/experiment/experiment.py", line 369, in __call__
        result = self.function(ctxt, **kwargs)
      File "first_exp.py", line 40, in trpo_pendulum_ray_sampler
        env = GymEnv('InvertedDoublePendulum-v2')
      File "/root/varReductionRl/rl_env/lib/python3.8/site-packages/garage/envs/gym_env.py", line 136, in __init__
        self._env = gym.make(env)
      File "/root/varReductionRl/rl_env/lib/python3.8/site-packages/gym/envs/registration.py", line 235, in make
        return registry.make(id, **kwargs)
      File "/root/varReductionRl/rl_env/lib/python3.8/site-packages/gym/envs/registration.py", line 129, in make
        env = spec.make(**kwargs)
      File "/root/varReductionRl/rl_env/lib/python3.8/site-packages/gym/envs/registration.py", line 89, in make
        cls = load(self.entry_point)
      File "/root/varReductionRl/rl_env/lib/python3.8/site-packages/gym/envs/registration.py", line 27, in load
        mod = importlib.import_module(mod_name)
      File "/usr/lib/python3.8/importlib/__init__.py", line 127, in import_module
        return _bootstrap._gcd_import(name[level:], package, level)
      File "<frozen importlib._bootstrap>", line 1014, in _gcd_import
      File "<frozen importlib._bootstrap>", line 991, in _find_and_load
      File "<frozen importlib._bootstrap>", line 975, in _find_and_load_unlocked
      File "<frozen importlib._bootstrap>", line 671, in _load_unlocked
      File "<frozen importlib._bootstrap_external>", line 848, in exec_module
      File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
      File "/root/varReductionRl/rl_env/lib/python3.8/site-packages/gym/envs/mujoco/__init__.py", line 1, in <module>
        from gym.envs.mujoco.mujoco_env import MujocoEnv
      File "/root/varReductionRl/rl_env/lib/python3.8/site-packages/gym/envs/mujoco/mujoco_env.py", line 14, in <module>
        raise error.DependencyNotInstalled(
    gym.error.DependencyNotInstalled: No module named 'mujoco_py'. (HINT: you need to install mujoco_py, and also perform the setup instructions here: https://github.com/openai/mujoco-py/.)
    Exception ignored in: <function Environment.__del__ at 0x7fbcf600c820>
    Traceback (most recent call last):
      File "/root/varReductionRl/rl_env/lib/python3.8/site-packages/garage/_environment.py", line 355, in __del__
        self.close()
      File "/root/varReductionRl/rl_env/lib/python3.8/site-packages/garage/envs/gym_env.py", line 294, in close
        self._env.close()
    AttributeError: 'NoneType' object has no attribute 'close'

`

I used garage the same as installation instructions and I don't know why it misses mujoco_py package. Anyway of I install the package the problem still stands ( I mean I will get AttributeError: 'NoneType' object has no attribute 'close' error still )

Anyone has any idea?

sadegh16 commented 2 years ago

@krzentner Can you help me here?

krzentner commented 2 years ago

Most likely it failed because you need to install Mujoco, and then mujoco_py. Did you install Mujoco and change LD_PRELOAD like it requires? What is the output of pip install mujoco_py>2?

sadegh16 commented 2 years ago

Most likely it failed because you need to install Mujoco, and then mujoco_py. Did you install Mujoco and change LD_PRELOAD like it requires? What is the output of pip install mujoco_py>2?

Hi @krzentner . Thanks for the response. the mujoco_py will be installed successfully but the problem stands. what is LD_PRELOAD ? When I install it as this link says

`

    Collecting mujoco_py
      Using cached mujoco_py-2.1.2.14-py3-none-any.whl (2.4 MB)
    Collecting Cython>=0.27.2
      Downloading Cython-0.29.25-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl (1.9 MB)
         |################################| 1.9 MB 3.1 MB/s 
    Collecting glfw>=1.4.0
      Using cached glfw-2.4.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38-none-manylinux2014_x86_64.whl (205 kB)
    Collecting cffi>=1.10
      Downloading cffi-1.15.0-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (427 kB)
         |################################| 427 kB 4.6 MB/s 
    Requirement already satisfied: numpy>=1.11 in /root/anaconda3/envs/myenv/lib/python3.7/site-packages (from mujoco_py) (1.21.4)
    Collecting fasteners~=0.15
      Using cached fasteners-0.16.3-py2.py3-none-any.whl (28 kB)
    Requirement already satisfied: imageio>=2.1.2 in /root/anaconda3/envs/myenv/lib/python3.7/site-packages (from mujoco_py) (2.13.3)
    Collecting pycparser
      Using cached pycparser-2.21-py2.py3-none-any.whl (118 kB)
    Requirement already satisfied: six in /root/anaconda3/envs/myenv/lib/python3.7/site-packages (from fasteners~=0.15->mujoco_py) (1.16.0)
    Requirement already satisfied: pillow>=8.3.2 in /root/anaconda3/envs/myenv/lib/python3.7/site-packages (from imageio>=2.1.2->mujoco_py) (8.4.0)
    Installing collected packages: pycparser, glfw, fasteners, Cython, cffi, mujoco-py
    Successfully installed Cython-0.29.25 cffi-1.15.0 fasteners-0.16.3 glfw-2.4.0 mujoco-py-2.1.2.14 pycparser-2.21

    `

if I run again I get below error: `

python -m garage.examples.torch.ddpg_pendulum

              2021-12-10 22:21:33.026518: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory
  2021-12-10 22:21:33.026550: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
  2021-12-10 22:21:35 | [ddpg_pendulum] Logging to /root/varReductionRL/data/local/experiment/ddpg_pendulum_5
  /root/varReductionRL/rl_env/lib/python3.6/site-packages/garage/experiment/deterministic.py:37: UserWarning: Enabeling deterministic mode in PyTorch can have a performance impact when using GPU.
    'Enabeling deterministic mode in PyTorch can have a performance '
  Traceback (most recent call last):
    File "/usr/lib/python3.6/runpy.py", line 193, in _run_module_as_main
      "__main__", mod_spec)
    File "/usr/lib/python3.6/runpy.py", line 85, in _run_code
      exec(code, run_globals)
    File "/root/varReductionRL/rl_env/lib/python3.6/site-packages/garage/examples/torch/ddpg_pendulum.py", line 78, in <module>
      ddpg_pendulum()
    File "/root/varReductionRL/rl_env/lib/python3.6/site-packages/garage/experiment/experiment.py", line 369, in __call__
      result = self.function(ctxt, **kwargs)
    File "/root/varReductionRL/rl_env/lib/python3.6/site-packages/garage/examples/torch/ddpg_pendulum.py", line 37, in ddpg_pendulum
      env = normalize(GymEnv('InvertedDoublePendulum-v2'))
    File "/root/varReductionRL/rl_env/lib/python3.6/site-packages/garage/envs/gym_env.py", line 136, in __init__
      self._env = gym.make(env)
    File "/root/varReductionRL/rl_env/lib/python3.6/site-packages/gym/envs/registration.py", line 235, in make
      return registry.make(id, **kwargs)
    File "/root/varReductionRL/rl_env/lib/python3.6/site-packages/gym/envs/registration.py", line 129, in make
      env = spec.make(**kwargs)
    File "/root/varReductionRL/rl_env/lib/python3.6/site-packages/gym/envs/registration.py", line 89, in make
      cls = load(self.entry_point)
    File "/root/varReductionRL/rl_env/lib/python3.6/site-packages/gym/envs/registration.py", line 27, in load
      mod = importlib.import_module(mod_name)
    File "/root/varReductionRL/rl_env/lib/python3.6/importlib/__init__.py", line 126, in import_module
      return _bootstrap._gcd_import(name[level:], package, level)
    File "<frozen importlib._bootstrap>", line 994, in _gcd_import
    File "<frozen importlib._bootstrap>", line 971, in _find_and_load
    File "<frozen importlib._bootstrap>", line 955, in _find_and_load_unlocked
    File "<frozen importlib._bootstrap>", line 665, in _load_unlocked
    File "<frozen importlib._bootstrap_external>", line 678, in exec_module
    File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
    File "/root/varReductionRL/rl_env/lib/python3.6/site-packages/gym/envs/mujoco/__init__.py", line 1, in <module>
      from gym.envs.mujoco.mujoco_env import MujocoEnv
    File "/root/varReductionRL/rl_env/lib/python3.6/site-packages/gym/envs/mujoco/mujoco_env.py", line 12, in <module>
      import mujoco_py
    File "/root/varReductionRL/rl_env/lib/python3.6/site-packages/mujoco_py/__init__.py", line 2, in <module>
      from mujoco_py.builder import cymj, ignore_mujoco_warnings, functions, MujocoException
    File "/root/varReductionRL/rl_env/lib/python3.6/site-packages/mujoco_py/builder.py", line 504, in <module>
      cymj = load_cython_ext(mujoco_path)
    File "/root/varReductionRL/rl_env/lib/python3.6/site-packages/mujoco_py/builder.py", line 74, in load_cython_ext
      _ensure_set_env_var("LD_LIBRARY_PATH", lib_path)
    File "/root/varReductionRL/rl_env/lib/python3.6/site-packages/mujoco_py/builder.py", line 124, in _ensure_set_env_var
      var_name, var_name, lib_path))
  Exception: 
  Missing path to your environment variable. 
  Current values LD_LIBRARY_PATH=
  Please add following line to .bashrc:
  export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/root/.mujoco/mujoco210/bin
  Exception ignored in: <bound method Environment.__del__ of <garage.envs.gym_env.GymEnv object at 0x7f789cee9828>>
  Traceback (most recent call last):
    File "/root/varReductionRL/rl_env/lib/python3.6/site-packages/garage/_environment.py", line 355, in __del__
      self.close()
    File "/root/varReductionRL/rl_env/lib/python3.6/site-packages/garage/envs/gym_env.py", line 294, in close
      self._env.close()
  AttributeError: 'NoneType' object has no attribute 'close'

`

To solve the above error I do this: ` export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/root/.mujoco/mujoco210/bin

`

but there comes another error:

`

  2021-12-10 22:46:03.230815: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /root/.mujoco/mujoco210/bin
  2021-12-10 22:46:03.230850: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
  2021-12-10 22:46:05 | [ddpg_pendulum] Logging to /root/varReductionRL/data/local/experiment/ddpg_pendulum_14
  /root/anaconda3/envs/myenv/lib/python3.7/site-packages/garage/experiment/deterministic.py:37: UserWarning: Enabeling deterministic mode in PyTorch can have a performance impact when using GPU.
    'Enabeling deterministic mode in PyTorch can have a performance '
  Traceback (most recent call last):
    File "/root/anaconda3/envs/myenv/lib/python3.7/runpy.py", line 193, in _run_module_as_main
      "__main__", mod_spec)
    File "/root/anaconda3/envs/myenv/lib/python3.7/runpy.py", line 85, in _run_code
      exec(code, run_globals)
    File "/root/anaconda3/envs/myenv/lib/python3.7/site-packages/garage/examples/torch/ddpg_pendulum.py", line 78, in <module>
      ddpg_pendulum()
    File "/root/anaconda3/envs/myenv/lib/python3.7/site-packages/garage/experiment/experiment.py", line 369, in __call__
      result = self.function(ctxt, **kwargs)
    File "/root/anaconda3/envs/myenv/lib/python3.7/site-packages/garage/examples/torch/ddpg_pendulum.py", line 37, in ddpg_pendulum
      env = normalize(GymEnv('InvertedDoublePendulum-v2'))
    File "/root/anaconda3/envs/myenv/lib/python3.7/site-packages/garage/envs/gym_env.py", line 136, in __init__
      self._env = gym.make(env)
    File "/root/anaconda3/envs/myenv/lib/python3.7/site-packages/gym/envs/registration.py", line 235, in make
      return registry.make(id, **kwargs)
    File "/root/anaconda3/envs/myenv/lib/python3.7/site-packages/gym/envs/registration.py", line 129, in make
      env = spec.make(**kwargs)
    File "/root/anaconda3/envs/myenv/lib/python3.7/site-packages/gym/envs/registration.py", line 89, in make
      cls = load(self.entry_point)
    File "/root/anaconda3/envs/myenv/lib/python3.7/site-packages/gym/envs/registration.py", line 27, in load
      mod = importlib.import_module(mod_name)
    File "/root/anaconda3/envs/myenv/lib/python3.7/importlib/__init__.py", line 127, in import_module
      return _bootstrap._gcd_import(name[level:], package, level)
    File "<frozen importlib._bootstrap>", line 1006, in _gcd_import
    File "<frozen importlib._bootstrap>", line 983, in _find_and_load
    File "<frozen importlib._bootstrap>", line 967, in _find_and_load_unlocked
    File "<frozen importlib._bootstrap>", line 677, in _load_unlocked
    File "<frozen importlib._bootstrap_external>", line 728, in exec_module
    File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
    File "/root/anaconda3/envs/myenv/lib/python3.7/site-packages/gym/envs/mujoco/__init__.py", line 1, in <module>
      from gym.envs.mujoco.mujoco_env import MujocoEnv
    File "/root/anaconda3/envs/myenv/lib/python3.7/site-packages/gym/envs/mujoco/mujoco_env.py", line 12, in <module>
      import mujoco_py
    File "/root/anaconda3/envs/myenv/lib/python3.7/site-packages/mujoco_py/__init__.py", line 2, in <module>
      from mujoco_py.builder import cymj, ignore_mujoco_warnings, functions, MujocoException
    File "/root/anaconda3/envs/myenv/lib/python3.7/site-packages/mujoco_py/builder.py", line 504, in <module>
      cymj = load_cython_ext(mujoco_path)
    File "/root/anaconda3/envs/myenv/lib/python3.7/site-packages/mujoco_py/builder.py", line 76, in load_cython_ext
      _ensure_set_env_var("LD_LIBRARY_PATH", get_nvidia_lib_dir())
    File "/root/anaconda3/envs/myenv/lib/python3.7/site-packages/mujoco_py/builder.py", line 124, in _ensure_set_env_var
      var_name, var_name, lib_path))
  Exception: 
  Missing path to your environment variable. 
  Current values LD_LIBRARY_PATH=/root/.mujoco/mujoco210/bin
  Please add following line to .bashrc:
  export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib/nvidia
  Exception ignored in: <function Environment.__del__ at 0x7f7f1b782950>
  Traceback (most recent call last):
    File "/root/anaconda3/envs/myenv/lib/python3.7/site-packages/garage/_environment.py", line 355, in __del__
      self.close()
    File "/root/anaconda3/envs/myenv/lib/python3.7/site-packages/garage/envs/gym_env.py", line 294, in close
      self._env.close()
  AttributeError: 'NoneType' object has no attribute 'close'

`

Again if I use the command export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib/nvidia as above says, I get lots of infinity shits printing out to the terminal if I run python -m garage.examples.torch.ddpg_pendulum:

`

    2021-12-10 22:48:57.371739: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /root/.mujoco/mujoco210/bin:/usr/lib/nvidia
    2021-12-10 22:48:57.371768: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
    2021-12-10 22:48:59 | [ddpg_pendulum] Logging to /root/varReductionRL/data/local/experiment/ddpg_pendulum_15
    /root/anaconda3/envs/myenv/lib/python3.7/site-packages/garage/experiment/deterministic.py:37: UserWarning: Enabeling deterministic mode in PyTorch can have a performance impact when using GPU.
      'Enabeling deterministic mode in PyTorch can have a performance '
    /root/anaconda3/envs/myenv/lib/python3.7/site-packages/gym/spaces/box.py:74: UserWarning: WARN: Box bound precision lowered by casting to float32
      "Box bound precision lowered by casting to {}".format(self.dtype)
    2021-12-10 22:49:07 | [ddpg_pendulum] Obtaining samples...
    /root/anaconda3/envs/myenv/lib/python3.7/site-packages/garage/_dtypes.py:1050: UserWarning: Observation array([-0.02584172,  0.0975075 , -0.02873373,  0.99523479,  0.9995871 ,
           -0.07543997,  0.10338352, -0.14566433,  0.        ,  0.        ,
            0.        ]) is outside observation_space Box([-inf -inf -inf -inf -inf -inf -inf -inf -inf -inf -inf], [inf inf inf inf inf inf inf inf inf inf inf], (11,), float32)
      f'Observation {value[0]!r} is outside '
    /root/anaconda3/envs/myenv/lib/python3.7/site-packages/garage/_dtypes.py:1050: UserWarning: Observation array([ 0.01983944, -0.02140418,  0.06770129,  0.9997709 ,  0.99770564,
            0.09051462, -0.02292237, -0.03486985,  0.        ,  0.        ,
            0.        ]) is outside observation_space Box([-inf -inf -inf -inf -inf -inf -inf -inf -inf -inf -inf], [inf inf inf inf inf inf inf inf inf inf inf], (11,), float32)
      f'Observation {value[0]!r} is outside '
    /root/anaconda3/envs/myenv/lib/python3.7/site-packages/garage/_dtypes.py:1050: UserWarning: Observation array([ 0.00337725,  0.00958483,  0.07754256,  0.99995406,  0.99698904,
           -0.0055185 , -0.08068346,  0.01616359,  0.        ,  0.        ,
            0.        ]) is outside observation_space Box([-inf -inf -inf -inf -inf -inf -inf -inf -inf -inf -inf], [inf inf inf inf inf inf inf inf inf inf inf], (11,), float32)
      f'Observation {value[0]!r} is outside '
    /root/anaconda3/envs/myenv/lib/python3.7/site-packages/garage/_dtypes.py:1050: UserWarning: Observation array([-0.01815734, -0.06201092,  0.04631587,  0.99807547,  0.99892684,
           -0.0378659 , -0.03774889, -0.13267713,  0.        ,  0.        ,
            0.        ]) is outside observation_space Box([-inf -inf -inf -inf -inf -inf -inf -inf -inf -inf -inf], [inf inf inf inf inf inf inf inf inf inf inf], (11,), float32)
      f'Observation {value[0]!r} is outside '
    /root/anaconda3/envs/myenv/lib/python3.7/site-packages/garage/_dtypes.py:1050: UserWarning: Observation array([-0.05197795, -0.07166224, -0.01771069,  0.99742896,  0.99984315,
           -0.13193943, -0.01721933, -0.04047698,  0.        ,  0.        ,
            0.        ]) is outside observation_space Box([-inf -inf -inf -inf -inf -inf -inf -inf -inf -inf -inf], [inf inf inf inf inf inf inf inf inf inf inf], (11,), float32)
      f'Observation {value[0]!r} is outside '

`

krzentner commented 2 years ago

Ah, sorry, I had mixed up LD_PRELOAD and LD_LIBRARY_PATH. It looks like after setting the variable how mujoco_py specifies, the code runs correctly. The warnings are because the observations space and actual observations of that environment are different floating point types. This is a bug in gym.spaces, which I would like to fix in akro one day. If you silence the warnings it should work fine.