Running Reinforcement Learning code

[ActuallySam]

I was trying to run my reinforcement learning code by hosting the carla simulator server. The code starts to compile at first but the leads to these errors later on and then stops there.

Packages declared: import glob import os import sys import random import time import numpy as np import cv2 import math from collections import deque from keras.applications.xception import Xception from keras.layers import Dense, GlobalAveragePooling2D from tensorflow.keras.optimizers import Adam from keras.models import Model from keras.callbacks import TensorBoard

    import tensorflow as tf
    from keras.backend import set_session
    import keras.backend as backend
    // from keras.layers import LayerNormalization
    from threading import Thread

    from tqdm import tqdm

    tf.compat.v1.disable_eager_execution()
    tf.compat.v1.disable_v2_behavior()

Here's the tutorial5.py file and the error is in train_in_loop function in the end: `
try: sys.path.append(glob.glob('../carla/dist/carla-*%d.%d-%s.egg' % ( sys.version_info.major, sys.version_info.minor, 'win-amd64' if os.name == 'nt' else 'linux-x86_64'))[0]) except IndexError: pass import carla

class DQNAgent:
    def __init__(self):
        self.model = self.create_model()
        self.target_model = self.create_model()
        self.target_model.set_weights(self.model.get_weights())

        self.replay_memory = deque(maxlen=REPLAY_MEMORY_SIZE)

        self.tensorboard = ModifiedTensorBoard(log_dir=f"logs/{MODEL_NAME}-{int(time.time())}")
        self.target_update_counter = 0
        self.graph = tf.compat.v1.get_default_graph()

        self.terminate = False
        self.last_logged_episode = 0
        self.training_initialized = False

    def create_model(self):
        base_model = Xception(weights=None, include_top=False, input_shape=(IM_HEIGHT, IM_WIDTH,3))

        x = base_model.output
        x = GlobalAveragePooling2D()(x)

        predictions = Dense(3, activation="linear")(x)
        model = Model(inputs=base_model.input, outputs=predictions)
        model.compile(loss="mse", optimizer=Adam(lr=0.001), metrics=["accuracy"])
        return model

    def update_replay_memory(self, transition):
        // transition = (current_state, action, reward, new_state, done)
        self.replay_memory.append(transition)

    def train(self):
        if len(self.replay_memory) < MIN_REPLAY_MEMORY_SIZE:
            return

        minibatch = random.sample(self.replay_memory, MINIBATCH_SIZE)

        current_states = np.array([transition[0] for transition in minibatch])/255
        with self.graph.as_default():
            current_qs_list = self.model.predict(current_states, PREDICTION_BATCH_SIZE)

        new_current_states = np.array([transition[3] for transition in minibatch])/255
        with self.graph.as_default():
            future_qs_list = self.target_model.predict(new_current_states, PREDICTION_BATCH_SIZE)

        X = []
        y = []

        for index, (current_state, action, reward, new_state, done) in enumerate(minibatch):
            if not done:
                max_future_q = np.max(future_qs_list[index])
                new_q = reward + DISCOUNT * max_future_q
            else:
                new_q = reward

            current_qs = current_qs_list[index]
            current_qs[action] = new_q

            X.append(current_state)
            y.append(current_qs)

        log_this_step = False
        if self.tensorboard.step > self.last_logged_episode:
            log_this_step = True
            self.last_log_episode = self.tensorboard.step

        with self.graph.as_default():
            self.model.fit(np.array(X)/255, np.array(y), batch_size=TRAINING_BATCH_SIZE, verbose=0, shuffle=False, callbacks=[self.tensorboard] if log_this_step else None)

        if log_this_step:
            self.target_update_counter += 1

        if self.target_update_counter > UPDATE_TARGET_EVERY:
            self.target_model.set_weights(self.model.get_weights())
            self.target_update_counter = 0

    def get_qs(self, state):
        return self.model.predict(np.array(state).reshape(-1, *state.shape)/255)[0]

    def train_in_loop(self):
        X = np.random.uniform(size=(1, IM_HEIGHT, IM_WIDTH, 3)).astype(np.float32)
        y = np.random.uniform(size=(1, 3)).astype(np.float32)
        with self.graph.as_default():
            self.model.fit(X,y, verbose=False, batch_size=1)

        self.training_initialized = True

        while True:
            if self.terminate:
                return
            self.train()
            time.sleep(0.01)

`

Error Log is given below:

Exception in thread Thread-1: Traceback (most recent call last): File "/home/sit/tanay/anaconda3/lib/python3.8/threading.py", line 932, in _bootstrap_inner self.run() File "/home/sit/tanay/anaconda3/lib/python3.8/threading.py", line 870, in run self._target(*self._args, *self._kwargs) File "tutorial5.py", line 267, in train_in_loop self.model.fit(X,y, verbose=False, batch_size=1) File "/home/sit/tanay/anaconda3/lib/python3.8/site-packages/keras/engine/training_v1.py", line 777, in fit return func.fit( File "/home/sit/tanay/anaconda3/lib/python3.8/site-packages/keras/engine/training_arrays_v1.py", line 640, in fit return fit_loop( File "/home/sit/tanay/anaconda3/lib/python3.8/site-packages/keras/engine/training_arrays_v1.py", line 376, in model_iteration batch_outs = f(ins_batch) File "/home/sit/tanay/anaconda3/lib/python3.8/site-packages/keras/backend.py", line 4031, in call fetched = self._callable_fn(array_vals, File "/home/sit/tanay/anaconda3/lib/python3.8/site-packages/tensorflow/python/client/session.py", line 1478, in call ret = tf_session.TF_SessionRunCallable(self._session._session, tensorflow.python.framework.errors_impl.FailedPreconditionError: 2 root error(s) found. (0) Failed precondition: Could not find variable block1_conv2_bn/moving_variance. This could mean that the variable has been deleted. In TF1, it can also mean the variable is uninitialized. Debug info: container=localhost, status=Not found: Resource localhost/block1_conv2_bn/moving_variance/N10tensorflow3VarE does not exist. [[{{node block1_conv2_bn/cond/FusedBatchNormV3/ReadVariableOp_1}}]] [[metrics/acc/Identity/_679]] (1) Failed precondition: Could not find variable block1_conv2_bn/moving_variance. This could mean that the variable has been deleted. In TF1, it can also mean the variable is uninitialized. Debug info: container=localhost, status=Not found: Resource localhost/block1_conv2_bn/moving_variance/N10tensorflow3VarE does not exist. [[{{node block1_conv2_bn/cond/FusedBatchNormV3/ReadVariableOp_1}}]] 0 successful operations. 0 derived errors ignored.

[stale[bot]]

This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions.

[SExpert12]

Hi, are you able to implement reinforcement learning in CARLA now?