Pygame Keyboard Event Handling Not Working with ALE Environment in WSL 2

[xiezhipeng-git]

I am experiencing an issue where Pygame is unable to detect keyboard events when running the Arcade Learning Environment (ALE) within WSL 2. This problem seems to be specific to the ALE environment, as other Gym environments do not exhibit this behavior and respond to keyboard inputs as expected. When attempting to capture keyboard events using Pygame's event loop, no events are registered, and the program does not respond to any keypresses. This issue persists despite having an X Server (such as VcXsrv or Xming) running and properly configured to handle GUI output from WSL 2. This lack of responsiveness from Pygame's event handling mechanism is hindering the ability to interact with the ALE environment in a real-time manner, which is essential for certain types of reinforcement learning experiments and interactive debugging sessions.

import gym
from ale_py import ALEInterface
# from ale_py.roms import Pitfall2
# from ale_py.roms import MontezumaRevenge
import time
# import keyboard
# import pynput
import pygame
import copy
# 不加初始化会有bug
pygame.init()
# global action
# action = 0
debug_print = False
testKey = False
keys = []
actionReplay = []
gifFrames = []
env_name = "Pitfall"
is_record_gif = False
# ale = ALEInterface()
# ale.loadROM(Pitfall2)
# ale.loadROM(MontezumaRevenge)

# env = gym.make('ALE/Pitfall2-v5',render_mode = 'human')
def to_action(env_name,keys,press_num = 0):
    if env_name=="Pitfall":
        action = pitfall_action(keys,press_num)
    else:
        print("不支持的游戏按键。")
        return 0
    return action
def pitfall_action(keys,press_num = 0):
    # keys.sort()
    action = 0
    if press_num == 0:
        return 0
    if keys[pygame.K_w] and keys[pygame.K_d] and keys[pygame.K_SPACE]:
        # 上右跳
        action = 14
    elif keys[pygame.K_w] and keys[pygame.K_a] and keys[pygame.K_SPACE]:
        # 上左跳
        action = 15
    elif keys[pygame.K_s] and keys[pygame.K_d] and keys[pygame.K_SPACE]:
        # 右下跳
        action = 16
    elif keys[pygame.K_s] and keys[pygame.K_a] and keys[pygame.K_SPACE]:
        # 左下跳
        action = 17
    elif keys[pygame.K_w] and keys[pygame.K_d]:
        # 上右
        action = 6
    elif keys[pygame.K_w] and keys[pygame.K_a]:
        # 上左
        action = 7
    elif keys[pygame.K_s] and keys[pygame.K_d]:
        # 右下
        action = 8
    elif keys[pygame.K_s] and keys[pygame.K_a]:
        # 左下
        action = 9
    elif keys[pygame.K_w] and keys[pygame.K_SPACE]:
        # 上跳
        action = 10
    elif keys[pygame.K_d] and keys[pygame.K_SPACE]:
        # 右跳
        action = 11
    elif keys[pygame.K_a] and keys[pygame.K_SPACE]:
        # 左跳
        action = 12
    elif keys[pygame.K_s] and keys[pygame.K_SPACE]:
        # 下跳
        action = 13
    elif keys[pygame.K_SPACE]:
        action = 1
    elif keys[pygame.K_w]:
        # 上
        action = 2
    elif keys[pygame.K_d]:
        # 右
        action = 3
    elif keys[pygame.K_a]:
        # 左
        action = 4
    elif keys[pygame.K_s]:
        # 下
        action = 5
    # elif len(keys)>=1:
    #     key1Arr = keys[0].split('f')
    #     if len(key1Arr)>1:
    #         action =10+ int(key1Arr[1])%10
    #     else:
    #         action = int(key1Arr[0])
    else:
        action = 0
    # print(keys,action)
    if action !=0:
        print(action)

    return action

def to_action_test(action,shiftkeys,fkeys,nums):
    if len(fkeys)>0:
        action = 10+ int(fkeys[0])%10
        print("action_test = ",action)
    elif len(nums)>0:
        action = int(nums[0])%10
        print("action_test = ",action)
    return action
saved_env = {}
def save(index,env):
    # env = copy.deepcopy(env)
    # saved_env.update({index:env})
    # print("存不符合预期改用重头开始动作")
    saved_env.update({index:copy.deepcopy(actionReplay)})
    print("存",index,len(actionReplay),"步")
    time.sleep(1)

def load(index,env):
    if index in saved_env.keys():
    #     env = saved_env[index]
    #     env.render()
    # print("取不符合预期改用重头开始动作")
        env.reset()
        actionReplay = saved_env[index]
        print("取",index,len(actionReplay),"步")
        for a in actionReplay:
            env.step(a)
        env.render()
def save_or_load(press_shift_keys,press_f_keys,env):
    if len(press_f_keys)==0:
        return
    if len(press_shift_keys)>=1:
        # findex = pygame.key.name(press_keys[1])
        # index = int(findex.split('f')[1])
        load(press_f_keys[0],env)
    else :
        # findex = pygame.key.name(press_keys[0])
        # index = int(findex.split('f')[1])
        save(press_f_keys[0],env)

# env = gym.make('ALE/MontezumaRevenge-v5',render_mode = 'human')
env = gym.make(f'ALE/{env_name}-v5',render_mode = 'human',frameskip = 4,repeat_action_probability = 0.25)

# env = gym.make(f'ALE/{env_name}-v5', render_mode="rgb_array", disable_env_checker=True)
# 查看允许的按钮
envaction =env.get_keys_to_action()
print(envaction)
s, done = env.reset()
actionReplay = []
env.render()

total_reward = 0
done = False
while True:
    keys = pygame.key.get_pressed()
    press_num = keys.count(True)
    press_shift_keys = []
    press_f_keys = []
    press_num_keys = []
    # for event in pygame.event.get():
    #     if event.type == pygame.KEYDOWN:
    #         # 处理按下事件
    #         print("KEYDOWN",)
    #     if event.type == pygame.KEYUP:
    #         # 处理松开事件
    #         print("KEYUP",)
    for i in range(1):
        shiftkeys = [keys[pygame.K_LSHIFT],keys[pygame.K_RSHIFT]]
        shiftNum = shiftkeys.count(True)
        if shiftNum>0:
            shiftindex = shiftkeys.index(True)
            if shiftindex!= None:
                press_shift_keys.append(shiftindex)

        fkeys = [keys[pygame.K_F1] , keys[pygame.K_F2] , keys[pygame.K_F3] , keys[pygame.K_F4] , keys[pygame.K_F5] , keys[pygame.K_F6] , keys[pygame.K_F7] , keys[pygame.K_F8] , keys[pygame.K_F9] , keys[pygame.K_F10] , keys[pygame.K_F11] , keys[pygame.K_F12]]
        fNum = fkeys.count(True)
        if fNum >0:
            findex = fkeys.index(True)
            if findex!=None:
                press_f_keys.append((findex+1)%13)
        numkeys = [keys[pygame.K_1] , keys[pygame.K_2] , keys[pygame.K_3] , keys[pygame.K_4] , keys[pygame.K_5] , keys[pygame.K_6] , keys[pygame.K_7] , keys[pygame.K_8] , keys[pygame.K_9] , keys[pygame.K_0]]
        num_num = numkeys.count(True)
        if num_num >0:
            numindex = numkeys.index(True)
            press_num_keys.append((numindex+1)%10)
    if press_num > 1 and debug_print == True:
        t=keys.index(True)
        print(keys.index(True))
        print(press_shift_keys)
        print(press_f_keys)
        print(press_num_keys)

    try:
        action = 0
        if press_num>=1:
            action = to_action(env_name,keys,press_num)
        if action == 0 and press_num>=1 and testKey == True:
            action = to_action_test(action,press_shift_keys,press_f_keys,press_num_keys)
        next_state,reward,done,istimeout,_ = env.step(action)
    except Exception as e:
        action = 0
        next_state,reward,done,istimeout,_ = env.step(action)
        # print(e)
    actionReplay.append(action) 
    total_reward += reward
    # print(action,total_reward,done)
    # time.sleep(0.1)
    if istimeout:
        print("istimeout")
    if done:
        print("done")
        s, done = env.reset()
        actionReplay = []
    if keys[pygame.K_ESCAPE]:
        break
    if keys[96] or keys[126]:
        s, done = env.reset()
        actionReplay = []
    save_or_load(press_shift_keys,press_f_keys,env)
    env.render()

# env.step(0)
print("结束")
env.close()
# listenerKey.stop()

# pygame.quit()
#Reproduction Code:

If use cartpole gym,it is ok. If work in windows,it is ok.

Name: pygame Version: 2.5.2 Name: ale-py Version: 0.8.1

[pseudo-rnd-thoughts]

Are you sure that this is a pygame issue? How many keyboard events are you having to process each second? I suspect that render_mode="human" could cause issues

Also, why are you not using gym.utils.play for keyboard based interactions?

[xiezhipeng-git]

你确定这是一个pygame问题吗？每秒必须处理多少个键盘事件？我怀疑这可能会导致问题render_mode="human"

另外，为什么您不使用基于键盘的交互？gym.utils.play

I'm not very sure, because I'm not sure if the WSL environment was also created by pygame. However, it seems that Gym recommends using "human" to showcase in the future. The main reason is that I didn't know Gym already integrated play functionality