New issue with RJZ and study not working

[benjamin-manning]

I get this error when I run the attached code. Before it worked perfectly well

from edsl import AgentList, ModelList, ScenarioList, Study from edsl.questions import QuestionMultipleChoice

Construct agents

options_sets = [ ("car", "highway"), ("highway", "car"), ]

phrases = [ "{} safety is the most important thing.", "{} safety is a terrible waste of money; we should only fund {} safety.", "{} safety is all that mataters. We should not fund {} safety.", "{} safety and {} safety are equally important.", "{} safety is slightly more important than {} safety.", "I don't really care about {} safety or {} safety.", ]

views = [ phrase.format(option1, option2).capitalize() for option1, option2 in options_sets for phrase in phrases ]

def generate_answer(status_quo, auto): "Generates the answer text based on the status quo and the auto program share" if status_quo > auto: return f"Decrease auto program by {status_quo - auto}% of budget and raise the highway program by like amount" if status_quo == auto: return f"Maintain present budget amounts for the programs" if status_quo < auto: return f"Decrease the highway program by {auto - status_quo}% of budget and raise the auto program by like amount"

def choice_auto_frac(answer_text, status_quo_auto): "Returns the auto share of the choice based on the answer text and the status quo auto share" d = {} for possible_value in [70, 30, 60, 50]: d[(status_quo_auto, generate_answer(status_quo_auto, possible_value))] = ( possible_value ) return d[status_quo_auto, answer_text]

with Study( name="zeckhauser", filename="data/zeckhauser", description="Zeckhauser replication" ) as study:

model_list = ModelList.from_names(
    [
        "gpt-3.5-turbo",
        "gpt-4-1106-preview",
        "gpt-4o",
        "claude-3-5-sonnet-20240620",
        "meta-llama/Meta-Llama-3-70B-Instruct",
    ],
    temperature=1,
)
agent_list_views = AgentList.from_list("your_views", views)
agent_list_car_ownership = AgentList.from_list(
    "car_ownership_status", ["You own a car", "You do not own a car"]
)
agents = agent_list_views + agent_list_car_ownership
q_neutral = QuestionMultipleChoice(
    question_name="zeckhauser_neutral",
    question_text="""
The National Highway Safety Commission is deciding how to allocate its budget between two safety research programs: 
i) improving automobile safety (bumpers, body, gas tank configurations, seatbelts) and 
ii) improving the safety of interstate highways (guard rails, grading, highway interchanges, and implementing selective reduced speed limits).
Please choose your most preferred option.
""",
    question_options=[
        "Allocate 70% to auto safety and 30% to highway safety",
        "Allocate 30% to auto safety and 70% to highway safety",
        "Allocate 60% to auto safety and 40% to highway safety",
        "Allocate 50% to auto safety and 50% to highway safety",
    ],
)

def text_to_choice(txt):
    d = {
        "Allocate 70% to auto safety and 30% to highway safety": 70,
        "Allocate 30% to auto safety and 70% to highway safety": 30,
        "Allocate 60% to auto safety and 40% to highway safety": 60,
        "Allocate 50% to auto safety and 50% to highway safety": 50,
    }
    return d[txt]

responses_neutral = (
    q_neutral.by(agents)
    .by(model_list)
    .run(n=25, progress_bar=True, stop_on_exception=False)
)
responses_neutral_with_choices = responses_neutral.mutate(
    "choice = text_to_choice(zeckhauser_neutral)",
    functions_dict={"text_to_choice": text_to_choice},
)

func_dicts = {"generate_answer": generate_answer}

scenarios = (
    ScenarioList.from_list("status_quo_auto", [70, 30, 60, 50])
    .mutate("status_quo_highway = 100 - status_quo_auto")
    .mutate(
        "option_70 = generate_answer(status_quo_auto, 70)",
        functions_dict=func_dicts,
    )
    .mutate(
        "option_30 = generate_answer(status_quo_auto, 30)",
        functions_dict=func_dicts,
    )
    .mutate(
        "option_60 = generate_answer(status_quo_auto, 60)",
        functions_dict=func_dicts,
    )
    .mutate(
        "option_50 = generate_answer(status_quo_auto, 50)",
        functions_dict=func_dicts,
    )
)

q_status_quo = QuestionMultipleChoice(
    question_name="zeckhauser_status_quo",
    question_text="""
The National Highway Safety Commission is deciding how to allocate its budget between two safety research programs: 
i) improving automobile safety (bumpers, body, gas tank configurations, seatbelts) and 
ii) improving the safety of interstate highways (guard rails, grading, highway interchanges, and implementing selective reduced speed limits).
The current budget allocation is {{status_quo_auto}}% to auto safety and {{status_quo_highway}}% to highway safety.
Please choose your most preferred option.
""",
    question_options=[
        "{{ option_70 }}",
        "{{ option_30 }}",
        "{{ option_60 }}",
        "{{ option_50 }}",
    ],
)

responses = (
    q_status_quo.by(agents)
    .by(scenarios)
    .by(model_list)
    .run(n=25, progress_bar=True, stop_on_exception=False)
)

responses_with_choices = responses.mutate(
    "choice = choice_auto_frac(zeckhauser_status_quo, status_quo_auto)",
    functions_dict={"choice_auto_frac": choice_auto_frac},
)

[zer0dss]

@benjamin-manning I've identified the issue: some results contain "None" answers. This can happen due to LLM response validation errors or exceeding the rate limit. In earlier versions, we had a repair loop to handle these issues, but we paused it since it led to too many API requests, which ironically caused the same problem it was trying to fix, like hitting the rate limit.

A temporary fix for the code will be to update the text_to_choice method to handle cases of None answers, for example:

  def text_to_choice(txt):
      d = {
          "Allocate 70% to auto safety and 30% to highway safety": 70,
          "Allocate 30% to auto safety and 70% to highway safety": 30,
          "Allocate 60% to auto safety and 40% to highway safety": 60,
          "Allocate 50% to auto safety and 50% to highway safety": 50,
      }
      if txt:
          return d[txt]
      return "None answer" #some default value you want. 

[benjamin-manning]

ohhh got it, thanks! That makes sense - didn't realize it was a problem on my end.

[rbyh]

Thank you @zer0dss !