ValidationError: Input should be a valid string

[orhansonmeztr]

Hi, I run the following sample script given in the README.md file after the installation of this evaluator with the command guardrails hub install hub://arize-ai/relevancy_evaluator.

import os
from dotenv import load_dotenv
load_dotenv()

# Import Guard and Validator
from guardrails.hub import RelevancyEvaluator
from guardrails import Guard

# Setup Guard
guard = Guard().use(
    RelevancyEvaluator(llm_callable="gpt-3.5-turbo")
)

# Example values
value = {
    "original_prompt": "What is the capital of France?",
    "reference_text": "The capital of France is Paris."
}

guard.validate(value)  # Validator passes if the text is relevant

When I run this code, I get the following error.

    "name": "ValidationError",
    "message": "1 validation error for Inputs
llm_output
  Input should be a valid string [type=string_type, input_value={'original_prompt': 'What...al of France is Paris.'}, input_type=dict]
    For further information visit https://errors.pydantic.dev/2.8/v/string_type",
    "stack": "---------------------------------------------------------------------------
ValidationError                           Traceback (most recent call last)
Cell In[2], line 17
     11 # Example values
     12 value = {
     13     \"original_prompt\": \"What is the capital of France?\",
     14     \"reference_text\": \"The capital of France is Paris.\"
     15 }
---> 17 guard.validate(value)  # Validator passes if the text is relevant

File c:\\Users\\orhan\\PycharmProjects\\test1\\.venv\\Lib\\site-packages\\guardrails\\guard.py:1153, in Guard.validate(self, llm_output, *args, **kwargs)
   1152 def validate(self, llm_output: str, *args, **kwargs) -> ValidationOutcome[OT]:
-> 1153     return self.parse(llm_output=llm_output, *args, **kwargs)

File c:\\Users\\orhan\\PycharmProjects\\test1\\.venv\\Lib\\site-packages\\guardrails\\guard.py:1020, in Guard.parse(self, llm_output, metadata, llm_api, num_reasks, prompt_params, full_schema_reask, *args, **kwargs)
   1017 default_msg_history = self._exec_opts.msg_history if llm_api else None
   1018 msg_history = kwargs.pop(\"msg_history\", default_msg_history)
-> 1020 return trace_guard_execution(
   1021     self.name,
   1022     self.history,
   1023     self._execute,  # type: ignore # streams are supported for parse
   1024     self._tracer,
   1025     *args,
   1026     llm_output=llm_output,
   1027     llm_api=llm_api,
   1028     prompt_params=prompt_params,
   1029     num_reasks=final_num_reasks,
   1030     prompt=prompt,
   1031     instructions=instructions,
   1032     msg_history=msg_history,
   1033     metadata=metadata,
   1034     full_schema_reask=full_schema_reask,
   1035     **kwargs,
   1036 )

File c:\\Users\\orhan\\PycharmProjects\\test1\\.venv\\Lib\\site-packages\\guardrails\\telemetry\\guard_tracing.py:181, in trace_guard_execution(guard_name, history, _execute_fn, tracer, *args, **kwargs)
    179         except Exception as e:
    180             guard_span.set_status(status=StatusCode.ERROR, description=str(e))
--> 181             raise e
    182 else:
    183     return _execute_fn(*args, **kwargs)

File c:\\Users\\orhan\\PycharmProjects\\test1\\.venv\\Lib\\site-packages\\guardrails\\telemetry\\guard_tracing.py:172, in trace_guard_execution(guard_name, history, _execute_fn, tracer, *args, **kwargs)
    169 guard_span.set_attribute(\"guard.name\", guard_name)
    171 try:
--> 172     result = _execute_fn(*args, **kwargs)
    173     if isinstance(result, Iterable) and not isinstance(
    174         result, ValidationOutcome
    175     ):
    176         return trace_stream_guard(guard_span, result, history)

File c:\\Users\\orhan\\PycharmProjects\\test1\\.venv\\Lib\\site-packages\\guardrails\\guard.py:837, in Guard._execute(self, llm_api, llm_output, prompt_params, num_reasks, prompt, instructions, msg_history, reask_prompt, reask_instructions, metadata, full_schema_reask, *args, **kwargs)
    834 current_otel_context = otel_context.get_current()
    835 wrapped__exec = wrap_with_otel_context(current_otel_context, __exec)
--> 837 return guard_context.run(
    838     wrapped__exec,
    839     self,
    840     llm_api=llm_api,
    841     llm_output=llm_output,
    842     prompt_params=prompt_params,
    843     num_reasks=num_reasks,
    844     prompt=prompt,
    845     instructions=instructions,
    846     msg_history=msg_history,
    847     metadata=metadata,
    848     full_schema_reask=full_schema_reask,
    849     *args,
    850     **kwargs,
    851 )

File c:\\Users\\orhan\\PycharmProjects\\test1\\.venv\\Lib\\site-packages\\guardrails\\telemetry\\common.py:99, in wrap_with_otel_context.<locals>.wrapped_func(*args, **kwargs)
     96 token = context.attach(outer_scope_otel_context)
     97 try:
     98     # Execute 'func' within the attached context
---> 99     return func(*args, **kwargs)
    100 finally:
    101     # Ensure the context is detached after execution
    102     #   to maintain correct context management
    103     context.detach(token)

File c:\\Users\\orhan\\PycharmProjects\\test1\\.venv\\Lib\\site-packages\\guardrails\\guard.py:814, in Guard._execute.<locals>.__exec(self, llm_api, llm_output, prompt_params, num_reasks, prompt, instructions, msg_history, metadata, full_schema_reask, *args, **kwargs)
    812 self.history.push(call_log)
    813 # Otherwise, call the LLM synchronously
--> 814 return self._exec(
    815     llm_api=llm_api,
    816     llm_output=llm_output,
    817     prompt_params=prompt_params,
    818     num_reasks=self._num_reasks,
    819     prompt=prompt,
    820     instructions=instructions,
    821     msg_history=msg_history,
    822     metadata=metadata,
    823     full_schema_reask=full_schema_reask,
    824     call_log=call_log,
    825     *args,
    826     **kwargs,
    827 )

File c:\\Users\\orhan\\PycharmProjects\\test1\\.venv\\Lib\\site-packages\\guardrails\\guard.py:915, in Guard._exec(self, llm_api, llm_output, call_log, prompt_params, num_reasks, metadata, full_schema_reask, prompt, instructions, msg_history, *args, **kwargs)
    897 else:
    898     # Otherwise, use Runner
    899     runner = Runner(
    900         output_type=self._output_type,
    901         output_schema=self.output_schema.to_dict(),
   (...)
    913         exec_options=self._exec_opts,
    914     )
--> 915     call = runner(call_log=call_log, prompt_params=prompt_params)
    916     return ValidationOutcome[OT].from_guard_history(call)

File c:\\Users\\orhan\\PycharmProjects\\test1\\.venv\\Lib\\site-packages\\guardrails\\run\\runner.py:242, in Runner.__call__(self, call_log, prompt_params)
    239 except Exception as e:
    240     # Because Pydantic v1 doesn't respect property setters
    241     call_log.exception = e
--> 242     raise e
    243 return call_log

File c:\\Users\\orhan\\PycharmProjects\\test1\\.venv\\Lib\\site-packages\\guardrails\\run\\runner.py:197, in Runner.__call__(self, call_log, prompt_params)
    194 index = 0
    195 for index in range(self.num_reasks + 1):
    196     # Run a single step.
--> 197     iteration = self.step(
    198         index=index,
    199         api=self.api,
    200         instructions=instructions,
    201         prompt=prompt,
    202         msg_history=msg_history,
    203         prompt_params=prompt_params,
    204         output_schema=output_schema,
    205         output=self.output if index == 0 else None,
    206         call_log=call_log,
    207     )
    209     # Loop again?
    210     if not self.do_loop(index, iteration.reasks):

File c:\\Users\\orhan\\PycharmProjects\\test1\\.venv\\Lib\\site-packages\\guardrails\\telemetry\\runner_tracing.py:79, in trace_step.<locals>.trace_step_wrapper(*args, **kwargs)
     77             step_span.set_status(status=StatusCode.ERROR, description=str(e))
     78             add_step_attributes(step_span, None, *args, **kwargs)
---> 79             raise e
     80 else:
     81     return fn(*args, **kwargs)

File c:\\Users\\orhan\\PycharmProjects\\test1\\.venv\\Lib\\site-packages\\guardrails\\telemetry\\runner_tracing.py:73, in trace_step.<locals>.trace_step_wrapper(*args, **kwargs)
     68 with tracer.start_as_current_span(
     69     name=\"step\",  # type: ignore
     70     context=current_otel_context,  # type: ignore
     71 ) as step_span:
     72     try:
---> 73         response = fn(*args, **kwargs)
     74         add_step_attributes(step_span, response, *args, **kwargs)
     75         return response

File c:\\Users\\orhan\\PycharmProjects\\test1\\.venv\\Lib\\site-packages\\guardrails\\run\\runner.py:261, in Runner.step(self, index, output_schema, call_log, api, instructions, prompt, msg_history, prompt_params, output)
    259 \"\"\"Run a full step.\"\"\"
    260 prompt_params = prompt_params or {}
--> 261 inputs = Inputs(
    262     llm_api=api,
    263     llm_output=output,
    264     instructions=instructions,
    265     prompt=prompt,
    266     msg_history=msg_history,
    267     prompt_params=prompt_params,
    268     num_reasks=self.num_reasks,
    269     metadata=self.metadata,
    270     full_schema_reask=self.full_schema_reask,
    271 )
    272 outputs = Outputs()
    273 iteration = Iteration(
    274     call_id=call_log.id, index=index, inputs=inputs, outputs=outputs
    275 )

File c:\\Users\\orhan\\PycharmProjects\\test1\\.venv\\Lib\\site-packages\\pydantic\\main.py:193, in BaseModel.__init__(self, **data)
    191 # `__tracebackhide__` tells pytest and some other tools to omit this function from tracebacks
    192 __tracebackhide__ = True
--> 193 self.__pydantic_validator__.validate_python(data, self_instance=self)

ValidationError: 1 validation error for Inputs
llm_output
  Input should be a valid string [type=string_type, input_value={'original_prompt': 'What...al of France is Paris.'}, input_type=dict]
    For further information visit https://errors.pydantic.dev/2.8/v/string_type"
}

I load the openai_api_key with load_dotenv() and ensure that the openai_api_key loaded correctly via print(os.environ['OPENAI_API_KEY']) that prints 'sk-...'.

Python version: 3.11.8

library versions: guardrails-ai==0.5.4 litellm==1.43.18 openai==1.41.0

Could you explain how to use relevancy_evaluator without this error? Any help is appreciated. Orhan

[nichwch]

Hi,

We've replicated the bug! We're working on changing the interface of this validator as well, to fit more with the patterns that other validators use - sit tight, and we'll have a fix out shortly. Thanks again for your patience, and helping report this issue.

[nichwch]

(edited to reflect @bgg01590 's correction)

Hey, we've merged in a fix for this issue.

Try initializing the Guard as follows:

import os
from dotenv import load_dotenv
load_dotenv()

# Import Guard and Validator
from guardrails.hub import RelevancyEvaluator
from guardrails import Guard

# Setup Guard
guard = Guard().use(
    RelevancyEvaluator(llm_callable="gpt-3.5-turbo")
)

# Example values
metadata = {
    "original_prompt": "What is the capital of France?",
}

guard.parse("The capital of France is Paris.", metadata=metadata)  # Validator passes if the text is relevant

[bgg01590]

...
guard.parse("The capital of France is Paris.", metadata)  # Validator passes if the text is relevant

should be

...
guard.parse("The capital of France is Paris.", metadata=metadata)  # Validator passes if the text is relevant

It only worked when I explicitly passed the metadata parameter.

[orhansonmeztr]

Hey, we've merged in a fix for this issue.

Try initializing the Guard as follows:

import os
from dotenv import load_dotenv
load_dotenv()

# Import Guard and Validator
from guardrails.hub import RelevancyEvaluator
from guardrails import Guard

# Setup Guard
guard = Guard().use(
    RelevancyEvaluator(llm_callable="gpt-3.5-turbo")
)

# Example values
metadata = {
    "original_prompt": "What is the capital of France?",
}

guard.parse("The capital of France is Paris.", metadata)  # Validator passes if the text is relevant

Thank you @nichwch But it gives the following error after upgrading the guardrails-ai library to 0.5.5 version and running your new script:

Traceback (most recent call last):
  File "C:\Users\orhan\PycharmProjects\test1\guardrail_test.py", line 19, in <module>
    guard.parse("The capital of France is Paris.", metadata)  # Validator passes if the text is relevant
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\guard.py", line 1025, in parse
    return trace_guard_execution(
           ^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\telemetry\guard_tracing.py", line 181, in trace_guard_execution
    raise e
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\telemetry\guard_tracing.py", line 172, in trace_guard_execution
    result = _execute_fn(*args, **kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\guard.py", line 842, in _execute
    return guard_context.run(
           ^^^^^^^^^^^^^^^^^^
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\telemetry\common.py", line 99, in wrapped_func
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\guard.py", line 819, in __exec
    return self._exec(
           ^^^^^^^^^^^
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\guard.py", line 920, in _exec
    call = runner(call_log=call_log, prompt_params=prompt_params)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\run\runner.py", line 242, in __call__
    raise e
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\run\runner.py", line 197, in __call__
    iteration = self.step(
                ^^^^^^^^^^
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\telemetry\runner_tracing.py", line 79, in trace_step_wrapper
    raise e
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\telemetry\runner_tracing.py", line 73, in trace_step_wrapper
    response = fn(*args, **kwargs)
               ^^^^^^^^^^^^^^^^^^^
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\run\runner.py", line 335, in step
    raise e
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\run\runner.py", line 320, in step
    validated_output = self.validate(
                       ^^^^^^^^^^^^^^
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\run\runner.py", line 596, in validate
    validated_output, metadata = validator_service.validate(
                                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\validator_service.py", line 1014, in validate
                      ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\validator_service.py", line 795, in run_validators
    result = await self.run_validator(
             ^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\validator_service.py", line 733, in run_validator
    result = await self.run_validator_async(
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\validator_service.py", line 702, in run_validator_async
    result: ValidatorResult = self.execute_validator(
                              ^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\validator_service.py", line 69, in execute_validator
    result = traced_validator(value, metadata)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\telemetry\validator_tracing.py", line 134, in trace_validator_wrapper
    raise e
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\telemetry\validator_tracing.py", line 106, in trace_validator_wrapper
    resp = fn(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\hub\arize_ai\relevancy_evaluator\validator\main.py", line 115, in validate
    original_prompt = value.get("original_prompt")
                      ^^^^^^^^^
AttributeError: 'str' object has no attribute 'get'

The command @bgg01590 wrote gave the same error. What am I doing wrong?

[nichwch]

...
guard.parse("The capital of France is Paris.", metadata)  # Validator passes if the text is relevant

should be

...
guard.parse("The capital of France is Paris.", metadata=metadata)  # Validator passes if the text is relevant

It only worked when I explicitly passed the metadata parameter.

My mistake, you do need to explicitly pass the metadata parameter, thanks for calling that out

[nichwch]

Hey, we've merged in a fix for this issue. Try initializing the Guard as follows:

import os
from dotenv import load_dotenv
load_dotenv()

# Import Guard and Validator
from guardrails.hub import RelevancyEvaluator
from guardrails import Guard

# Setup Guard
guard = Guard().use(
    RelevancyEvaluator(llm_callable="gpt-3.5-turbo")
)

# Example values
metadata = {
    "original_prompt": "What is the capital of France?",
}

guard.parse("The capital of France is Paris.", metadata)  # Validator passes if the text is relevant

Thank you @nichwch But it gives the following error after upgrading the guardrails-ai library to 0.5.5 version and running your new script:

Traceback (most recent call last):
  File "C:\Users\orhan\PycharmProjects\test1\guardrail_test.py", line 19, in <module>
    guard.parse("The capital of France is Paris.", metadata)  # Validator passes if the text is relevant
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\guard.py", line 1025, in parse
    return trace_guard_execution(
           ^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\telemetry\guard_tracing.py", line 181, in trace_guard_execution
    raise e
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\telemetry\guard_tracing.py", line 172, in trace_guard_execution
    result = _execute_fn(*args, **kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\guard.py", line 842, in _execute
    return guard_context.run(
           ^^^^^^^^^^^^^^^^^^
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\telemetry\common.py", line 99, in wrapped_func
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\guard.py", line 819, in __exec
    return self._exec(
           ^^^^^^^^^^^
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\guard.py", line 920, in _exec
    call = runner(call_log=call_log, prompt_params=prompt_params)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\run\runner.py", line 242, in __call__
    raise e
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\run\runner.py", line 197, in __call__
    iteration = self.step(
                ^^^^^^^^^^
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\telemetry\runner_tracing.py", line 79, in trace_step_wrapper
    raise e
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\telemetry\runner_tracing.py", line 73, in trace_step_wrapper
    response = fn(*args, **kwargs)
               ^^^^^^^^^^^^^^^^^^^
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\run\runner.py", line 335, in step
    raise e
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\run\runner.py", line 320, in step
    validated_output = self.validate(
                       ^^^^^^^^^^^^^^
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\run\runner.py", line 596, in validate
    validated_output, metadata = validator_service.validate(
                                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\validator_service.py", line 1014, in validate
                      ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\validator_service.py", line 795, in run_validators
    result = await self.run_validator(
             ^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\validator_service.py", line 733, in run_validator
    result = await self.run_validator_async(
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\validator_service.py", line 702, in run_validator_async
    result: ValidatorResult = self.execute_validator(
                              ^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\validator_service.py", line 69, in execute_validator
    result = traced_validator(value, metadata)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\telemetry\validator_tracing.py", line 134, in trace_validator_wrapper
    raise e
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\telemetry\validator_tracing.py", line 106, in trace_validator_wrapper
    resp = fn(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^
  File "C:\Users\orhan\PycharmProjects\test1\.venv\Lib\site-packages\guardrails\hub\arize_ai\relevancy_evaluator\validator\main.py", line 115, in validate
    original_prompt = value.get("original_prompt")
                      ^^^^^^^^^
AttributeError: 'str' object has no attribute 'get'

The command @bgg01590 wrote gave the same error. What am I doing wrong?

Can you try reinstalling the relevancy_evaluator validator?

[orhansonmeztr]

it works now, thanks

[nichwch]

Glad I could help!