Unable to use computed values from relations after Pydantic v2 upgrade

[plusiv]

Describe the bug I'm unable to use computed values from relations since upgraded to Pydantic v2. When using the class method from_tortoise_orm the following error are occurring

  File "/some-path/projects/debug-tortoise-orm/.venv/lib/python3.11/site-packages/pydantic/main.py", line 353, in model_dump_json
    return self.__pydantic_serializer__.to_json(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
pydantic_core._pydantic_core.PydanticSerializationError: Error serializing to JSON: AttributeError: '__main__.Employee.leaf' object has no attribute 'team_members'

To Reproduce [CODE UPDATED] The following snippet shows how to reproduce.

from tortoise import Tortoise, fields, run_async
from tortoise.contrib.pydantic import pydantic_model_creator
from tortoise.exceptions import NoValuesFetched
from tortoise.models import Model

class Employee(Model):
    name = fields.CharField(max_length=50)

    manager: fields.ForeignKeyNullableRelation["Employee"] = fields.ForeignKeyField(
        "models.Employee", related_name="team_members", null=True
    )
    team_members: fields.ReverseRelation["Employee"]

    talks_to: fields.ManyToManyRelation["Employee"] = fields.ManyToManyField(
        "models.Employee", related_name="gets_talked_to"
    )
    gets_talked_to: fields.ManyToManyRelation["Employee"]

    def name_length(self) -> int:
        return len(self.name)

    def team_size(self) -> int:
        try:
            return len(self.team_members)
        except NoValuesFetched:
            return -1

    class PydanticMeta:
        computed = ["name_length", "team_size"]
        allow_cycles = True
        max_recursion = 4

async def run():
    await Tortoise.init(db_url="sqlite://:memory:", modules={"models": ["__main__"]})
    await Tortoise.generate_schemas()

    Employee_Pydantic = pydantic_model_creator(Employee)

    root = await Employee.create(name="Root")
    loose = await Employee.create(name="Loose")
    _1 = await Employee.create(name="1. First H1", manager=root)
    _2 = await Employee.create(name="2. Second H1", manager=root)

    await _1.talks_to.add(_2, root, loose)
    await _2.gets_talked_to.add(_1, loose)

    # The error ocurs here, after calling `from_tortoise_orm`
    p = await Employee_Pydantic.from_tortoise_orm(await Employee.get(name="Root"))
    print(p.model_dump_json(indent=4))

if __name__ == "__main__":
    run_async(run())

Expected behavior Pydantic serializes the model.

[long2ice]

Try develop source code

[plusiv]

Try develop source code

Actually, I'm already using it.

[long2ice]

Change NoValuesFetched to AttributeError

[plusiv]

I think that changing NoValuesFetched to AttributeError will bypass the real error, since team_members should be available if the relationship exists.

[waketzheng]

@plusiv The code snippet failed to run:

line 47, in run
    await _1.talks_to.add(_2, _1_1_1, loose)
NameError: name '_1_1_1' is not defined

[plusiv]

@waketzheng thanks, I've updated the example and the error message. Now it's able to reproduce.

[plusiv]

I figured out that the problem is for attributes that are not in the serialized class. Since backward relations fields are not being migrated when the new Pydantic class is created, that class is unable to find that attribute.

https://github.com/tortoise/tortoise-orm/blob/743843c0005b270f9fe91b84722822a7963e7ae9/tortoise/contrib/pydantic/creator.py#L426-L432

Actually, if you try the following code:

class Employee(Model):
    name = fields.CharField(max_length=50)

    manager: fields.ForeignKeyNullableRelation["Employee"] = fields.ForeignKeyField(
        "models.Employee", related_name="team_members", null=True
    )
    team_members: fields.ReverseRelation["Employee"]

    talks_to: fields.ManyToManyRelation["Employee"] = fields.ManyToManyField(
        "models.Employee", related_name="gets_talked_to"
    )
    gets_talked_to: fields.ManyToManyRelation["Employee"]

    def name_length(self) -> int:
        return len(self.name)

    def team_size(self) -> int:
        try:
            return 1
        except NoValuesFetched:
            return -1

    class PydanticMeta:
        computed = ["name_length", "team_size"]
        allow_cycles = True
        max_recursion = 4

async def run():
    await Tortoise.init(db_url="sqlite://:memory:", modules={"models": ["__main__"]})
    await Tortoise.generate_schemas()

    Employee_Pydantic = pydantic_model_creator(Employee)

    root = await Employee.create(name="Root")
    loose = await Employee.create(name="Loose")
    _1 = await Employee.create(name="1. First H1", manager=root)
    _2 = await Employee.create(name="2. Second H1", manager=root)
    print("Root:", root.team_members)

    await _1.talks_to.add(_2, root, loose)
    await _2.gets_talked_to.add(_1, loose)

    # The error ocurs here, after calling `from_tortoise_orm`
    p = await Employee_Pydantic.from_tortoise_orm(await Employee.get(name="Root"))
    print(p.model_dump_json(indent=4))

if __name__ == "__main__":
    run_async(run())

The operation will success. This is because self.name is an attribute already included in the Pydantic class since is an attribute that is already included in the serialization.

BUT if exclude the attribute name by setting:

    class PydanticMeta:
        computed = ["name_length", "team_size"]
        allow_cycles = True
        max_recursion = 4
        # Exclude the `name` field from the generated Pydantic model
        exclude = ("name",)

You will see the same error as above:

  File "some-path/projects/debug-tortoise-orm/.venv/lib/python3.11/site-packages/pydantic/main.py", line 353, in model_dump_json
    return self.__pydantic_serializer__.to_json(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
pydantic_core._pydantic_core.PydanticSerializationError: Error serializing to JSON: AttributeError: '__main__.Employee.leaf' object has no attribute 'name'

[plusiv]

@long2ice I think the most suitable option is to resolve computed fields before Pydantic model has been serialized, moreover, I understand that the concept being sought for computed_fields is a bit different from the already serialized Pydantic model's computed_fields. The one being sought would be more like an additional component to be added before serialization.

[gokhanmeteerturk]

Now this may sound a little bit weird, but can you try changing this: team_members: fields.ReverseRelation["Employee"] with this: team_members= fields.ReverseRelation["Employee"]

And initialize model structures before using the pydantic_model_creator:

Tortoise.init_models(["__main__"], "models") # might be different for you

This is how I fixed a very similar problem. It is terribly hacky, but saved the day.

[janusheide]

Having a similar issue, with a computed field where the behavior depends on whether a relationship is set or not, so something like this where an url (str) is generated if a relationship is set or nothing if it is not.

class Entity(Model):

  id = IntField(pk=True) 
  name = CharField(256)
  other_entity: relational.OneToOneNullableRelation[Entity] = OneToOneField(
      "Entity", related_name=False, null=True, on_delete=SET_NULL,
  )

  def entity_url(self: Entity) -> str | None:
    self.other_entity:
      return "some_url" + name
    return None

  class PydanticMeta:
      computed = ("entity_url",)

This gives an error like this: AttributeError: 'Entity' object has no attribute 'other_entity'....

When using pydantic_model_creator(...) so it seems like the field is not initialized once the computed field is created?

Anyone found a meaningfull way to work around the problem?

[waketzheng]

@janusheide Seems that it is not a bug, just use getattr instead of .attr

The following code snippet worked at my machine (MacOS+Python3.11+tortoise-orm0.20.1)

from __future__ import annotations

from tortoise import Tortoise, fields, run_async
from tortoise.contrib.pydantic import pydantic_model_creator
from tortoise.fields import SET_NULL, CharField, IntField, OneToOneField
from tortoise.models import Model

class Entity(Model):
    id = IntField(pk=True)
    name = CharField(256)
    other_entity: fields.OneToOneNullableRelation[Entity] = OneToOneField(
        "models.Entity",
        related_name=False,
        null=True,
        on_delete=SET_NULL,
    )

    def entity_url(self: Entity) -> str | None:
        if getattr(self, "other_entity", None):
            return "some_url" + self.name
        return None

    class PydanticMeta:
        computed = ("entity_url",)

async def run() -> None:
    await Tortoise.init(db_url="sqlite://:memory:", modules={"models": ["__main__"]})
    await Tortoise.generate_schemas()

    Entity_Pydantic = pydantic_model_creator(Entity)
    root = await Entity.create(name="Root")
    print(f"{root.entity_url() = }")

    loose = await Entity.create(name="Loose", other_entity=root)
    print(f"{loose.entity_url() = }")

    p = await Entity_Pydantic.from_tortoise_orm(await Entity.get(name="Root"))
    print(p.model_dump_json(indent=4))

    p2 = await Entity_Pydantic.from_tortoise_orm(await Entity.get(name="Loose"))
    print(p2.model_dump_json(indent=4))

if __name__ == "__main__":
    run_async(run())

[Abdeldjalil-H]

@waketzheng Actually it is a bug. The issue is the value of self is being set to the pydantic model itself. Your example works because you pass all needed attributes to the pydantic model, i.e self=Entity_Pydantic. You can try a computed filled that calls a method defined on the tortoise model.

[janusheide]

@waketzheng Thanks!

In my original model the relationship is also excluded configured using PydanticMeta, if I do that here the resulting schema behaves differently than the model. If however excluded is configured as an argument to the model_creator then the model and schema both behaves correct:

from tortoise import Tortoise, fields, run_async
from tortoise.contrib.pydantic import pydantic_model_creator
from tortoise.fields import SET_NULL, CharField, IntField, OneToOneField
from tortoise.models import Model

class Entity(Model):
    id = IntField(pk=True)
    name = CharField(256)
    other_entity: fields.OneToOneNullableRelation["Entity"] = OneToOneField(
        "models.Entity",
        related_name=False,
        null=True,
        on_delete=SET_NULL,
    )

    def entity_url(self) -> str | None:
        if getattr(self, "other_entity", None):
            return "some_url" + self.name
        return None

    class PydanticMeta:
        computed = ("entity_url",)
        # exclude = ("other_entity",) # <- causes last line to fail

async def run() -> None:
    await Tortoise.init(db_url="sqlite://:memory:", modules={"models": ["__main__"]})
    await Tortoise.generate_schemas()

    # Entity_Pydantic = pydantic_model_creator(Entity)
    Entity_Pydantic = pydantic_model_creator(Entity, exclude="other_entity")
    root = await Entity.create(name="Root")
    print(f"{root.entity_url() = }")
    assert root.entity_url() is None

    p = await Entity_Pydantic.from_tortoise_orm(await Entity.get(name="Root"))
    print(p.model_dump_json(indent=4))
    assert p.entity_url is None

    loose = await Entity.create(name="Loose", other_entity=root)
    print(f"{loose.entity_url() = }")
    assert loose.entity_url is not None

    p2 = await Entity_Pydantic.from_tortoise_orm(await Entity.get(name="Loose"))
    print(p2.model_dump_json(indent=4))
    assert p2.entity_url is not None

if __name__ == "__main__":
    run_async(run())

Unfortunately, this did not fix my problem, but I'll see if I can construct a standalone example that demonstrate it.

[waketzheng]

@Abdeldjalil-H OK, I get it.

@janusheide Before the bug fixed, you can do it like this:

from __future__ import annotations

from tortoise import Tortoise, fields, run_async
from tortoise.contrib.pydantic import pydantic_model_creator
from tortoise.fields import SET_NULL, CharField, IntField, OneToOneField
from tortoise.models import Model

class Entity(Model):
    id = IntField(pk=True)
    name = CharField(256)
    other_entity: fields.OneToOneNullableRelation["Entity"] = OneToOneField(
        "models.Entity",
        related_name=False,
        null=True,
        on_delete=SET_NULL,
    )

    def entity_url(self) -> str | None:
        if isinstance(self, Entity_Pydantic):
            if self._validating_obj is None:
                return None
            self = self._validating_obj
        if getattr(self, "other_entity", None):
            return "some_url" + self.name
        return None

    class PydanticMeta:
        computed = ("entity_url",)
        exclude = ("other_entity",)

class Entity_Pydantic(pydantic_model_creator(Entity)):  # type:ignore[misc]
    _validating_obj: Entity | None = None

    @classmethod
    async def from_tortoise_orm(cls, obj: Entity) -> Entity_Pydantic:
        cls._validating_obj = obj
        return await super().from_tortoise_orm(obj)

async def run() -> None:
    await Tortoise.init(db_url="sqlite://:memory:", modules={"models": ["__main__"]})
    await Tortoise.generate_schemas()

    # Entity_Pydantic = pydantic_model_creator(Entity, exclude="other_entity")
    root = await Entity.create(name="Root")
    print(f"{root.entity_url() = }")
    assert root.entity_url() is None

    p = await Entity_Pydantic.from_tortoise_orm(await Entity.get(name="Root"))
    print(p.model_dump_json(indent=4))
    assert p.entity_url is None

    loose = await Entity.create(name="Loose", other_entity=root)
    print(f"{loose.entity_url() = }")
    assert loose.entity_url is not None

    p2 = await Entity_Pydantic.from_tortoise_orm(await Entity.get(name="Loose"))
    print(p2.model_dump_json(indent=4))
    assert p2.entity_url is not None

if __name__ == "__main__":
    run_async(run())

[Aleksey512]

Since this issue has not yet been solved. I propose a temporary solution. Based on the solution from @waketzheng

monkey_patch.py

def pydantic_model_creator(
    cls: "Type[Model]",
    *,
    name=None,
    exclude: Tuple[str, ...] = (),
    include: Tuple[str, ...] = (),
    computed: Tuple[str, ...] = (),
    optional: Tuple[str, ...] = (),
    allow_cycles: Optional[bool] = None,
    sort_alphabetically: Optional[bool] = None,
    _stack: tuple = (),
    exclude_readonly: bool = False,
    meta_override: Optional[Type] = None,
    model_config: Optional[ConfigDict] = None,
    validators: Optional[Dict[str, Any]] = None,
    module: str = __name__,
) -> Type[PydanticModel]:
    pydantic_model = pmd(
        cls,
        name=name,
        exclude=exclude,
        include=include,
        computed=computed,
        optional=optional,
        allow_cycles=allow_cycles,
        sort_alphabetically=sort_alphabetically,
        exclude_readonly=exclude_readonly,
        meta_override=meta_override,
        model_config=model_config,
        validators=validators,
        module=module,
    )

    class MonkeyPatchedModel(pydantic_model):
        @classmethod
        async def from_tortoise_orm(cls, obj: "Model"):
            instance = await super().from_tortoise_orm(obj)
            setattr(instance, '_initial_obj', obj)
            return instance

    return MonkeyPatchedModel

base_model.py

def add_functionality_to_computed_method(method):
    @wraps(method)
    def _impl(self, *method_args, **method_kwargs):
        target = getattr(self, "_initial_obj", self)
        return method(target, *method_args, **method_kwargs)

    return _impl

class ComputedMethodsMeta(ModelMeta):
    def __new__(cls, name: str, bases: Tuple[Type, ...], attrs: dict):
        pydantic_meta = attrs.get("PydanticMeta")
        if pydantic_meta:
            computed_methods = getattr(pydantic_meta, "computed", [])
            for method_name in computed_methods:
                if method_name in attrs:
                    original_method = attrs[method_name]
                    attrs[method_name] = add_functionality_to_computed_method(
                        original_method
                    )
        return super().__new__(cls, name, bases, attrs)

class BaseModel(models.Model, metaclass=ComputedMethodsMeta):
    # fields

    class Meta:
        abstract = True

class Employe(BaseModel):
    other_entity: fields.OneToOneNullableRelation["Entity"] = OneToOneField(
        "models.Entity",
        related_name=False,
        null=True,
        on_delete=SET_NULL,
    )

    def entity_url(self) -> str | None:
        if getattr(self, "other_entity", None):
            return "some_url" + self.name
        return None

    class PydanticMeta:
        computed = ("entity_url",)