search

tortoise / tortoise-orm

Familiar asyncio ORM for python, built with relations in mind
https://tortoise.github.io
Apache License 2.0
4.63k stars 385 forks source link

Creating pydantic model from a DB model not working when ReverseRelation is present #1238

Open kraghuprasad opened 2 years ago

kraghuprasad commented 2 years ago

I am using Tortoise-ORM with FastAPI based application. All python files are in multiple folders under src/backend/, like src/backend/db/, src/backend/schemas/, src/backend/common etc. and my PYTHONPATH env variable is set to src/.

Following are from my DB models module:

from typing import Optional
from datetime import datetime
from pydantic import Json
from tortoise.fields import (
    IntField, DecimalField, CharField, DatetimeField, BooleanField,
    BinaryField, JSONField, ManyToManyField)
from tortoise.fields.relational import (ForeignKeyRelation, ForeignKeyField)
from tortoise.fields import RESTRICT, ReverseRelation, ManyToManyRelation
from tortoise.models import Model

class StatusCheckMixin:
    def is_active(self):
        """
        Function to check if the object is in active state.
        """
        return self.status == "A"

    def is_inactive(self):
        """
        Function to check if the object is in inactive state.
        """
        return self.status == "I"

    def is_deleted(self):
        """
        Function to check if the object is in deleted state.
        """
        return self.status == "D"

class IdentityMixin(Model, StatusCheckMixin):
    """
    Provides numerical identifier and status fields to any model.
    """
    id: int = IntField(
        pk=True, index=True, null=False,
        description="The primary identifier of this entity.")
    status: str = CharField(1, default='A', index=True,
                            description="Status of this entity: A, I, or D.")

    class Meta:
        abstract = True
        ordering = ("-id",)  # Sort in reverse order of value of attribute id

class EntityMixin(Model, StatusCheckMixin):
    """
    Entity (common entity) class for other entities to inherit from. Provides
    numerical identifier, name, and status fields to any model.
    """
    id: int = IntField(
        pk=True, index=True, null=False,
        description="The primary identifier of this entity.")
    name: str = CharField(64, index=True, null=False,
                          description="The name of this entity.")
    status: str = CharField(1, default='A', index=True,
                            description="Status of this entity: A, I, or D.")

    def __str__(self):
        return self.name

    class Meta:
        abstract = True

class CreatorMixin():
    """
    Mixin class to define creator and creation time related attributes for
    Entity type objects.
    """
    created_at: Optional[datetime] = DatetimeField(
        auto_now_add=True, null=False)
    # TODO: Need to pull logged-in user-info to populate it automatically
    # creator_id: Optional[int] = ForeignKeyField(
    #     "models.Authuser", to_field="id", null=True, on_delete=RESTRICT)

    class Meta:
        abstract = True

class LastModifierMixin():
    """
    Mixin class to define modifier and modification time related attributes for
    Entity type objects.
    """
    last_modified_at: Optional[datetime] = DatetimeField(
        null=True, index=True, auto_now=True)
    # TODO: Need to pull logged-in user-info to populate it automatically
    # last_modifier_id: Optional[int] = ForeignKeyField(
    #     "models.Authuser", to_field=id, null=True, on_delete=RESTRICT)
    # last_modifier: Optional["Authuser"] = Relationship()

    class Meta:
        abstract = True

class RemarksMixin():
    """
    Mixin class to define remarks field for an Entity type object.
    """
    remarks: Optional[str] = CharField(
        256, default=None, index=False, null=True)

    class Meta:
        abstract = True

class Authuser(EntityMixin, CreatorMixin, LastModifierMixin):
    """
    Authuser data model.
    """
    login_name: Optional[str] = CharField(
        80, description="Login identifier string.", unique=True, null=False)
    email_address: Optional[str] = CharField(
        80, description="E-mail address of the user.", unique=True, null=False)
    password: Optional[str] = CharField(
        512, description="Password associated with the user.", null=False)
    is_superuser: bool = BooleanField(
        description="Indicates if the user is super-user.", null=False)
    user_realms: ReverseRelation["AuthuserRealm"]

    class Meta:
        table = "authusers"
        table_description = "Stores records of users who can authenticate."
        ordering = ["name", "created_at"]

class Realm(EntityMixin, CreatorMixin, LastModifierMixin, RemarksMixin):
    """
    Class to represent a realm in a multi-tenant application.
    """
    code: Optional[str] = CharField(
        48, description="Realm Code", unique=True, null=False)
    support_pin: Optional[str] = CharField(
        8, index=True, null=True,
        description="PIN to authenticate with support staff.")
    website_url: Optional[str] = CharField(
        80, default=None, description="Website URL", null=True)
    logo_url: Optional[str] = CharField(
        80, default=None, description="Logo URL", null=True)

    class Meta:
        table = 'realms'
        table_description = "Stores records of realms."
        ordering = ["name"]
        unique_together = (("code", "support_pin"), )
        indexes = (("code", "support_pin"), )

class AuthuserRealm(IdentityMixin, RemarksMixin):
    """
    The association class to hold relationship between a user and a realm.
    """
    authuser: ForeignKeyRelation[Authuser] = ForeignKeyField(
        "models.Authuser", related_name="user_realms", on_delete=RESTRICT,
        to_field="id",
        description="The authuser who is mapped to one or more realms.")
    realm: ForeignKeyRelation[Realm] = ForeignKeyField(
        "models.Realm", related_name="user_realms", on_delete=RESTRICT,
        to_field="id",
        description="The associated realm.")
    profile_image_path: Optional[str] = CharField(
        max_length=128, index=True, null=True,
        description="The relative path of profile image.")
    contact_number: Optional[str] = CharField(
        max_length=16, index=True, null=True,
        description="The contact number associated with this mapping.")
    registered_at: Optional[datetime] = DatetimeField(
        auto_now_add=True, description="Timestamp when this mapping was made.")

    async def name(self) -> str:
        return f"{self.authuser.name or ''} ({self.realm.code or ''})".strip()

    class Meta:
        table = 'authuser_realm'
        table_description = "Stores records of authuser-realm mappings."
        ordering = ["id"]
        unique_together = (("authuser_id", "realm_id"), )
        indexes = (("authuser_id", "realm_id", "contact_number", "status"), )

Following are from my schema models module:

from pydantic.main import ModelMetaclass, BaseModel
from tortoise.contrib.pydantic.base import PydanticModel
from typing import Any, Dict, Optional, Tuple
from tortoise.contrib.pydantic import pydantic_model_creator
from datetime import datetime
from tortoise import Tortoise

from backend.db import models as m

Tortoise.init_models(["backend.db.models"], "models")

class AllOptional(ModelMetaclass):
    """
    Fix to enable all optional fields in models used in PUT and PATCH
    HTTP methods.
    """
    def __new__(mcs, name: str, bases: Tuple[type],
                namespaces: Dict[str, Any], **kwargs):
        annotations: dict = namespaces.get('__annotations__', {})
        for base in bases:
            for base_ in base.__mro__:
                if base_ is BaseModel or base_ is PydanticModel:
                    break
                annotations.update(base_.__annotations__)

        for field, value in annotations.items():
            if not field.startswith('__'):
                annotations[field] = Optional[value]
        namespaces['__annotations__'] = annotations
        return super().__new__(mcs, name, bases, namespaces, **kwargs)

AuthuserIn = pydantic_model_creator(
    m.Authuser, name="AuthuserIn", include=(
        "name", "login_name", "email_address", "password", "is_superuser",
        "status"))
AuthuserOut = pydantic_model_creator(
    m.Authuser, name="AuthuserOut", exclude=("password", "user_realms"),
    exclude_readonly=False)
AuthuserMod_ = pydantic_model_creator(
    m.Authuser, name="AuthuserMod_", include=(
        "name", "email_address", "password", "is_superuser", "status"))

class AuthuserMod(AuthuserMod_, metaclass=AllOptional):
    pass

RealmIn = pydantic_model_creator(
    m.Realm, name="RealmIn", include=(
        "name", "code", "status", "logo_url", "website_url", "remarks"))
RealmOut = pydantic_model_creator(
    m.Realm, name="RealmOut", exclude=("support_pin", "user_realms"))
RealmMod_ = pydantic_model_creator(
    m.Realm, name="RealmMod_", exclude=(
        "id", "code", "support_pin", "creator_id", "created_at",
        "last_modifier_id", "last_modified_at", "support_pin"))

class RealmMod(RealmMod_, metaclass=AllOptional):
    pass

class AuthuserRealmOut(BaseModel):
    id: int
    authuser_id: int
    realm_id: int
    registered_at: Optional[datetime]
    profile_image_path: Optional[str]
    contact_number: Optional[str]
    remarks: Optional[str]
    status: Optional[str]

    class Config:
        orm_mode = True

AuthuserRealmMod_ = pydantic_model_creator(
    m.AuthuserRealm, name="AuthuserRealmMod_", include=(
        "profile_image_path", "contact_number", "remarks", "status"))

class AuthuserRealmMod(AuthuserRealmMod_, metaclass=AllOptional):
    pass

I am trying to create an API end point for /realms/ for HTTP POST request in backend/routers/realms.py and following is part of it:

from typing import List
from datetime import datetime, timezone
from fastapi import APIRouter, Query, HTTPException  # , Depends

from ..schemas.models import RealmIn, RealmOut, RealmMod
from ..core.common import get_logger
from ..db import models as m

router = APIRouter(
    prefix='/realms',
    tags=['Realms'],
    responses={404: {"description": "Realm not found."}}
)

@router.post("/", response_model=RealmOut)
async def create_realm(
        realm_in: RealmIn) -> RealmOut:
    """
    Create a new realm from supplied RealmIn object, persist it in DB,
    and then return its Out model along with its id.
    """
    attrs = realm_in.dict()
    realm = await m.Realm.create(**attrs)
    realmout = await RealmOut.from_orm(realm)
    return realmout

I invoked a POST request to this end point with an equivalent command:

curl -X 'POST' \
  'https://my.server:8000/realms/' \
  -H 'accept: application/json' \
  -H 'Content-Type: application/json' \
  -d '{
  "name": "Mina Hotels",
  "status": "A",
  "remarks": "Mina Hotels is a Three Star hotel.",
  "code": "mina",
  "website_url": "http://mina.example.in",
  "logo_url": "http://mina.example.in/logo.jpg"
}'

I am getting following exception, when pydantic model's from_orm() method is invoked. I was expecting that the RealmOut model would have been received as response.

...
<snipped>

INFO:     Application startup complete.
Application startup complete.
INFO:     Uvicorn running on http://my.server:8000 (Press CTRL+C to quit)
Uvicorn running on http://my.server:8000 (Press CTRL+C to quit)
INSERT INTO "realms" ("name","status","created_at","last_modified_at","remarks","code","support_pin","website_url","logo_url") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) RETURNING "id": ['Mina Hotels', 'A', datetime.datetime(2022, 8, 31, 11, 13, 45, 322861, tzinfo=<UTC>), datetime.datetime(2022, 8, 31, 11, 13, 45, 322997, tzinfo=<UTC>), 'Mina Hotels is a Three Start hotel.', 'mina', None, 'http://mina.example.in', 'http://images.mina.example.in/logo.jpg']
INFO:     172.18.0.2:57232 - "POST /realms/ HTTP/1.1" 500 Internal Server Error
ERROR:    Exception in ASGI application
Traceback (most recent call last):
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/anyio/streams/memory.py", line 94, in receive
    return self.receive_nowait()
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/anyio/streams/memory.py", line 89, in receive_nowait
    raise WouldBlock
anyio.WouldBlock

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/starlette/middleware/base.py", line 43, in call_next
    message = await recv_stream.receive()
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/anyio/streams/memory.py", line 114, in receive
    raise EndOfStream
anyio.EndOfStream

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/uvicorn/protocols/http/h11_impl.py", line 366, in run_asgi
    result = await app(self.scope, self.receive, self.send)
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/uvicorn/middleware/proxy_headers.py", line 75, in __call__
    return await self.app(scope, receive, send)
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/fastapi/applications.py", line 269, in __call__
    await super().__call__(scope, receive, send)
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/starlette/applications.py", line 124, in __call__
    await self.middleware_stack(scope, receive, send)
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/starlette/middleware/errors.py", line 184, in __call__
    raise exc
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/starlette/middleware/errors.py", line 162, in __call__
    await self.app(scope, receive, _send)
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/starlette/middleware/base.py", line 68, in __call__
    response = await self.dispatch_func(request, call_next)
  File "/config/workspace/my_project/src/backend/main.py", line 73, in set_structlog_contextvars
    response = await call_next(request)
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/starlette/middleware/base.py", line 46, in call_next
    raise app_exc
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/starlette/middleware/base.py", line 36, in coro
    await self.app(scope, request.receive, send_stream.send)
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/starlette/middleware/gzip.py", line 24, in __call__
    await responder(scope, receive, send)
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/starlette/middleware/gzip.py", line 43, in __call__
    await self.app(scope, receive, self.send_with_gzip)
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/starlette/middleware/cors.py", line 92, in __call__
    await self.simple_response(scope, receive, send, request_headers=headers)
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/starlette/middleware/cors.py", line 147, in simple_response
    await self.app(scope, receive, send)
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/starlette/exceptions.py", line 93, in __call__
    raise exc
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/starlette/exceptions.py", line 82, in __call__
    await self.app(scope, receive, sender)
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/fastapi/middleware/asyncexitstack.py", line 21, in __call__
    raise e
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/fastapi/middleware/asyncexitstack.py", line 18, in __call__
    await self.app(scope, receive, send)
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/starlette/routing.py", line 670, in __call__
    await route.handle(scope, receive, send)
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/starlette/routing.py", line 266, in handle
    await self.app(scope, receive, send)
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/starlette/routing.py", line 65, in app
    response = await func(request)
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/fastapi/routing.py", line 227, in app
    raw_response = await run_endpoint_function(
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/fastapi/routing.py", line 160, in run_endpoint_function
    return await dependant.call(**values)
  File "/config/workspace/my_project/src/backend/routers/realms.py", line 88, in create_realm
    realmout = await RealmOut.from_orm(realm)
  File "pydantic/main.py", line 562, in pydantic.main.BaseModel.from_orm
  File "pydantic/main.py", line 1022, in pydantic.main.validate_model
  File "pydantic/fields.py", line 837, in pydantic.fields.ModelField.validate
  File "pydantic/fields.py", line 1118, in pydantic.fields.ModelField._apply_validators
  File "pydantic/class_validators.py", line 280, in pydantic.class_validators._generic_validator_cls.lambda3
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/tortoise/contrib/pydantic/base.py", line 59, in _tortoise_convert
    return list(value)
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/tortoise/fields/relational.py", line 104, in __len__
    self._raise_if_not_fetched()
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/tortoise/fields/relational.py", line 163, in _raise_if_not_fetched
    raise NoValuesFetched(
tortoise.exceptions.NoValuesFetched: No values were fetched for this relation, first use .fetch_related()
Exception in ASGI application
Traceback (most recent call last):
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/anyio/streams/memory.py", line 94, in receive
    return self.receive_nowait()
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/anyio/streams/memory.py", line 89, in receive_nowait
    raise WouldBlock
anyio.WouldBlock

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/starlette/middleware/base.py", line 43, in call_next
    message = await recv_stream.receive()
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/anyio/streams/memory.py", line 114, in receive
    raise EndOfStream
anyio.EndOfStream

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/uvicorn/protocols/http/h11_impl.py", line 366, in run_asgi
    result = await app(self.scope, self.receive, self.send)
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/uvicorn/middleware/proxy_headers.py", line 75, in __call__
    return await self.app(scope, receive, send)
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/fastapi/applications.py", line 269, in __call__
    await super().__call__(scope, receive, send)
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/starlette/applications.py", line 124, in __call__
    await self.middleware_stack(scope, receive, send)
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/starlette/middleware/errors.py", line 184, in __call__
    raise exc
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/starlette/middleware/errors.py", line 162, in __call__
    await self.app(scope, receive, _send)
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/starlette/middleware/base.py", line 68, in __call__
    response = await self.dispatch_func(request, call_next)
  File "/config/workspace/my_project/src/backend/main.py", line 73, in set_structlog_contextvars
    response = await call_next(request)
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/starlette/middleware/base.py", line 46, in call_next
    raise app_exc
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/starlette/middleware/base.py", line 36, in coro
    await self.app(scope, request.receive, send_stream.send)
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/starlette/middleware/gzip.py", line 24, in __call__
    await responder(scope, receive, send)
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/starlette/middleware/gzip.py", line 43, in __call__
    await self.app(scope, receive, self.send_with_gzip)
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/starlette/middleware/cors.py", line 92, in __call__
    await self.simple_response(scope, receive, send, request_headers=headers)
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/starlette/middleware/cors.py", line 147, in simple_response
    await self.app(scope, receive, send)
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/starlette/exceptions.py", line 93, in __call__
    raise exc
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/starlette/exceptions.py", line 82, in __call__
    await self.app(scope, receive, sender)
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/fastapi/middleware/asyncexitstack.py", line 21, in __call__
    raise e
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/fastapi/middleware/asyncexitstack.py", line 18, in __call__
    await self.app(scope, receive, send)
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/starlette/routing.py", line 670, in __call__
    await route.handle(scope, receive, send)
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/starlette/routing.py", line 266, in handle
    await self.app(scope, receive, send)
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/starlette/routing.py", line 65, in app
    response = await func(request)
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/fastapi/routing.py", line 227, in app
    raw_response = await run_endpoint_function(
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/fastapi/routing.py", line 160, in run_endpoint_function
    return await dependant.call(**values)
  File "/config/workspace/my_project/src/backend/routers/realms.py", line 88, in create_realm
    realmout = await RealmOut.from_orm(realm)
  File "pydantic/main.py", line 562, in pydantic.main.BaseModel.from_orm
  File "pydantic/main.py", line 1022, in pydantic.main.validate_model
  File "pydantic/fields.py", line 837, in pydantic.fields.ModelField.validate
  File "pydantic/fields.py", line 1118, in pydantic.fields.ModelField._apply_validators
  File "pydantic/class_validators.py", line 280, in pydantic.class_validators._generic_validator_cls.lambda3
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/tortoise/contrib/pydantic/base.py", line 59, in _tortoise_convert
    return list(value)
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/tortoise/fields/relational.py", line 104, in __len__
    self._raise_if_not_fetched()
  File "/config/workspace/.virtualenvs/py-3.9/lib/python3.9/site-packages/tortoise/fields/relational.py", line 163, in _raise_if_not_fetched
    raise NoValuesFetched(
tortoise.exceptions.NoValuesFetched: No values were fetched for this relation, first use .fetch_related()

Trying out solutions given for similar errors, like invoking fetch_related() on realm object has not fixed the issue. I am not sure if it is a bug in pydantic model creator code of Tortoise, or I have been doing something incorrectly. I have tried to fix the issues as suggestted below, but the error is still coming up:

https://stackoverflow.com/questions/63754830/python-tortoise-orm-use-related-model-field-in-str

https://github.com/tortoise/tortoise-orm/issues/1069

I am using tortoise-orm 0.19.0 with fastapi 0.78.0.

Hillsir commented 1 year ago

I'm using pydantic_model_creator(include=(fk_field)) and I'm also getting an error, KeyError: fk_field_id No found