Error in handling class - can't handle type None

[rjurney]

@ffinfo This schema won't validate...

class Movie(NodeBase):
    """A film node in hollywood."""

    entity_type: str = "movie"
    genre: str
    title: str
    year: str
    length: typing.Optional[typing.Union[str, int]]
    gross: typing.Optional[int]
    rating: str

    @validator("length")
    def convert_hours_minutes_to_int_minutes(cls, x):
        if x and isinstance(x, str):
            x = text_runtime_to_minutes(x)
        return x

In [5]: Movie.spark_schema()
---------------------------------------------------------------------------
NotImplementedError                       Traceback (most recent call last)
Input In [5], in <cell line: 1>()
----> 1 Movie.spark_schema()

File ~/opt/anaconda3/envs/graphlet/lib/python3.10/site-packages/pydantic_spark/base.py:13, in SparkBase.spark_schema(cls)
     11 """Return the avro schema for the pydantic class"""
     12 schema = cls.schema()
---> 13 return cls._spark_schema(schema)

File ~/opt/anaconda3/envs/graphlet/lib/python3.10/site-packages/pydantic_spark/base.py:109, in SparkBase._spark_schema(schema)
    106         fields.append(struct_field)
    107     return fields
--> 109 fields = get_fields(schema)
    111 return {"fields": fields, "type": "struct"}

File ~/opt/anaconda3/envs/graphlet/lib/python3.10/site-packages/pydantic_spark/base.py:97, in SparkBase._spark_schema.<locals>.get_fields(s)
     95 required = s.get("required", [])
     96 for key, value in s.get("properties", {}).items():
---> 97     spark_type, metadata = get_type(value)
     98     metadata["parentClass"] = s.get("title")
     99     struct_field = {
    100         "name": key,
    101         "nullable": "default" not in metadata and key not in required,
    102         "metadata": metadata,
    103         "type": spark_type,
    104     }

File ~/opt/anaconda3/envs/graphlet/lib/python3.10/site-packages/pydantic_spark/base.py:85, in SparkBase._spark_schema.<locals>.get_type(value)
     83     spark_type = {"keyType": "string", "type": "map", "valueContainsNull": True, "valueType": value_type}
     84 else:
---> 85     raise NotImplementedError(
     86         f"Type '{t}' not support yet, "
     87         f"please report this at https://github.com/godatadriven/pydantic-avro/issues"
     88     )
     89 return spark_type, metadata

NotImplementedError: Type 'None' not support yet, please report this at https://github.com/godatadriven/pydantic-avro/issues

[dan1elt0m]

fixed in v1