TF 2.12 (and Python 3.11) support

[jonas-eschle]

Hi there, I was wondering what's the status of supporting TensorFlow version 2.12 and Python 3.11?

This currently fails with TF 2.12 installed (but setting PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python as suggested works afai see). It seems like kernels just have to be recompiled if I get that right? I.e. a new version with the new protobuf library, and that's it?

Details

``` In [1]: import tf_quant_finance --------------------------------------------------------------------------- TypeError Traceback (most recent call last) Cell In[1], line 1 ----> 1 import tf_quant_finance File ~/mambaforge/envs/tmp310/lib/python3.10/site-packages/tf_quant_finance/__init__.py:73 71 from tf_quant_finance import black_scholes 72 from tf_quant_finance import datetime ---> 73 from tf_quant_finance import experimental 74 from tf_quant_finance import math 75 from tf_quant_finance import models File ~/mambaforge/envs/tmp310/lib/python3.10/site-packages/tf_quant_finance/experimental/__init__.py:17 14 """Experimental modules.""" 16 from tf_quant_finance.experimental import instruments ---> 17 from tf_quant_finance.experimental import local_stochastic_volatility 18 from tf_quant_finance.experimental import local_volatility 19 from tf_quant_finance.experimental import lsm_algorithm File ~/mambaforge/envs/tmp310/lib/python3.10/site-packages/tf_quant_finance/experimental/local_stochastic_volatility/__init__.py:16 1 # Copyright 2020 Google LLC 2 # 3 # Licensed under the Apache License, Version 2.0 (the "License"); (...) 12 # See the License for the specific language governing permissions and 13 # limitations under the License. 14 """Local volatility model.""" ---> 16 from tf_quant_finance.experimental.local_stochastic_volatility.local_stochastic_volatility_model import LocalStochasticVolatilityModel 17 from tf_quant_finance.experimental.local_stochastic_volatility.lsv_variance_model import LSVVarianceModel 19 from tensorflow.python.util.all_util import remove_undocumented # pylint: disable=g-direct-tensorflow-import File ~/mambaforge/envs/tmp310/lib/python3.10/site-packages/tf_quant_finance/experimental/local_stochastic_volatility/local_stochastic_volatility_model.py:25 23 from tf_quant_finance import math 24 from tf_quant_finance import utils as tff_utils ---> 25 from tf_quant_finance.experimental import local_volatility as lvm 26 from tf_quant_finance.experimental.pricing_platform.framework.market_data import utils 27 from tf_quant_finance.math import pde File ~/mambaforge/envs/tmp310/lib/python3.10/site-packages/tf_quant_finance/experimental/local_volatility/__init__.py:17 1 # Copyright 2020 Google LLC 2 # 3 # Licensed under the Apache License, Version 2.0 (the "License"); (...) 12 # See the License for the specific language governing permissions and 13 # limitations under the License. 15 """Local volatility model.""" ---> 17 from tf_quant_finance.experimental.local_volatility.local_volatility_model import LocalVolatilityModel 19 from tensorflow.python.util.all_util import remove_undocumented # pylint: disable=g-direct-tensorflow-import 21 _allowed_symbols = [ 22 "LocalVolatilityModel", 23 ] File ~/mambaforge/envs/tmp310/lib/python3.10/site-packages/tf_quant_finance/experimental/local_volatility/local_volatility_model.py:26 24 from tf_quant_finance import math 25 from tf_quant_finance import types ---> 26 from tf_quant_finance.experimental.pricing_platform.framework.market_data import volatility_surface 27 from tf_quant_finance.math import interpolation 28 from tf_quant_finance.math import piecewise File ~/mambaforge/envs/tmp310/lib/python3.10/site-packages/tf_quant_finance/experimental/pricing_platform/__init__.py:16 1 # Copyright 2020 Google LLC 2 # 3 # Licensed under the Apache License, Version 2.0 (the "License"); (...) 12 # See the License for the specific language governing permissions and 13 # limitations under the License. 14 """Pricing platform module.""" ---> 16 from tf_quant_finance.experimental.pricing_platform import framework 17 from tf_quant_finance.experimental.pricing_platform import instrument_protos 19 from tensorflow.python.util.all_util import remove_undocumented # pylint: disable=g-direct-tensorflow-import File ~/mambaforge/envs/tmp310/lib/python3.10/site-packages/tf_quant_finance/experimental/pricing_platform/framework/__init__.py:16 1 # Copyright 2020 Google LLC 2 # 3 # Licensed under the Apache License, Version 2.0 (the "License"); (...) 12 # See the License for the specific language governing permissions and 13 # limitations under the License. 14 """Framework module.""" ---> 16 from tf_quant_finance.experimental.pricing_platform.framework import core 17 from tf_quant_finance.experimental.pricing_platform.framework import equity_instruments 18 from tf_quant_finance.experimental.pricing_platform.framework import market_data File ~/mambaforge/envs/tmp310/lib/python3.10/site-packages/tf_quant_finance/experimental/pricing_platform/framework/core/__init__.py:16 1 # Copyright 2020 Google LLC 2 # 3 # Licensed under the Apache License, Version 2.0 (the "License"); (...) 12 # See the License for the specific language governing permissions and 13 # limitations under the License. 14 """Pricing platform module.""" ---> 16 from tf_quant_finance.experimental.pricing_platform.framework.core import business_days 17 from tf_quant_finance.experimental.pricing_platform.framework.core import currencies 18 from tf_quant_finance.experimental.pricing_platform.framework.core import curve_types File ~/mambaforge/envs/tmp310/lib/python3.10/site-packages/tf_quant_finance/experimental/pricing_platform/framework/core/business_days.py:17 14 """Supported bank holidays and business day conventions.""" 16 import enum ---> 17 from tf_quant_finance.experimental.pricing_platform.instrument_protos import business_days_pb2 19 __all__ = ["BusinessDayConvention", 20 "convention_from_proto_value", 21 "BankHolidays", 22 "holiday_from_proto_value"] 25 BusinessDayConvention = enum.Enum( 26 "BusinessDayConvention", 27 zip(business_days_pb2.BusinessDayConvention.keys(), 28 business_days_pb2.BusinessDayConvention.keys())) File ~/mambaforge/envs/tmp310/lib/python3.10/site-packages/tf_quant_finance/experimental/pricing_platform/instrument_protos/__init__.py:16 1 # Copyright 2020 Google LLC 2 # 3 # Licensed under the Apache License, Version 2.0 (the "License"); (...) 12 # See the License for the specific language governing permissions and 13 # limitations under the License. 14 """Instrument protos module.""" ---> 16 from tf_quant_finance.experimental.pricing_platform.instrument_protos import all_instruments_pb2 as instruments 17 from tf_quant_finance.experimental.pricing_platform.instrument_protos import american_equity_option_pb2 as american_equity_option 18 from tf_quant_finance.experimental.pricing_platform.instrument_protos import business_days_pb2 as business_days File ~/mambaforge/envs/tmp310/lib/python3.10/site-packages/tf_quant_finance/experimental/pricing_platform/instrument_protos/all_instruments_pb2.py:14 9 # @@protoc_insertion_point(imports) 11 _sym_db = _symbol_database.Default() ---> 14 from tf_quant_finance.experimental.pricing_platform.instrument_protos import american_equity_option_pb2 as tf__quant__finance_dot_experimental_dot_pricing__platform_dot_instrument__protos_dot_american__equity__option__pb2 15 from tf_quant_finance.experimental.pricing_platform.instrument_protos import interest_rate_swap_pb2 as tf__quant__finance_dot_experimental_dot_pricing__platform_dot_instrument__protos_dot_interest__rate__swap__pb2 16 from tf_quant_finance.experimental.pricing_platform.instrument_protos import forward_rate_agreement_pb2 as tf__quant__finance_dot_experimental_dot_pricing__platform_dot_instrument__protos_dot_forward__rate__agreement__pb2 File ~/mambaforge/envs/tmp310/lib/python3.10/site-packages/tf_quant_finance/experimental/pricing_platform/instrument_protos/american_equity_option_pb2.py:14 9 # @@protoc_insertion_point(imports) 11 _sym_db = _symbol_database.Default() ---> 14 from tf_quant_finance.experimental.pricing_platform.instrument_protos import business_days_pb2 as tf__quant__finance_dot_experimental_dot_pricing__platform_dot_instrument__protos_dot_business__days__pb2 15 from tf_quant_finance.experimental.pricing_platform.instrument_protos import currencies_pb2 as tf__quant__finance_dot_experimental_dot_pricing__platform_dot_instrument__protos_dot_currencies__pb2 16 from tf_quant_finance.experimental.pricing_platform.instrument_protos import date_pb2 as tf__quant__finance_dot_experimental_dot_pricing__platform_dot_instrument__protos_dot_date__pb2 File ~/mambaforge/envs/tmp310/lib/python3.10/site-packages/tf_quant_finance/experimental/pricing_platform/instrument_protos/business_days_pb2.py:33 12 _sym_db = _symbol_database.Default() 17 DESCRIPTOR = _descriptor.FileDescriptor( 18 name='tf_quant_finance/experimental/pricing_platform/instrument_protos/business_days.proto', 19 package='tf_quant_finance.experimental.pricing_platform.instrument_protos', (...) 23 serialized_pb=b'\nTtf_quant_finance/experimental/pricing_platform/instrument_protos/business_days.proto\x12@tf_quant_finance.experimental.pricing_platform.instrument_protos*\xd7\x01\n\x15\x42usinessDayConvention\x12#\n\x1f\x42USINESS_DAY_CONVENTION_UNKNOWN\x10\x00\x12\x11\n\rNO_ADJUSTMENT\x10\x01\x12\x0c\n\x08PREVIOUS\x10\x02\x12\r\n\tFOLLOWING\x10\x03\x12\x15\n\x11MODIFIED_PREVIOUS\x10\x04\x12\x16\n\x12MODIFIED_FOLLOWING\x10\x05\x12\x15\n\x11\x45OM_NO_ADJUSTMENT\x10\x06\x12\x10\n\x0c\x45OM_PREVIOUS\x10\x07\x12\x11\n\rEOM_FOLLOWING\x10\x08*9\n\x0c\x42\x61nkHolidays\x12\x19\n\x15\x42\x41NK_HOLIDAYS_UNKNOWN\x10\x00\x12\x06\n\x02US\x10\x01\x12\x06\n\x02UK\x10\x02\x42\x02P\x01\x62\x06proto3' 24 ) 26 _BUSINESSDAYCONVENTION = _descriptor.EnumDescriptor( 27 name='BusinessDayConvention', 28 full_name='tf_quant_finance.experimental.pricing_platform.instrument_protos.BusinessDayConvention', 29 filename=None, 30 file=DESCRIPTOR, 31 create_key=_descriptor._internal_create_key, 32 values=[ ---> 33 _descriptor.EnumValueDescriptor( 34 name='BUSINESS_DAY_CONVENTION_UNKNOWN', index=0, number=0, 35 serialized_options=None, 36 type=None, 37 create_key=_descriptor._internal_create_key), 38 _descriptor.EnumValueDescriptor( 39 name='NO_ADJUSTMENT', index=1, number=1, 40 serialized_options=None, 41 type=None, 42 create_key=_descriptor._internal_create_key), 43 _descriptor.EnumValueDescriptor( 44 name='PREVIOUS', index=2, number=2, 45 serialized_options=None, 46 type=None, 47 create_key=_descriptor._internal_create_key), 48 _descriptor.EnumValueDescriptor( 49 name='FOLLOWING', index=3, number=3, 50 serialized_options=None, 51 type=None, 52 create_key=_descriptor._internal_create_key), 53 _descriptor.EnumValueDescriptor( 54 name='MODIFIED_PREVIOUS', index=4, number=4, 55 serialized_options=None, 56 type=None, 57 create_key=_descriptor._internal_create_key), 58 _descriptor.EnumValueDescriptor( 59 name='MODIFIED_FOLLOWING', index=5, number=5, 60 serialized_options=None, 61 type=None, 62 create_key=_descriptor._internal_create_key), 63 _descriptor.EnumValueDescriptor( 64 name='EOM_NO_ADJUSTMENT', index=6, number=6, 65 serialized_options=None, 66 type=None, 67 create_key=_descriptor._internal_create_key), 68 _descriptor.EnumValueDescriptor( 69 name='EOM_PREVIOUS', index=7, number=7, 70 serialized_options=None, 71 type=None, 72 create_key=_descriptor._internal_create_key), 73 _descriptor.EnumValueDescriptor( 74 name='EOM_FOLLOWING', index=8, number=8, 75 serialized_options=None, 76 type=None, 77 create_key=_descriptor._internal_create_key), 78 ], 79 containing_type=None, 80 serialized_options=None, 81 serialized_start=155, 82 serialized_end=370, 83 ) 84 _sym_db.RegisterEnumDescriptor(_BUSINESSDAYCONVENTION) 86 BusinessDayConvention = enum_type_wrapper.EnumTypeWrapper(_BUSINESSDAYCONVENTION) File ~/mambaforge/envs/tmp310/lib/python3.10/site-packages/google/protobuf/descriptor.py:796, in EnumValueDescriptor.__new__(cls, name, index, number, type, options, serialized_options, create_key) 793 def __new__(cls, name, index, number, 794 type=None, # pylint: disable=redefined-builtin 795 options=None, serialized_options=None, create_key=None): --> 796 _message.Message._CheckCalledFromGeneratedFile() 797 # There is no way we can build a complete EnumValueDescriptor with the 798 # given parameters (the name of the Enum is not known, for example). 799 # Fortunately generated files just pass it to the EnumDescriptor() 800 # constructor, which will ignore it, so returning None is good enough. 801 return None TypeError: Descriptors cannot not be created directly. If this call came from a _pb2.py file, your generated code is out of date and must be regenerated with protoc >= 3.19.0. If you cannot immediately regenerate your protos, some other possible workarounds are: 1. Downgrade the protobuf package to 3.20.x or lower. 2. Set PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python (but this will use pure-Python parsing and will be much slower). More information: https://developers.google.com/protocol-buffers/docs/news/2022-05-06#python-updates ```

[jonas-eschle]

@perezju I was wondering what are the plans for this? Is this on the timeline? Or anything we can do about it?

[YuMan-Tam]

Encountered the same error with newer python version. @jonas-eschle's method works.