search

confluentinc / confluent-kafka-dotnet

Confluent's Apache Kafka .NET client
https://github.com/confluentinc/confluent-kafka-dotnet/wiki
Apache License 2.0
49 stars 857 forks source link

JsonConvert.DefaultSettings set by application code can cause serialization to fail. Use JsonSerializer.Create() when calling JObject.Parse(responseJson).ToObject<T>() #1926

Open fridayhut opened 1 year ago

fridayhut commented 1 year ago

Description

nuget package Confluent.SchemaRegistry Version="1.9.3" Operating system: Macos 13 M1

Currently theinternal class RestService RestService calls JObject.Parse T t = JObject.Parse(responseJson).ToObject<T>(); using default settings that can be specified by the application code.

If an application has settings that aren't compatible than this can cause the schema serialization to fail due to different settings being used by the calling application.

The fix is to pass JsonSerializer.Create() to the ToObject() method.

Newtonsoft Json docs

I have created a PR for this here

How to reproduce

Try to retrieve an avro schema by calling the method CachedSchemaRegistryClient.GetSchemaIdAsync(...)

Checklist

Please provide the following information:

mhowlett commented 1 year ago

i think this looks right.

out of interest, can you provide a specific example of this failing?

fridayhut commented 1 year ago

@mhowlett

Hi,

Here is the code for a reproduction of the problem: This is a .net core 6 console app Program.cs file

using Newtonsoft.Json;
using Newtonsoft.Json.Linq;

Console.WriteLine("*** Bug Example ***");

// our application has this Json Serializer Setting among many others this is done for historical reasons 
JsonConvert.DefaultSettings = () => new JsonSerializerSettings()
{
    ConstructorHandling = ConstructorHandling.AllowNonPublicDefaultConstructor
};

try
{
    var exampleUsingGlobalSettings = JObject.Parse("{\n    \"subject\": \"campaigns.Campaign-value\",\n    \"version\": 8,\n    \"id\": 1491,\n    \"references\": [\n        {\n            \"name\": \"com.rokt.campaigns.campaign.CampaignDomain\",\n            \"subject\": \"campaigns.campaign.CampaignDomain-value\",\n            \"version\": 4\n        },\n        {\n            \"name\": \"com.rokt.campaigns.campaign.CampaignOutcomeOptimiser\",\n            \"subject\": \"campaigns.campaign.CampaignOutcomeOptimiser-value\",\n            \"version\": 6\n        },\n        {\n            \"name\": \"com.rokt.campaigns.campaign.CampaignSecondaryProductCategory\",\n            \"subject\": \"campaigns.campaign.CampaignSecondaryProductCategory-value\",\n            \"version\": 4\n        },\n        {\n            \"name\": \"com.rokt.campaigns.campaign.CampaignTermsAndConditions\",\n            \"subject\": \"campaigns.campaign.CampaignTermsAndConditions-value\",\n            \"version\": 4\n        }\n    ],\n    \"schema\": \"{\\\"type\\\":\\\"record\\\",\\\"name\\\":\\\"Campaign\\\",\\\"namespace\\\":\\\"com.rokt.campaigns.campaign\\\",\\\"doc\\\":\\\"avro schema for the campaigns data change event com.rokt.campaigns.campaign.Campaign\\\",\\\"fields\\\":[{\\\"name\\\":\\\"accountId\\\",\\\"type\\\":\\\"long\\\"},{\\\"name\\\":\\\"advertiserReportEnabled\\\",\\\"type\\\":[\\\"null\\\",\\\"boolean\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"advertiserReportRecipients\\\",\\\"type\\\":[\\\"null\\\",\\\"string\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"allowAllCountries\\\",\\\"type\\\":\\\"boolean\\\",\\\"default\\\":false},{\\\"name\\\":\\\"allowedIpAddress\\\",\\\"type\\\":[\\\"null\\\",\\\"string\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"appleStoreUrl\\\",\\\"type\\\":[\\\"null\\\",\\\"string\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"approvalTaskId\\\",\\\"type\\\":[\\\"null\\\",\\\"long\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"askForMobileNumber\\\",\\\"type\\\":[\\\"null\\\",\\\"boolean\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"atsCompliant\\\",\\\"type\\\":\\\"boolean\\\",\\\"default\\\":false},{\\\"name\\\":\\\"businessManagerId\\\",\\\"type\\\":[\\\"null\\\",\\\"long\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"calendarAccountCode\\\",\\\"type\\\":[\\\"null\\\",\\\"string\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"calendarCode\\\",\\\"type\\\":[\\\"null\\\",\\\"string\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"calendarEventProvider\\\",\\\"type\\\":[\\\"null\\\",\\\"string\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"calendarId\\\",\\\"type\\\":[\\\"null\\\",\\\"string\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"calendarStartPage\\\",\\\"type\\\":[\\\"null\\\",{\\\"type\\\":\\\"enum\\\",\\\"name\\\":\\\"CalendarCampaignStartPage\\\",\\\"symbols\\\":[\\\"NotSet\\\",\\\"Subscription\\\",\\\"SubscriptionCompact\\\",\\\"CalendarDetails\\\",\\\"MultiListing\\\"],\\\"default\\\":\\\"NotSet\\\"}],\\\"default\\\":null},{\\\"name\\\":\\\"calReplyAccountCode\\\",\\\"type\\\":[\\\"null\\\",\\\"string\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"calReplyCalendarCode\\\",\\\"type\\\":[\\\"null\\\",\\\"string\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"campaignBrandName\\\",\\\"type\\\":[\\\"null\\\",\\\"string\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"campaignDomains\\\",\\\"type\\\":{\\\"type\\\":\\\"array\\\",\\\"items\\\":\\\"CampaignDomain\\\"},\\\"default\\\":[]},{\\\"name\\\":\\\"campaignId\\\",\\\"type\\\":\\\"long\\\"},{\\\"name\\\":\\\"campaignLanguageCode\\\",\\\"type\\\":[\\\"null\\\",\\\"string\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"campaignName\\\",\\\"type\\\":[\\\"null\\\",\\\"string\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"campaignOutcomeOptimiser\\\",\\\"type\\\":[\\\"null\\\",\\\"CampaignOutcomeOptimiser\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"campaignSecondaryProductCategorys\\\",\\\"type\\\":{\\\"type\\\":\\\"array\\\",\\\"items\\\":\\\"CampaignSecondaryProductCategory\\\"},\\\"default\\\":[]},{\\\"name\\\":\\\"campaignTermsAndConditions\\\",\\\"type\\\":{\\\"type\\\":\\\"array\\\",\\\"items\\\":[\\\"null\\\",\\\"CampaignTermsAndConditions\\\"]},\\\"default\\\":[]},{\\\"name\\\":\\\"campaignType\\\",\\\"type\\\":{\\\"type\\\":\\\"enum\\\",\\\"name\\\":\\\"CampaignType\\\",\\\"symbols\\\":[\\\"NotSet\\\",\\\"SelfService\\\",\\\"Managed\\\",\\\"Demo\\\",\\\"Internal\\\",\\\"Market\\\"],\\\"default\\\":\\\"NotSet\\\"}},{\\\"name\\\":\\\"referralType\\\",\\\"type\\\":{\\\"type\\\":\\\"enum\\\",\\\"name\\\":\\\"CampaignReferralType\\\",\\\"symbols\\\":[\\\"NotSet\\\",\\\"Email\\\",\\\"Phone\\\",\\\"Traffic\\\",\\\"EmailTraffic\\\",\\\"AppDownload\\\",\\\"Calendar\\\",\\\"CrossSell\\\",\\\"AddToCart\\\",\\\"FacebookShare\\\",\\\"TrafficCatalog\\\",\\\"RoktCalendar\\\",\\\"Promotion\\\",\\\"CustomerFeedback\\\",\\\"IntegratedApplication\\\"],\\\"default\\\":\\\"NotSet\\\"}},{\\\"name\\\":\\\"catalogId\\\",\\\"type\\\":[\\\"null\\\",\\\"string\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"catalogProviderConfig\\\",\\\"type\\\":[\\\"null\\\",{\\\"type\\\":\\\"map\\\",\\\"values\\\":\\\"string\\\"}],\\\"default\\\":null},{\\\"name\\\":\\\"countryId\\\",\\\"type\\\":[\\\"null\\\",\\\"int\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"couponCode\\\",\\\"type\\\":[\\\"null\\\",\\\"string\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"couponType\\\",\\\"type\\\":{\\\"type\\\":\\\"enum\\\",\\\"name\\\":\\\"CampaignCouponType\\\",\\\"symbols\\\":[\\\"NotSet\\\",\\\"Static\\\",\\\"UniqueAutoGen\\\",\\\"UniqueUpload\\\"],\\\"default\\\":\\\"NotSet\\\"}},{\\\"name\\\":\\\"defaultCultureId\\\",\\\"type\\\":[\\\"null\\\",\\\"int\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"disclaimer\\\",\\\"type\\\":[\\\"null\\\",\\\"string\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"disclaimerMarkdown\\\",\\\"type\\\":[\\\"null\\\",\\\"string\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"durationStart\\\",\\\"type\\\":[\\\"null\\\",\\\"long\\\"],\\\"default\\\":null,\\\"logicalType\\\":\\\"timestamp-millis\\\"},{\\\"name\\\":\\\"durationEnd\\\",\\\"type\\\":[\\\"null\\\",\\\"long\\\"],\\\"default\\\":null,\\\"logicalType\\\":\\\"timestamp-millis\\\"},{\\\"name\\\":\\\"durationTimezoneOffset\\\",\\\"type\\\":[\\\"null\\\",\\\"float\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"googlePlayUrl\\\",\\\"type\\\":[\\\"null\\\",\\\"string\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"isArchived\\\",\\\"type\\\":\\\"boolean\\\",\\\"default\\\":false},{\\\"name\\\":\\\"isEnabled\\\",\\\"type\\\":\\\"boolean\\\",\\\"default\\\":false},{\\\"name\\\":\\\"isManualCampaign\\\",\\\"type\\\":\\\"boolean\\\",\\\"default\\\":false},{\\\"name\\\":\\\"lastCouponUploadFailed\\\",\\\"type\\\":[\\\"null\\\",\\\"boolean\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"latestTermsAndConditions\\\",\\\"type\\\":[\\\"null\\\",\\\"string\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"latestTermsAndConditionsMarkdown\\\",\\\"type\\\":[\\\"null\\\",\\\"string\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"measurementGroupId\\\",\\\"type\\\":[\\\"null\\\",\\\"long\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"pendingCouponUploads\\\",\\\"type\\\":[\\\"null\\\",\\\"int\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"priceType\\\",\\\"type\\\":[\\\"null\\\",{\\\"type\\\":\\\"enum\\\",\\\"name\\\":\\\"AudiencePriceTypes\\\",\\\"symbols\\\":[\\\"NotSet\\\",\\\"FixedReferral\\\",\\\"AutoReferral\\\",\\\"CostPerAcquisition\\\",\\\"CostPerSale\\\",\\\"CostPerSaleItem\\\",\\\"CostPerSalePercentage\\\",\\\"CostPerImpression\\\",\\\"ExposureControl\\\"],\\\"default\\\":\\\"NotSet\\\"}]},{\\\"name\\\":\\\"prioritize\\\",\\\"type\\\":[\\\"null\\\",\\\"boolean\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"privacyPolicy\\\",\\\"type\\\":[\\\"null\\\",\\\"string\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"privacyPolicyMarkdown\\\",\\\"type\\\":[\\\"null\\\",\\\"string\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"productCategory\\\",\\\"type\\\":[\\\"null\\\",{\\\"type\\\":\\\"record\\\",\\\"name\\\":\\\"CampaignProductCategory\\\",\\\"fields\\\":[{\\\"name\\\":\\\"productCategoryId\\\",\\\"type\\\":\\\"int\\\"},{\\\"name\\\":\\\"productSubCategoryId\\\",\\\"type\\\":\\\"int\\\"}]}],\\\"default\\\":null},{\\\"name\\\":\\\"referralDeliveryTypes\\\",\\\"type\\\":{\\\"type\\\":\\\"array\\\",\\\"items\\\":{\\\"type\\\":\\\"enum\\\",\\\"name\\\":\\\"ReferralDeliveryType\\\",\\\"symbols\\\":[\\\"DeliverWithoutPI\\\",\\\"Email\\\",\\\"Phone\\\",\\\"All\\\"],\\\"default\\\":\\\"DeliverWithoutPI\\\"}},\\\"default\\\":[]},{\\\"name\\\":\\\"referralDeliveryTypesFlagValue\\\",\\\"type\\\":[\\\"null\\\",\\\"int\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"refusalSuppressionDays\\\",\\\"type\\\":[\\\"null\\\",\\\"int\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"saasFee\\\",\\\"type\\\":\\\"double\\\"},{\\\"name\\\":\\\"saasVarianceRate\\\",\\\"type\\\":\\\"double\\\"},{\\\"name\\\":\\\"salesManagerEmail\\\",\\\"type\\\":[\\\"null\\\",\\\"string\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"salesManagerId\\\",\\\"type\\\":[\\\"null\\\",\\\"long\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"showNotification\\\",\\\"type\\\":\\\"boolean\\\",\\\"default\\\":false},{\\\"name\\\":\\\"timezoneId\\\",\\\"type\\\":[\\\"null\\\",\\\"long\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"transactionSuppressionDays\\\",\\\"type\\\":[\\\"null\\\",\\\"int\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"createdDateTime\\\",\\\"type\\\":[\\\"null\\\",\\\"long\\\"],\\\"default\\\":null,\\\"logicalType\\\":\\\"timestamp-millis\\\"},{\\\"name\\\":\\\"deletedDateTime\\\",\\\"type\\\":[\\\"null\\\",\\\"long\\\"],\\\"default\\\":null,\\\"logicalType\\\":\\\"timestamp-millis\\\"},{\\\"name\\\":\\\"lastUpdatedDateTime\\\",\\\"type\\\":[\\\"null\\\",\\\"long\\\"],\\\"default\\\":null,\\\"logicalType\\\":\\\"timestamp-millis\\\"},{\\\"name\\\":\\\"version\\\",\\\"type\\\":[\\\"null\\\",\\\"int\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"versionDateTime\\\",\\\"type\\\":[\\\"null\\\",\\\"long\\\"],\\\"default\\\":null,\\\"logicalType\\\":\\\"timestamp-millis\\\"}]}\"\n}")
    .ToObject<Confluent.SchemaRegistry.RegisteredSchema>();

    // this line will throw because SchemaType_String is null
    Console.WriteLine(exampleUsingGlobalSettings.SchemaType);
}
catch(Exception ex)
{
    Console.WriteLine($"Exception being thrown when using global json default settings {ex.Message}");
}

// working example calling JsonSerializer.Create()
var exampleNotUsingGlobalSettings = JObject.Parse("{\n    \"subject\": \"objectStatus.ObjectStatusResourceChanged-value\",\n    \"version\": 3,\n    \"id\": 1296,\n    \"references\": [\n        {\n            \"name\": \"com.rokt.objectStatus.Status\",\n            \"subject\": \"objectStatus.Status-value\",\n            \"version\": 3\n        }\n    ],\n    \"schema\": \"{\\\"type\\\":\\\"record\\\",\\\"name\\\":\\\"ObjectStatusResourceChanged\\\",\\\"namespace\\\":\\\"com.rokt.objectStatus\\\",\\\"doc\\\":\\\"avro schema for the Object Status resource changed event com.rokt.objectStatus.ObjectStatusResourceChanged\\\",\\\"fields\\\":[{\\\"name\\\":\\\"accountId\\\",\\\"type\\\":\\\"long\\\"},{\\\"name\\\":\\\"campaignId\\\",\\\"type\\\":[\\\"null\\\",\\\"long\\\"],\\\"default\\\":null},{\\\"name\\\":\\\"objectId\\\",\\\"type\\\":\\\"string\\\"},{\\\"name\\\":\\\"objectType\\\",\\\"type\\\":\\\"ObjectType\\\"},{\\\"name\\\":\\\"allRuleEvaluationsPassed\\\",\\\"type\\\":\\\"boolean\\\"},{\\\"name\\\":\\\"objectStatusEventId\\\",\\\"type\\\":\\\"string\\\"},{\\\"name\\\":\\\"updatedTimestamp\\\",\\\"type\\\":\\\"long\\\",\\\"logicalType\\\":\\\"timestamp-millis\\\"},{\\\"name\\\":\\\"statuses\\\",\\\"type\\\":{\\\"type\\\":\\\"array\\\",\\\"items\\\":\\\"Status\\\"},\\\"default\\\":[]}]}\"\n}")
    .ToObject<Confluent.SchemaRegistry.RegisteredSchema>(JsonSerializer.Create());

Console.WriteLine($"No exception thrown accessing SchemaType  {exampleNotUsingGlobalSettings.SchemaType}");
fridayhut commented 1 year ago

Hi @mhowlett ,

Any idea on when this might be able to be addressed?

Thanks!