search

PacoVK / tapir

A Private Terraform Registry
https://pascal.euhus.dev/tapir/
Apache License 2.0
187 stars 12 forks source link

Uploading a new provider version fails when using CosmosDB as a backend #410

Closed Kirimson closed 3 months ago

Kirimson commented 3 months ago

Version: latest docker image/0.7.1

I've noticed that uploading a new version of a provider fails if CosmosDB is being used as a backend, with Cosmos returning a ConflictException with the message: Resource with specified id or name already exists.

Looking at the stack trace, it might be caused by the injestProviderData method using createItem when creating/updating the provider within the Cosmos container:

https://github.com/PacoVK/tapir/blob/eb455facb0b9013fb5c91b51ce170f2054cfccf3/src/main/java/core/backend/azure/cosmosdb/CosmosDbRepository.java#L195-L199

While injestModuleData uses upsertItem:

https://github.com/PacoVK/tapir/blob/eb455facb0b9013fb5c91b51ce170f2054cfccf3/src/main/java/core/backend/azure/cosmosdb/CosmosDbRepository.java#L169-L173

Stack Trace:

2024-05-30T13:15:56.514889052Z  Suppressed: java.lang.Exception: #block terminated with an error
2024-05-30T13:15:56.514895805Z      at reactor.core.publisher.BlockingSingleSubscriber.blockingGet(BlockingSingleSubscriber.java:100)
2024-05-30T13:15:56.514902477Z      at reactor.core.publisher.Mono.block(Mono.java:1742)
2024-05-30T13:15:56.514909220Z      at com.azure.cosmos.CosmosContainer.blockItemResponse(CosmosContainer.java:270)
2024-05-30T13:15:56.514915802Z      at com.azure.cosmos.CosmosContainer.createItem(CosmosContainer.java:207)
2024-05-30T13:15:56.514922505Z      at core.backend.azure.cosmosdb.CosmosDbRepository.ingestProviderData(CosmosDbRepository.java:195)
2024-05-30T13:15:56.514929297Z      at core.backend.azure.cosmosdb.CosmosDbRepository_ClientProxy.ingestProviderData(Unknown Source)
2024-05-30T13:15:56.514935849Z      at core.service.ProviderService.ingestProviderData(ProviderService.java:48)
2024-05-30T13:15:56.514948022Z      at core.service.ProviderService_ClientProxy.ingestProviderData(Unknown Source)
2024-05-30T13:15:56.514954694Z      at core.upload.service.UploadService.uploadProvider(UploadService.java:67)
2024-05-30T13:15:56.514961237Z      at core.upload.service.UploadService_ClientProxy.uploadProvider(Unknown Source)
2024-05-30T13:15:56.514967979Z      at api.Providers.uploadProvider(Providers.java:73)
2024-05-30T13:15:56.514974481Z      at api.Providers$quarkusrestinvoker$uploadProvider_b68a044ef86b0f23fd44d8fa5cae7bb31d1e67e2.invoke(Unknown Source)
2024-05-30T13:15:56.514981394Z      at org.jboss.resteasy.reactive.server.handlers.InvocationHandler.handle(InvocationHandler.java:29)
2024-05-30T13:15:56.514988287Z      at io.quarkus.resteasy.reactive.server.runtime.QuarkusResteasyReactiveRequestContext.invokeHandler(QuarkusResteasyReactiveRequestContext.java:141)
2024-05-30T13:15:56.514995059Z      at org.jboss.resteasy.reactive.common.core.AbstractResteasyReactiveContext.run(AbstractResteasyReactiveContext.java:147)
2024-05-30T13:15:56.515001842Z      at io.quarkus.vertx.core.runtime.VertxCoreRecorder$14.runWith(VertxCoreRecorder.java:599)
2024-05-30T13:15:56.515008444Z      at org.jboss.threads.EnhancedQueueExecutor$Task.doRunWith(EnhancedQueueExecutor.java:2516)
2024-05-30T13:15:56.515015207Z      at org.jboss.threads.EnhancedQueueExecutor$Task.run(EnhancedQueueExecutor.java:2495)
2024-05-30T13:15:56.515021759Z      at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1521)
2024-05-30T13:15:56.515034553Z      at org.jboss.threads.DelegatingRunnable.run(DelegatingRunnable.java:11)
2024-05-30T13:15:56.515041475Z      at org.jboss.threads.ThreadLocalResettingRunnable.run(ThreadLocalResettingRunnable.java:11)
2024-05-30T13:15:56.515048258Z      ... 2 more
2024-05-30T13:15:56.515080631Z 2024-05-30 13:15:56,504 SEVERE [api.map.exc.ThrowableMapper] (executor-thread-5) errorId 730998d6-0a3e-4cfb-afd2-fbe0c8b02266: {"ClassName":"ConflictException","userAgent":"azsdk-java-cosmos/4.61.0 Linux/5.15.153.1-2.cm2 JRE/21.0.3","statusCode":409,"resourceAddress":"rntbd://cdb-ms-prod-uksouth1-be17.documents.azure.com:14153/apps/.../services/.../partitions/.../replicas/133598370177412741p/","error":"{\"Errors\":[\"Resource with specified id or name already exists.\"]}","innerErrorMessage":"[\"Resource with specified id or name already exists.\"]","causeInfo":null,"responseHeaders":"{x-ms-current-replica-set-size=4, x-ms-last-state-change-utc=Wed, 29 May 2024 02:24:47.171 GMT, x-ms-request-duration-ms=0.813, x-ms-session-token=0:-1#7, lsn=7, x-ms-request-charge=5.48, x-ms-schemaversion=1.17, x-ms-transport-request-id=54, x-ms-number-of-read-regions=0, x-ms-current-write-quorum=3, x-ms-cosmos-quorum-acked-llsn=7, x-ms-quorum-acked-lsn=7, x-ms-activity-id=22868e67-358f-41f7-abd8-8fb3c3bc3c4a, x-ms-xp-role=1, x-ms-global-Committed-lsn=7, x-ms-cosmos-llsn=7, x-ms-serviceversion= version=2.14.0.0}","cosmosDiagnostics":{"userAgent":"azsdk-java-cosmos/4.61.0 Linux/5.15.153.1-2.cm2 JRE/21.0.3","activityId":"22868e67-358f-41f7-abd8-8fb3c3bc3c4a","requestLatencyInMs":4,"requestStartTimeUTC":"2024-05-30T13:15:56.496358087Z","requestEndTimeUTC":"2024-05-30T13:15:56.501189160Z","responseStatisticsList":[{"storeResult":{"storePhysicalAddress":"rntbd://cdb-ms-prod-uksouth1-be17.documents.azure.com:14153/apps/.../services/.../partitions/.../replicas/133598370177412741p/","lsn":7,"globalCommittedLsn":7,"partitionKeyRangeId":"0","isValid":true,"statusCode":409,"subStatusCode":0,"isGone":false,"isNotFound":false,"isInvalidPartition":false,"isThroughputControlRequestRateTooLarge":false,"requestCharge":5.48,"itemLSN":-1,"sessionToken":"0:-1#7","backendLatencyInMs":0.813,"retryAfterInMs":null,"exceptionMessage":"[\"Resource with specified id or name already exists.\"]","exceptionResponseHeaders":"{x-ms-current-replica-set-size=4, x-ms-last-state-change-utc=Wed, 29 May 2024 02:24:47.171 GMT, x-ms-request-duration-ms=0.813, x-ms-session-token=0:-1#7, lsn=7, x-ms-request-charge=5.48, x-ms-schemaversion=1.17, x-ms-transport-request-id=54, x-ms-number-of-read-regions=0, x-ms-current-write-quorum=3, x-ms-cosmos-quorum-acked-llsn=7, x-ms-quorum-acked-lsn=7, x-ms-activity-id=22868e67-358f-41f7-abd8-8fb3c3bc3c4a, x-ms-xp-role=1, x-ms-global-Committed-lsn=7, x-ms-cosmos-llsn=7, x-ms-documentdb-partitionkeyrangeid=0, x-ms-serviceversion= version=2.14.0.0}","replicaStatusList":["14153:Connected"],"transportRequestTimeline":[{"eventName":"created","startTimeUTC":"2024-05-30T13:15:56.498202070Z","durationInMilliSecs":0.031148},{"eventName":"queued","startTimeUTC":"2024-05-30T13:15:56.498233218Z","durationInMilliSecs":6.01E-4},{"eventName":"channelAcquisitionStarted","startTimeUTC":"2024-05-30T13:15:56.498233819Z","durationInMilliSecs":0.194},{"eventName":"pipelined","startTimeUTC":"2024-05-30T13:15:56.498427819Z","durationInMilliSecs":0.309159},{"eventName":"transitTime","startTimeUTC":"2024-05-30T13:15:56.498736978Z","durationInMilliSecs":1.614457},{"eventName":"decodeTime","startTimeUTC":"2024-05-30T13:15:56.500351435Z","durationInMilliSecs":0.100136},{"eventName":"received","startTimeUTC":"2024-05-30T13:15:56.500451571Z","durationInMilliSecs":0.604603},{"eventName":"completed","startTimeUTC":"2024-05-30T13:15:56.501056174Z","durationInMilliSecs":0.009998}],"rntbdRequestLengthInBytes":4628,"rntbdResponseLengthInBytes":275,"requestPayloadLengthInBytes":4198,"responsePayloadLengthInBytes":275,"channelStatistics":{"channelId":"c47981a1","channelTaskQueueSize":1,"pendingRequestsCount":0,"lastReadTime":"2024-05-30T13:15:56.494839812Z","waitForConnectionInit":false},"serviceEndpointStatistics":{"availableChannels":1,"acquiredChannels":0,"executorTaskQueueSize":0,"inflightRequests":1,"lastSuccessfulRequestTime":"2024-05-30T13:15:56.496Z","lastRequestTime":"2024-05-30T13:15:56.491Z","createdTime":"2024-05-30T12:58:11.658099847Z","isClosed":false,"cerMetrics":{}}},"requestResponseTimeUTC":"2024-05-30T13:15:56.501189160Z","requestStartTimeUTC":"2024-05-30T13:15:56.498202070Z","requestResourceType":"Document","requestOperationType":"Create","requestSessionToken":null,"e2ePolicyCfg":null,"excludedRegions":null,"sessionTokenEvaluationResults":[]}],"supplementalResponseStatisticsList":[],"addressResolutionStatistics":{},"regionsContacted":["uk south"],"retryContext":{"statusAndSubStatusCodes":null,"retryLatency":0,"retryCount":0},"metadataDiagnosticsContext":{"metadataDiagnosticList":null},"serializationDiagnosticsContext":{"serializationDiagnosticsList":[{"serializationType":"ITEM_SERIALIZATION","startTimeUTC":"2024-05-30T13:15:56.496516451Z","endTimeUTC":"2024-05-30T13:15:56.497166698Z","durationInMilliSecs":0.650247}]},"gatewayStatisticsList":[],"samplingRateSnapshot":1.0,"bloomFilterInsertionCountSnapshot":0,"systemInformation":{"usedMemory":"103909 KB","availableMemory":"944667 KB","systemCpuLoad":"(2024-05-30T13:15:30.917723281Z 0.3%), (2024-05-30T13:15:35.917718511Z 0.4%), (2024-05-30T13:15:40.917719735Z 0.3%), (2024-05-30T13:15:45.917729722Z 0.3%), (2024-05-30T13:15:50.917720638Z 0.4%), (2024-05-30T13:15:55.917768297Z 20.9%)","availableProcessors":2},"clientCfgs":{"id":1,"machineId":"uuid:2c59872a-0835-43d5-902e-bf79deb869cf","connectionMode":"DIRECT","numberOfClients":1,"excrgns":"[]","clientEndpoints":{"https://<COSMOSDB>.documents.azure.com":1},"connCfg":{"rntbd":"(cto:PT5S, nrto:PT5S, icto:PT0S, ieto:PT1H, mcpe:130, mrpc:30, cer:true)","gw":"(cps:1000, nrto:PT1M, icto:PT1M, p:false)","other":"(ed: true, cs: false, rv: true)"},"consistencyCfg":"(consistency: Eventual, mm: true, prgns: [])","proactiveInitCfg":"","e2ePolicyCfg":"","sessionRetryCfg":""}}}
PacoVK commented 3 months ago

thanks for the issue @Kirimson and also for the investigation :) I will release a fix soon