search

airbytehq / airbyte

The leading data integration platform for ETL / ELT data pipelines from APIs, databases & files to data warehouses, data lakes & data lakehouses. Both self-hosted and Cloud-hosted.
https://airbyte.com
Other
16.04k stars 4.11k forks source link

Source CommerceTools: JSON Schema `start_date` is self-conflicting; impossible to fill out. #34133

Open theomarshanti opened 9 months ago

theomarshanti commented 9 months ago

Connector Name

source-commercetools

Connector Version

0.2.0

What step the error happened?

Configuring a new connector

Relevant information

Right now it is impossible to register a new CommerceTools source. This is because the json-schema for start-date specifies two mutually incompatible conditions. It declares the type be a start-date, which requires HHMMSS and it declares the type to be a date format. This causes JSON-Schema failures like the following:

Relevant log output

POST http://localhost:8000/api/v1/sources/create

{
    "name": "Commercetools",
    "sourceDefinitionId": "008b2e26-11a3-11ec-82a8-0242ac130003",
    "workspaceId": "4853dbc7-d665-4c32-844e-a48156633689",
    "connectionConfiguration": {
        "start_date": "2024-01-01",
        "client_id": "<>",
        "client_secret": "<>",
        "project_key": "<>",
        "region": "<>",
        "host": "<>"
    }
}
{
    "message": "The provided configuration does not fulfill the specification. Errors: json schema validation failed when comparing the data to the json schema. \nErrors: $.start_date: is missing but it is required \nSchema: \n{\n  \"type\" : \"object\",\n  \"$schema\" : \"http://json-schema.org/draft-07/schema#\",\n  \"required\" : [ \"start_date\", \"client_id\", \"client_secret\", \"host\", \"project_key\", \"region\" ],\n  \"properties\" : {\n    \"host\" : {\n      \"enum\" : [ \"gcp\", \"aws\" ],\n      \"type\" : \"string\",\n      \"order\" : 3,\n      \"title\" : \"Host\",\n      \"description\" : \"The cloud provider your shop is hosted. See: https://docs.commercetools.com/api/authorization\"\n    },\n    \"region\" : {\n      \"type\" : \"string\",\n      \"order\" : 5,\n      \"title\" : \"Region\",\n      \"examples\" : [ \"us-central1\", \"australia-southeast1\" ],\n      \"description\" : \"The region of the platform.\"\n    },\n    \"client_id\" : {\n      \"type\" : \"string\",\n      \"order\" : 1,\n      \"title\" : \"Client ID\",\n      \"description\" : \"Id of API Client.\",\n      \"airbyte_secret\" : true\n    },\n    \"start_date\" : {\n      \"type\" : \"string\",\n      \"order\" : 0,\n      \"title\" : \"Start date\",\n      \"format\" : \"date-time\",\n      \"pattern\" : \"^[0-9]{4}-[0-9]{2}-[0-9]{2}$\"\n    },\n    \"project_key\" : {\n      \"type\" : \"string\",\n      \"order\" : 4,\n      \"title\" : \"Project key\",\n      \"description\" : \"The project key\"\n    },\n    \"client_secret\" : {\n      \"type\" : \"string\",\n      \"order\" : 2,\n      \"title\" : \"Client secret\",\n      \"description\" : \"The password of secret of API Client.\",\n      \"airbyte_secret\" : true\n    }\n  },\n  \"additionalProperties\" : true\n}",
    "exceptionClassName": "io.airbyte.commons.server.errors.BadObjectSchemaKnownException",
    "exceptionStack": [
        "io.airbyte.commons.server.errors.BadObjectSchemaKnownException: The provided configuration does not fulfill the specification. Errors: json schema validation failed when comparing the data to the json schema. ",
        "Errors: $.start_date: is missing but it is required ",
        "Schema: ",
        "{",
        "  \"type\" : \"object\",",
        "  \"$schema\" : \"http://json-schema.org/draft-07/schema#\",",
        "  \"required\" : [ \"start_date\", \"client_id\", \"client_secret\", \"host\", \"project_key\", \"region\" ],",
        "  \"properties\" : {",
        "    \"host\" : {",
        "      \"enum\" : [ \"gcp\", \"aws\" ],",
        "      \"type\" : \"string\",",
        "      \"order\" : 3,",
        "      \"title\" : \"Host\",",
        "      \"description\" : \"The cloud provider your shop is hosted. See: https://docs.commercetools.com/api/authorization\"",
        "    },",
        "    \"region\" : {",
        "      \"type\" : \"string\",",
        "      \"order\" : 5,",
        "      \"title\" : \"Region\",",
        "      \"examples\" : [ \"us-central1\", \"australia-southeast1\" ],",
        "      \"description\" : \"The region of the platform.\"",
        "    },",
        "    \"client_id\" : {",
        "      \"type\" : \"string\",",
        "      \"order\" : 1,",
        "      \"title\" : \"Client ID\",",
        "      \"description\" : \"Id of API Client.\",",
        "      \"airbyte_secret\" : true",
        "    },",
        "    \"start_date\" : {",
        "      \"type\" : \"string\",",
        "      \"order\" : 0,",
        "      \"title\" : \"Start date\",",
        "      \"format\" : \"date-time\",",
        "      \"pattern\" : \"^[0-9]{4}-[0-9]{2}-[0-9]{2}$\"",
        "    },",
        "    \"project_key\" : {",
        "      \"type\" : \"string\",",
        "      \"order\" : 4,",
        "      \"title\" : \"Project key\",",
        "      \"description\" : \"The project key\"",
        "    },",
        "    \"client_secret\" : {",
        "      \"type\" : \"string\",",
        "      \"order\" : 2,",
        "      \"title\" : \"Client secret\",",
        "      \"description\" : \"The password of secret of API Client.\",",
        "      \"airbyte_secret\" : true",
        "    }",
        "  },",
        "  \"additionalProperties\" : true",
        "}",
        "\tat io.airbyte.server.apis.ApiHelper.execute(ApiHelper.java:40)",
        "\tat io.airbyte.server.apis.SourceApiController.createSource(SourceApiController.java:101)",
        "\tat io.airbyte.server.apis.$SourceApiController$Definition$Exec.dispatch(Unknown Source)",
        "\tat io.micronaut.context.AbstractExecutableMethodsDefinition$DispatchedExecutableMethod.invoke(AbstractExecutableMethodsDefinition.java:371)",
        "\tat io.micronaut.context.DefaultBeanContext$4.invoke(DefaultBeanContext.java:594)",
        "\tat io.micronaut.web.router.AbstractRouteMatch.execute(AbstractRouteMatch.java:303)",
        "\tat io.micronaut.web.router.RouteMatch.execute(RouteMatch.java:111)",
        "\tat io.micronaut.http.context.ServerRequestContext.with(ServerRequestContext.java:103)",
        "\tat io.micronaut.http.server.RouteExecutor.lambda$executeRoute$14(RouteExecutor.java:659)",
        "\tat reactor.core.publisher.FluxDeferContextual.subscribe(FluxDeferContextual.java:49)",
        "\tat reactor.core.publisher.InternalFluxOperator.subscribe(InternalFluxOperator.java:62)",
        "\tat reactor.core.publisher.FluxSubscribeOn$SubscribeOnSubscriber.run(FluxSubscribeOn.java:194)",
        "\tat io.micronaut.reactive.reactor.instrument.ReactorInstrumentation.lambda$init$0(ReactorInstrumentation.java:62)",
        "\tat reactor.core.scheduler.WorkerTask.call(WorkerTask.java:84)",
        "\tat reactor.core.scheduler.WorkerTask.call(WorkerTask.java:37)",
        "\tat io.micronaut.scheduling.instrument.InvocationInstrumenterWrappedCallable.call(InvocationInstrumenterWrappedCallable.java:53)",
        "\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)",
        "\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)",
        "\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)",
        "\tat java.base/java.lang.Thread.run(Thread.java:1583)",
        "Caused by: io.airbyte.validation.json.JsonValidationException: json schema validation failed when comparing the data to the json schema. ",
        "Errors: $.start_date: is missing but it is required ",
        "Schema: ",
        "{",
        "  \"type\" : \"object\",",
        "  \"$schema\" : \"http://json-schema.org/draft-07/schema#\",",
        "  \"required\" : [ \"start_date\", \"client_id\", \"client_secret\", \"host\", \"project_key\", \"region\" ],",
        "  \"properties\" : {",
        "    \"host\" : {",
        "      \"enum\" : [ \"gcp\", \"aws\" ],",
        "      \"type\" : \"string\",",
        "      \"order\" : 3,",
        "      \"title\" : \"Host\",",
        "      \"description\" : \"The cloud provider your shop is hosted. See: https://docs.commercetools.com/api/authorization\"",
        "    },",
        "    \"region\" : {",
        "      \"type\" : \"string\",",
        "      \"order\" : 5,",
        "      \"title\" : \"Region\",",
        "      \"examples\" : [ \"us-central1\", \"australia-southeast1\" ],",
        "      \"description\" : \"The region of the platform.\"",
        "    },",
        "    \"client_id\" : {",
        "      \"type\" : \"string\",",
        "      \"order\" : 1,",
        "      \"title\" : \"Client ID\",",
        "      \"description\" : \"Id of API Client.\",",
        "      \"airbyte_secret\" : true",
        "    },",
        "    \"start_date\" : {",
        "      \"type\" : \"string\",",
        "      \"order\" : 0,",
        "      \"title\" : \"Start date\",",
        "      \"format\" : \"date-time\",",
        "      \"pattern\" : \"^[0-9]{4}-[0-9]{2}-[0-9]{2}$\"",
        "    },",
        "    \"project_key\" : {",
        "      \"type\" : \"string\",",
        "      \"order\" : 4,",
        "      \"title\" : \"Project key\",",
        "      \"description\" : \"The project key\"",
        "    },",
        "    \"client_secret\" : {",
        "      \"type\" : \"string\",",
        "      \"order\" : 2,",
        "      \"title\" : \"Client secret\",",
        "      \"description\" : \"The password of secret of API Client.\",",
        "      \"airbyte_secret\" : true",
        "    }",
        "  },",
        "  \"additionalProperties\" : true",
        "}",
        "\tat io.airbyte.validation.json.JsonSchemaValidator.ensure(JsonSchemaValidator.java:157)",
        "\tat io.airbyte.commons.server.handlers.SourceHandler.validateSource(SourceHandler.java:444)",
        "\tat io.airbyte.commons.server.handlers.SourceHandler.createSource(SourceHandler.java:162)",
        "\tat io.airbyte.commons.server.handlers.SourceHandler.createSourceWithOptionalSecret(SourceHandler.java:129)",
        "\tat io.airbyte.server.apis.SourceApiController.lambda$createSource$4(SourceApiController.java:101)",
        "\tat io.airbyte.server.apis.ApiHelper.execute(ApiHelper.java:28)",
        "\t... 19 more"
    ],
    "rootCauseExceptionClassName": "java.lang.Class",
    "rootCauseExceptionStack": [
        "io.airbyte.validation.json.JsonValidationException: json schema validation failed when comparing the data to the json schema. ",
        "Errors: $.start_date: is missing but it is required ",
        "Schema: ",
        "{",
        "  \"type\" : \"object\",",
        "  \"$schema\" : \"http://json-schema.org/draft-07/schema#\",",
        "  \"required\" : [ \"start_date\", \"client_id\", \"client_secret\", \"host\", \"project_key\", \"region\" ],",
        "  \"properties\" : {",
        "    \"host\" : {",
        "      \"enum\" : [ \"gcp\", \"aws\" ],",
        "      \"type\" : \"string\",",
        "      \"order\" : 3,",
        "      \"title\" : \"Host\",",
        "      \"description\" : \"The cloud provider your shop is hosted. See: https://docs.commercetools.com/api/authorization\"",
        "    },",
        "    \"region\" : {",
        "      \"type\" : \"string\",",
        "      \"order\" : 5,",
        "      \"title\" : \"Region\",",
        "      \"examples\" : [ \"us-central1\", \"australia-southeast1\" ],",
        "      \"description\" : \"The region of the platform.\"",
        "    },",
        "    \"client_id\" : {",
        "      \"type\" : \"string\",",
        "      \"order\" : 1,",
        "      \"title\" : \"Client ID\",",
        "      \"description\" : \"Id of API Client.\",",
        "      \"airbyte_secret\" : true",
        "    },",
        "    \"start_date\" : {",
        "      \"type\" : \"string\",",
        "      \"order\" : 0,",
        "      \"title\" : \"Start date\",",
        "      \"format\" : \"date-time\",",
        "      \"pattern\" : \"^[0-9]{4}-[0-9]{2}-[0-9]{2}$\"",
        "    },",
        "    \"project_key\" : {",
        "      \"type\" : \"string\",",
        "      \"order\" : 4,",
        "      \"title\" : \"Project key\",",
        "      \"description\" : \"The project key\"",
        "    },",
        "    \"client_secret\" : {",
        "      \"type\" : \"string\",",
        "      \"order\" : 2,",
        "      \"title\" : \"Client secret\",",
        "      \"description\" : \"The password of secret of API Client.\",",
        "      \"airbyte_secret\" : true",
        "    }",
        "  },",
        "  \"additionalProperties\" : true",
        "}",
        "\tat io.airbyte.validation.json.JsonSchemaValidator.ensure(JsonSchemaValidator.java:157)",
        "\tat io.airbyte.commons.server.handlers.SourceHandler.validateSource(SourceHandler.java:444)",
        "\tat io.airbyte.commons.server.handlers.SourceHandler.createSource(SourceHandler.java:162)",
        "\tat io.airbyte.commons.server.handlers.SourceHandler.createSourceWithOptionalSecret(SourceHandler.java:129)",
        "\tat io.airbyte.server.apis.SourceApiController.lambda$createSource$4(SourceApiController.java:101)",
        "\tat io.airbyte.server.apis.ApiHelper.execute(ApiHelper.java:28)",
        "\tat io.airbyte.server.apis.SourceApiController.createSource(SourceApiController.java:101)",
        "\tat io.airbyte.server.apis.$SourceApiController$Definition$Exec.dispatch(Unknown Source)",
        "\tat io.micronaut.context.AbstractExecutableMethodsDefinition$DispatchedExecutableMethod.invoke(AbstractExecutableMethodsDefinition.java:371)",
        "\tat io.micronaut.context.DefaultBeanContext$4.invoke(DefaultBeanContext.java:594)",
        "\tat io.micronaut.web.router.AbstractRouteMatch.execute(AbstractRouteMatch.java:303)",
        "\tat io.micronaut.web.router.RouteMatch.execute(RouteMatch.java:111)",
        "\tat io.micronaut.http.context.ServerRequestContext.with(ServerRequestContext.java:103)",
        "\tat io.micronaut.http.server.RouteExecutor.lambda$executeRoute$14(RouteExecutor.java:659)",
        "\tat reactor.core.publisher.FluxDeferContextual.subscribe(FluxDeferContextual.java:49)",
        "\tat reactor.core.publisher.InternalFluxOperator.subscribe(InternalFluxOperator.java:62)",
        "\tat reactor.core.publisher.FluxSubscribeOn$SubscribeOnSubscriber.run(FluxSubscribeOn.java:194)",
        "\tat io.micronaut.reactive.reactor.instrument.ReactorInstrumentation.lambda$init$0(ReactorInstrumentation.java:62)",
        "\tat reactor.core.scheduler.WorkerTask.call(WorkerTask.java:84)",
        "\tat reactor.core.scheduler.WorkerTask.call(WorkerTask.java:37)",
        "\tat io.micronaut.scheduling.instrument.InvocationInstrumenterWrappedCallable.call(InvocationInstrumenterWrappedCallable.java:53)",
        "\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)",
        "\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)",
        "\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)",
        "\tat java.base/java.lang.Thread.run(Thread.java:1583)"
    ]
}

Contribute

theomarshanti commented 9 months ago

I have a fix here: https://github.com/airbytehq/airbyte/pull/34130