search

aws / aws-sdk-js-v3

Modularized AWS SDK for JavaScript.
Apache License 2.0
3.1k stars 578 forks source link

KafkaConnectClient.CreateConnectorCommand(input) not work #4425

Closed amliuyong closed 1 year ago

amliuyong commented 1 year ago

Checkboxes for prior research

Describe the bug

The KafkaConnectClient.CreateConnectorCommand(input) does not work

Tried to create a S3SinkConnector in lambda, but get error.

The command:

{
    "middlewareStack": {},
    "input": {
        "connectorName": "sink4-S3SinkConnector-s3SinkConnectorCustomResource0ef1395c223d",
        "plugins": [
            {
                "customPlugin": {
                    "customPluginArn": "arn:aws:kafkaconnect:us-east-1:080766874269:custom-plugin/sink4-ConnectorS3Plugin-s3SinkConnectorCustomResource0ef1395c223d/9469c7f0-7f02-45cb-974c-485a69339cb8-2",
                    "revision": 1
                }
            }
        ],
        "capacity": {
            "autoScaling": {
                "maxWorkerCount": "3",
                "mcuCount": "1",
                "minWorkerCount": "1",
                "scaleInPolicy": {
                    "cpuUtilizationPercentage": 20
                },
                "scaleOutPolicy": {
                    "cpuUtilizationPercentage": 80
                }
            }
        },
        "connectorConfiguration": {
            "connector.class": "io.confluent.connect.s3.S3SinkConnector",
            "topics": "cs-msk-small-topic",
            "s3.region": "us-east-1",
            "s3.bucket.name": "clickstream-infra-s3sink4dfdadf4-10ewmceey09vp",
            "topics.dir": "sink-data",
            "flush.size": "50000",
            "rotate.interval.ms": "3000000",
            "s3.compression.type": "gzip",
            "storage.class": "io.confluent.connect.s3.storage.S3Storage",
            "format.class": "io.confluent.connect.s3.format.json.JsonFormat",
            "partitioner.class": "io.confluent.connect.storage.partitioner.TimeBasedPartitioner",
            "path.format": "'year'=YYYY/'month'=MM/'day'=dd/'hour'=HH",
            "partition.duration.ms": "60000",
            "timezone": "UTC",
            "locale": "en-US",
            "schema.compatibility": "NONE"
        },
        "kafkaCluster": {
            "apacheKafkaCluster": {
                "bootstrapServers": "b-4.csmsksmallmskcluster.26tc9t.c2.kafka.us-east-1.amazonaws.com:9092,b-9.csmsksmallmskcluster.26tc9t.c2.kafka.us-east-1.amazonaws.com:9092,b-1.csmsksmallmskcluster.26tc9t.c2.kafka.us-east-1.amazonaws.com:9092",
                "vpc": {
                    "securityGroups": [
                        "sg-0e1ba465b935af7c5"
                    ],
                    "subnets": [
                        "subnet-003ce686f7c88605c",
                        "subnet-02996047b3c1cbcf3"
                    ]
                }
            }
        },
        "kafkaClusterClientAuthentication": {
            "authenticationType": "NONE"
        },
        "kafkaClusterEncryptionInTransit": {
            "encryptionType": "PLAINTEXT"
        },
        "kafkaConnectVersion": "2.7.1",
        "logDelivery": {
            "workerLogDelivery": {
                "s3": {
                    "bucket": "clickstream-infra-s3sink4dfdadf4-10ewmceey09vp",
                    "enabled": true,
                    "prefix": "sink-log"
                }
            }
        },
        "serviceExecutionRoleArn": "arn:aws:iam::080766874269:role/sink4-s3kafkaconnectroleAF13586E-1B9DIYD1M8WMD"
    }
}

Error message:

2023-02-10T03:02:14.940Z    1fb6006a-7004-41cf-aea7-d384af4ae95a    ERROR   BadRequestException: [format attribute "password" not supported, instance type (string) does not match any allowed primitive type (allowed: ["integer"]), format attribute "int32" not supported]
    at deserializeAws_restJson1BadRequestExceptionResponse (/var/runtime/node_modules/@aws-sdk/client-kafkaconnect/dist-cjs/protocols/Aws_restJson1.js:1037:23)
    at deserializeAws_restJson1CreateConnectorCommandError (/var/runtime/node_modules/@aws-sdk/client-kafkaconnect/dist-cjs/protocols/Aws_restJson1.js:306:25)
    at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
    at async /var/runtime/node_modules/@aws-sdk/middleware-serde/dist-cjs/deserializerMiddleware.js:7:24
    at async /var/runtime/node_modules/@aws-sdk/middleware-signing/dist-cjs/middleware.js:13:20
    at async StandardRetryStrategy.retry (/var/runtime/node_modules/@aws-sdk/middleware-retry/dist-cjs/StandardRetryStrategy.js:51:46)
    at async /var/runtime/node_modules/@aws-sdk/middleware-logger/dist-cjs/loggerMiddleware.js:6:22
    at async createConnector (/var/task/index.js:252:13)
    at async onCreate (/var/task/index.js:152:5)
    at async _handler (/var/task/index.js:110:5) {
  '$fault': 'client',
  '$metadata': {
    httpStatusCode: 400,
    requestId: 'df56485b-9a7f-4348-a4ff-113e8fea66f4',
    extendedRequestId: undefined,
    cfId: 'pIPzapexMStBwUPzfseJHslpbYoRFG7ykNiG2Z5PQ1qmTi0D3MWtiA==',
    attempts: 1,
    totalRetryDelay: 0
  }
}

SDK version number

@aws-sdk/client-kafkaconnect@3.267.0

Which JavaScript Runtime is this issue in?

Node.js

Details of the browser/Node.js/ReactNative version

lambda: Runtime.NODEJS_18_X,

Reproduction Steps

Tried to create a S3SinkConnector in lambda:

The command:

{
    "middlewareStack": {},
    "input": {
        "connectorName": "sink4-S3SinkConnector-s3SinkConnectorCustomResource0ef1395c223d",
        "plugins": [
            {
                "customPlugin": {
                    "customPluginArn": "arn:aws:kafkaconnect:us-east-1:080766874269:custom-plugin/sink4-ConnectorS3Plugin-s3SinkConnectorCustomResource0ef1395c223d/9469c7f0-7f02-45cb-974c-485a69339cb8-2",
                    "revision": 1
                }
            }
        ],
        "capacity": {
            "autoScaling": {
                "maxWorkerCount": "3",
                "mcuCount": "1",
                "minWorkerCount": "1",
                "scaleInPolicy": {
                    "cpuUtilizationPercentage": 20
                },
                "scaleOutPolicy": {
                    "cpuUtilizationPercentage": 80
                }
            }
        },
        "connectorConfiguration": {
            "connector.class": "io.confluent.connect.s3.S3SinkConnector",
            "topics": "cs-msk-small-topic",
            "s3.region": "us-east-1",
            "s3.bucket.name": "clickstream-infra-s3sink4dfdadf4-10ewmceey09vp",
            "topics.dir": "sink-data",
            "flush.size": "50000",
            "rotate.interval.ms": "3000000",
            "s3.compression.type": "gzip",
            "storage.class": "io.confluent.connect.s3.storage.S3Storage",
            "format.class": "io.confluent.connect.s3.format.json.JsonFormat",
            "partitioner.class": "io.confluent.connect.storage.partitioner.TimeBasedPartitioner",
            "path.format": "'year'=YYYY/'month'=MM/'day'=dd/'hour'=HH",
            "partition.duration.ms": "60000",
            "timezone": "UTC",
            "locale": "en-US",
            "schema.compatibility": "NONE"
        },
        "kafkaCluster": {
            "apacheKafkaCluster": {
                "bootstrapServers": "b-4.csmsksmallmskcluster.26tc9t.c2.kafka.us-east-1.amazonaws.com:9092,b-9.csmsksmallmskcluster.26tc9t.c2.kafka.us-east-1.amazonaws.com:9092,b-1.csmsksmallmskcluster.26tc9t.c2.kafka.us-east-1.amazonaws.com:9092",
                "vpc": {
                    "securityGroups": [
                        "sg-0e1ba465b935af7c5"
                    ],
                    "subnets": [
                        "subnet-003ce686f7c88605c",
                        "subnet-02996047b3c1cbcf3"
                    ]
                }
            }
        },
        "kafkaClusterClientAuthentication": {
            "authenticationType": "NONE"
        },
        "kafkaClusterEncryptionInTransit": {
            "encryptionType": "PLAINTEXT"
        },
        "kafkaConnectVersion": "2.7.1",
        "logDelivery": {
            "workerLogDelivery": {
                "s3": {
                    "bucket": "clickstream-infra-s3sink4dfdadf4-10ewmceey09vp",
                    "enabled": true,
                    "prefix": "sink-log"
                }
            }
        },
        "serviceExecutionRoleArn": "arn:aws:iam::080766874269:role/sink4-s3kafkaconnectroleAF13586E-1B9DIYD1M8WMD"
    }
}

Error message:

2023-02-10T03:02:14.940Z    1fb6006a-7004-41cf-aea7-d384af4ae95a    ERROR   BadRequestException: [format attribute "password" not supported, instance type (string) does not match any allowed primitive type (allowed: ["integer"]), format attribute "int32" not supported]
    at deserializeAws_restJson1BadRequestExceptionResponse (/var/runtime/node_modules/@aws-sdk/client-kafkaconnect/dist-cjs/protocols/Aws_restJson1.js:1037:23)
    at deserializeAws_restJson1CreateConnectorCommandError (/var/runtime/node_modules/@aws-sdk/client-kafkaconnect/dist-cjs/protocols/Aws_restJson1.js:306:25)
    at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
    at async /var/runtime/node_modules/@aws-sdk/middleware-serde/dist-cjs/deserializerMiddleware.js:7:24
    at async /var/runtime/node_modules/@aws-sdk/middleware-signing/dist-cjs/middleware.js:13:20
    at async StandardRetryStrategy.retry (/var/runtime/node_modules/@aws-sdk/middleware-retry/dist-cjs/StandardRetryStrategy.js:51:46)
    at async /var/runtime/node_modules/@aws-sdk/middleware-logger/dist-cjs/loggerMiddleware.js:6:22
    at async createConnector (/var/task/index.js:252:13)
    at async onCreate (/var/task/index.js:152:5)
    at async _handler (/var/task/index.js:110:5) {
  '$fault': 'client',
  '$metadata': {
    httpStatusCode: 400,
    requestId: 'df56485b-9a7f-4348-a4ff-113e8fea66f4',
    extendedRequestId: undefined,
    cfId: 'pIPzapexMStBwUPzfseJHslpbYoRFG7ykNiG2Z5PQ1qmTi0D3MWtiA==',
    attempts: 1,
    totalRetryDelay: 0
  }
}

Observed Behavior

Error message:

2023-02-10T03:02:14.940Z    1fb6006a-7004-41cf-aea7-d384af4ae95a    ERROR   BadRequestException: [format attribute "password" not supported, instance type (string) does not match any allowed primitive type (allowed: ["integer"]), format attribute "int32" not supported]
    at deserializeAws_restJson1BadRequestExceptionResponse (/var/runtime/node_modules/@aws-sdk/client-kafkaconnect/dist-cjs/protocols/Aws_restJson1.js:1037:23)
    at deserializeAws_restJson1CreateConnectorCommandError (/var/runtime/node_modules/@aws-sdk/client-kafkaconnect/dist-cjs/protocols/Aws_restJson1.js:306:25)
    at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
    at async /var/runtime/node_modules/@aws-sdk/middleware-serde/dist-cjs/deserializerMiddleware.js:7:24
    at async /var/runtime/node_modules/@aws-sdk/middleware-signing/dist-cjs/middleware.js:13:20
    at async StandardRetryStrategy.retry (/var/runtime/node_modules/@aws-sdk/middleware-retry/dist-cjs/StandardRetryStrategy.js:51:46)
    at async /var/runtime/node_modules/@aws-sdk/middleware-logger/dist-cjs/loggerMiddleware.js:6:22
    at async createConnector (/var/task/index.js:252:13)
    at async onCreate (/var/task/index.js:152:5)
    at async _handler (/var/task/index.js:110:5) {
  '$fault': 'client',
  '$metadata': {
    httpStatusCode: 400,
    requestId: 'df56485b-9a7f-4348-a4ff-113e8fea66f4',
    extendedRequestId: undefined,
    cfId: 'pIPzapexMStBwUPzfseJHslpbYoRFG7ykNiG2Z5PQ1qmTi0D3MWtiA==',
    attempts: 1,
    totalRetryDelay: 0
  }
}

Expected Behavior

the connector should be created successfully

Possible Solution

No response

Additional Information/Context

No response

amliuyong commented 1 year ago

below value should be int, not string

               "maxWorkerCount": "3",
                "mcuCount": "1",
                "minWorkerCount": "1",
github-actions[bot] commented 1 year ago

This thread has been automatically locked since there has not been any recent activity after it was closed. Please open a new issue for related bugs and link to relevant comments in this thread.