alibaba / DataX

DataX是阿里云DataWorks数据集成的开源版本。
Other
15.83k stars 5.4k forks source link

Feat: 支持kafkawriter插件 #2147

Open thaiyang2009 opened 3 months ago

thaiyang2009 commented 3 months ago

基于: org.apache.kafka:kafka-clients:2.5.1

配置示例:

{
  "job": {
    "content": [
      {
        "reader": {
          "name": "streamreader",
          "parameter": {
            "sliceRecordCount": 10,
            "column": [
              {
                "type": "long",
                "value": "10"
              },
              {
                "type": "string",
                "value": "hello,你好,世界-DataX"
              },
              {
                "type": "date",
                "value": "2014-07-07 00:00:00"
              }
            ]
          }
        },
        "writer": {
          "name": "kafkawriter",
          "parameter": {
            "bootstrapServers": "localhost:9092",
            "topic": "ods_stream_test",
            "keyIndex": 0,
            "column": [ // json对象字段
              "id",
              "content",
              "date"
            ],
            "props": {  // kafka配置参数
              "acks": "1"
            }
          }
        }
      }
    ],
    "setting": {
      "speed": {
        "channel": 1
      }
    }
  }
}

Kafka消息: key: 10 value: {"date":"2014-07-07 00:00:00","content":"hello,你好,世界-DataX","id":"10"}