search

kgneng2 / blokg

blog
MIT License
0 stars 0 forks source link

kafka connector server 설정 #64

Open kgneng2 opened 2 years ago

kgneng2 commented 2 years ago

distributed

bootstrap.servers=
key.converter=org.apache.kafka.connect.storage.StringConverter
key.converter.schemas.enable=false
#key.converter=org.apache.kafka.connect.storage.StringConverter
#value.converter=io.confluent.connect.avro.AvroConverter

value.converter=org.apache.kafka.connect.json.JsonConverter
value.converter.schemas.enable=false

#key.converter.schema.registry.url=http://dev.schema.navercorp.com:8081
value.converter.schema.registry.url=http://dev.schema.navercorp.com:8081
group.id=test-velvet-connect-cluster

offset.storage.topic=test-velvet-connect-offsets
offset.storage.replication.factor=1

config.storage.topic=test-velvet-connect-configs
config.storage.replication.factor=1

status.storage.topic=test-velvet-connect-status
status.storage.replication.factor=1

offset.flush.interval.ms=10000

#plugin.path=/home1/irteam/apps/kafka-connector-hdfs/plugins
plugin.path=/home1/irteam/apps/confluent/share/java,/home1/irteam/apps/confluent/share/confluent-hub-components

access.control.allow.methods=GET,POST,PUT,OPTIONS
access.control.allow.origin=*

#topic.schema.ignore=true
#topic.key.ignore=true
#drop.invalid.message=true
#behavior.on.null.values=ignore
#behavior.on.malformed.documents=ignore

# error
errors.tolerance=all
erros.log.enable=true
errors.log.include.messages=true

worker

bootstrap.servers=
key.converter=org.apache.kafka.connect.storage.StringConverter
key.converter.schemas.enable=false
#key.converter=org.apache.kafka.connect.storage.StringConverter
#value.converter=io.confluent.connect.avro.AvroConverter

value.converter=org.apache.kafka.connect.json.JsonConverter
value.converter.schemas.enable=false

#key.converter.schema.registry.url=http://dev.schema.navercorp.com:8081/
value.converter.schema.registry.url=http://dev.schema.navercorp.com:8081
group.id=connect-cluster

offset.storage.topic=connect-offsets
offset.storage.replication.factor=1

config.storage.topic=connect-configs
config.storage.replication.factor=1

status.storage.topic=connect-status
status.storage.replication.factor=1

offset.flush.interval.ms=10000

#plugin.path=/home1/irteam/apps/kafka-connector-hdfs/plugins
plugin.path=/home1/irteam/apps/confluent-5.1.2/share/java,/home1/irteam/apps/confluent-5.1.2/share/confluent-hub-components

access.control.allow.methods=GET,POST,PUT,OPTIONS
access.control.allow.origin=*
kgneng2 commented 2 years ago

expose

{
  "clusterId": "bizcloud",
  "connectorId": "vshopping-log-expose-hdfs-sink",
  "description": "https://oss.navercorp.com/biz-platform/faucet-issues/issues/2",
  "managers": [
    "junyoung.kang@navercorp.com"
  ],
  "connector": {
    "type": "kafkaToHdfsWrite",
    "configuration": {
      "name": "vshopping-log-expose-hdfs-sink",
      "flush.size": "2000000",
      "rotate.schedule.interval.ms": "600000",
      "partition.duration.ms": "1800000",
      "tasks.max": "10",
      "topics": "vshopping-log-expose",
      "path.format": "yyyyMMdd/HH/mm/yyyyMMddHHmm",
      "hdfs.url": "hdfs://joy",
      "connector.class": "io.confluent.connect.hdfs.HdfsSinkConnector",
      "format.class": "io.confluent.connect.hdfs.parquet.ParquetFormat",
      "topics.dir": "/data/log/shopping/vshopping-log-expose-hdfs-sink",
      "logs.dir": "/data/kafka-connect/shopping/vshopping-log-expose-hdfs-sink",
      "hadoop.conf.dir": "/etc/hadoop/conf/",
      "schema.compatibility": "BACKWARD",
      "value.converter.schemas.enable": "false",
      "timezone": "Asia/Seoul",
      "locale": "kor",
      "hadoop.home": "/opt/cloudera/parcels/CDH/lib/hadoop",
      "partitioner.class": "com.navercorp.bizcloud.faucet.partitioner.UnixTimeBasedPartitioner",
      "timestamp.extractor": "RecordField",
      "timestamp.field": "exposeTimestamp"
    }
  }
}

clk

{
  "clusterId": "bizcloud",
  "connectorId": "vshopping-log-click-hdfs-sink",
  "description": "https://oss.navercorp.com/biz-platform/faucet-issues/issues/2",
  "managers": [
    "junyoung.kang@navercorp.com"
  ],
  "connector": {
    "type": "kafkaToHdfsWrite",
    "configuration": {
      "name": "vshopping-log-click-hdfs-sink",
      "flush.size": "50000",
      "rotate.schedule.interval.ms": "600000",
      "partition.duration.ms": "1800000",
      "tasks.max": "3",
      "topics": "vshopping-log-click",
      "path.format": "yyyyMMdd/HH/mm/yyyyMMddHHmm",
      "hdfs.url": "hdfs://joy",
      "connector.class": "io.confluent.connect.hdfs.HdfsSinkConnector",
      "format.class": "io.confluent.connect.hdfs.parquet.ParquetFormat",
      "topics.dir": "/data/log/shopping/vshopping-log-click-hdfs-sink",
      "logs.dir": "/data/kafka-connect/shopping/vshopping-log-click-hdfs-sink",
      "hadoop.conf.dir": "/etc/hadoop/conf/",
      "schema.compatibility": "BACKWARD",
      "value.converter.schemas.enable": "false",
      "timezone": "Asia/Seoul",
      "locale": "kor",
      "hadoop.home": "/opt/cloudera/parcels/CDH/lib/hadoop",
      "partitioner.class": "com.navercorp.bizcloud.faucet.partitioner.UnixTimeBasedPartitioner",
      "timestamp.extractor": "RecordField",
      "timestamp.field": "clkTimestamp"
    }
  }
}

aitems

{
  "clusterId": "bizcloud",
  "connectorId": "vshopping-aitems-hdfs-sink",
  "description": "https://oss.navercorp.com/biz-platform/faucet-issues/issues/9",
  "managers": [
    "junyoung.kang@navercorp.com"
  ],
  "connector": {
    "type": "kafkaToHdfsWrite",
    "configuration": {
      "name": "vshopping-aitems-hdfs-sink",
      "flush.size": "50000",
      "rotate.schedule.interval.ms": "600000",
      "partition.duration.ms": "1800000",
      "tasks.max": "3",
      "topics": "vshopping-aitems",
      "path.format": "yyyyMMdd/HH/mm/yyyyMMddHHmm",
      "hdfs.url": "hdfs://joy",
      "connector.class": "io.confluent.connect.hdfs.HdfsSinkConnector",
      "format.class": "io.confluent.connect.hdfs.parquet.ParquetFormat",
      "topics.dir": "/data/log/shopping/vshopping-aitems-hdfs-sink",
      "logs.dir": "/data/kafka-connect/shopping/vshopping-aitems-hdfs-sink",
      "hadoop.conf.dir": "/etc/hadoop/conf/",
      "schema.compatibility": "BACKWARD",
      "value.converter.schemas.enable": "false",
      "timezone": "Asia/Seoul",
      "locale": "kor",
      "hadoop.home": "/opt/cloudera/parcels/CDH/lib/hadoop",
      "partitioner.class": "com.navercorp.bizcloud.faucet.partitioner.UnixTimeBasedPartitioner",
      "timestamp.extractor": "RecordField",
      "timestamp.field": "timestamp"
    }
  }
}