Logstash 迁移ES 数据

  elasticsearch {
    hosts => "es.production.mysite.org"
    index => "mydata-2018.09.*"
    query => '{ "query": { "query_string": { "query": "*" } } }'
    size => 500
    scroll => "5m"
    docinfo => true
  }

input {
  elasticsearch {
    hosts => "9200"
    index => "-*-2019.06.01"
    query => '{ "query": { "query_string": { "query": "sid:110093789 OR  sid:100551383 OR  sid:183366188 OR  sid:112318502 OR  sid:113416916 OR  sid:486897964 OR  sid:113234331 OR  sid:113238288 OR  sid:566578383 OR  sid:3000001 OR  sid:117548108 OR  sid:300039451 OR  sid:101496588 OR  sid:108644974 OR  sid:111041057 OR  sid:117726953 OR  sid:64246482 OR  sid:69153752 OR  sid:105720053 OR  sid:106821734 OR  sid:63716160 OR  sid:412284613 OR  sid:101190526 OR  sid:162847549 OR  sid:110093789 OR  sid:261991723 OR  sid: 114661967 OR  sid: 57302891" } } }'
    size => 500
    scroll => "5m"
    docinfo => true

  }
}
filter {
  mutate {
    copy => { "[@metadata][_index]" => "_index" }
    copy => {  "[@metadata][_type]" =>  "_type" }
    copy => { "[@metadata][_id]" => "_id" }
 }
}
output {
  file {
  path => "/data/%{[@metadata][_index]}"
}
}

input {
  file {
    path => "/data/-*-2019.06.01"
    start_position => "beginning"
    codec => json
  }
}
filter {
  mutate {
    rename => {"_id" => "[@metadata][_id]" }
    rename => {"_type" =>  "[@metadata][_type]" }
    rename => {"_index" =>  "[@metadata][_index]" }
    remove_field => [ "@version", "path", "host"]
  }
}
output {
  elasticsearch {
  hosts => "es-.elasticsearch.aliyuncs.com:9200"
  index => "%{[@metadata][_index]}"
  document_type => "%{[@metadata][_type]}"
  document_id => "%{[@metadata][_id]}"
 }
}

arterhuo / blog

Logstash 迁移ES 数据 #9