ILM memo

[TomonoriSoejima]

How to reproduce

GET ipfixdata-2020.04.22-000041-reindex/_ilm/explain

{
  "indices" : {
    "ipfixdata-2020.04.22-000041-reindex" : {
      "index" : "ipfixdata-2020.04.22-000041-reindex",
      "managed" : true,
      "policy" : "ipfix_rollover",
      "lifecycle_date_millis" : 1590974359829,
      "age" : "5.07m",
      "phase" : "hot",
      "phase_time_millis" : 1590974654348,
      "action" : "rollover",
      "action_time_millis" : 1590974364536,
      "step" : "ERROR",
      "step_time_millis" : 1590974659346,
      "failed_step" : "check-rollover-ready",
      "is_auto_retryable_error" : true,
      "failed_step_retry_count" : 29,
      "step_info" : {
        "type" : "illegal_argument_exception",
        "reason" : "index.lifecycle.rollover_alias [ipfixdata] does not point to index [ipfixdata-2020.04.22-000041-reindex]",
        "stack_trace" : """java.lang.IllegalArgumentException: index.lifecycle.rollover_alias [ipfixdata] does not point to index [ipfixdata-2020.04.22-000041-reindex]
    at org.elasticsearch.xpack.core.ilm.WaitForRolloverReadyStep.evaluateCondition(WaitForRolloverReadyStep.java:104)
    at org.elasticsearch.xpack.ilm.IndexLifecycleRunner.runPeriodicStep(IndexLifecycleRunner.java:173)
    at org.elasticsearch.xpack.ilm.IndexLifecycleService.triggerPolicies(IndexLifecycleService.java:329)
    at org.elasticsearch.xpack.ilm.IndexLifecycleService.triggered(IndexLifecycleService.java:267)
    at org.elasticsearch.xpack.core.scheduler.SchedulerEngine.notifyListeners(SchedulerEngine.java:183)
    at org.elasticsearch.xpack.core.scheduler.SchedulerEngine$ActiveSchedule.run(SchedulerEngine.java:211)
    at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
    at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
    at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
    at java.base/java.lang.Thread.run(Thread.java:834)
"""
      },
      "phase_execution" : {
        "policy" : "ipfix_rollover",
        "phase_definition" : {
          "min_age" : "0ms",
          "actions" : {
            "rollover" : {
              "max_docs" : 2
            }
          }
        },
        "version" : 3,
        "modified_date_in_millis" : 1590974323183
      }
    }
  }
}

• update refresh interval to 5 sec to have the problem surface faster.

PUT /_cluster/settings
{
    "persistent" : {
        "indices.lifecycle.poll_interval": "5s"
    }
}

• create policy

PUT _ilm/policy/ipfix_rollover
{
  "policy": {
    "phases": {
      "hot": {                      
        "actions": {
          "rollover": {
            "max_docs": 2
          }
        }
      }
    }
  }
}

• create an index template

PUT _template/my_template
{
  "index_patterns": ["ipfixdata-*"],                 
  "settings": {
    "number_of_shards": 1,
    "number_of_replicas": 1,
    "index.lifecycle.name": "ipfix_rollover",      
    "index.lifecycle.rollover_alias": "ipfixdata"    
  }
}

• set alias ipfixdata to the wrong index

PUT wrong_index
{
  "aliases": {
    "ipfixdata": {
      "is_write_index": true
    }
  }
}

GET /_alias/ipfixdata

# result
{
  "wrong_index" : {
    "aliases" : {
      "ipfixdata" : {
        "is_write_index" : true
      }
    }
  }
}

• push documents

POST ipfixdata-2020.04.22-000041-reindex/_doc/1
{
  "name" : "a"
}

POST ipfixdata-2020.04.22-000041-reindex/_doc/2
{
  "name" : "a"
}

POST ipfixdata-2020.04.22-000041-reindex/_doc/3
{
  "name" : "a"
}

GET ipfixdata-2020.04.22-000041-reindex/_ilm/explain should now reveal the error.

https://gist.github.com/TomonoriSoejima/e13a2459515ee84c68c94a222706cf1a is the replayable script.

[TomonoriSoejima]

create strong_user and login to kibana as this user.

PUT _security/role/super_duper_user
{
  "cluster" : [
    "all"
  ],
  "indices" : [
    {
      "names" : [
        "*"
      ],
      "privileges" : [
        "all"
      ],
      "field_security" : {
        "grant" : [
          "*"
        ],
        "except" : [ ]
      },
      "allow_restricted_indices" : true
    }
  ],
  "applications" : [ ],
  "run_as" : [ ],
  "metadata" : { },
  "transient_metadata" : {
    "enabled" : true
  }
}

PUT _security/user/strong_user
{

    "username": "strong_user",
    "password" : "password",
    "roles": [
      "super_duper_user",
      "superuser"
    ],
    "full_name": "",
    "email": "",
    "metadata": {},
    "enabled": true

}