[CI] XPackRestIT test {p0=ml/data_frame_analytics_crud/Test get stats given multiple analytics} failing

[martijnvg]

Build scan: https://gradle-enterprise.elastic.co/s/nxecxmjlmwwr2/tests/:x-pack:plugin:yamlRestTest/org.elasticsearch.xpack.test.rest.XPackRestIT/test%20%7Bp0=ml%2Fdata_frame_analytics_crud%2FTest%20get%20stats%20given%20multiple%20analytics%7D

Reproduction line: ./gradlew ':x-pack:plugin:yamlRestTest' --tests "org.elasticsearch.xpack.test.rest.XPackRestIT.test {p0=ml/data_frame_analytics_crud/Test get stats given multiple analytics}" -Dtests.seed=565C69B0D75B366D -Dtests.security.manager=true -Dtests.locale=ar-BH -Dtests.timezone=Africa/Libreville -Druntime.java=11 -Dtests.rest.blacklist=getting_started/10_monitor_cluster_health/*

Applicable branches: 7.12

Reproduces locally?: No

Failure history: https://gradle-enterprise.elastic.co/scans/tests?tests.container=org.elasticsearch.xpack.test.rest.XPackRestIT&tests.test=test%20%7Bp0%3Dml/data_frame_analytics_crud/Test%20get%20stats%20given%20multiple%20analytics%7D

Failure excerpt:

java.lang.AssertionError: Failure at [ml/data_frame_analytics_crud:896]: expected [2xx] status code but api [ml.get_data_frame_analytics_stats] returned [500 Internal Server Error] [{"error":{"root_cause":[{"type":"no_shard_available_action_exception","reason":"[yamlRestTest-0][127.0.0.1:45375][indices:data/read/search[phase/query]]","index_uuid":"E_wG5w0PQB6n1bpzin7lyA","shard":"0","index":".ml-stats-000001","stack_trace":"[.ml-stats-000001/E_wG5w0PQB6n1bpzin7lyA][[.ml-stats-000001][0]] NoShardAvailableActionException[[yamlRestTest-0][127.0.0.1:45375][indices:data/read/search[phase/query]]]\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:473)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:422)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.access$000(AbstractSearchAsyncAction.java:62)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction$1.onFailure(AbstractSearchAsyncAction.java:289)\n\tat org.elasticsearch.action.search.SearchExecutionStatsCollector.onFailure(SearchExecutionStatsCollector.java:62)\n\tat org.elasticsearch.action.ActionListenerResponseHandler.handleException(ActionListenerResponseHandler.java:48)\n\tat org.elasticsearch.action.search.SearchTransportService$ConnectionCountingHandler.handleException(SearchTransportService.java:400)\n\tat org.elasticsearch.transport.TransportService$6.handleException(TransportService.java:745)\n\tat org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1290)\n\tat org.elasticsearch.transport.TransportService$DirectResponseChannel.processException(TransportService.java:1399)\n\tat org.elasticsearch.transport.TransportService$DirectResponseChannel.sendResponse(TransportService.java:1373)\n\tat org.elasticsearch.transport.TaskTransportChannel.sendResponse(TaskTransportChannel.java:50)\n\tat org.elasticsearch.transport.TransportChannel.sendErrorResponse(TransportChannel.java:45)\n\tat org.elasticsearch.action.support.ChannelActionListener.onFailure(ChannelActionListener.java:40)\n\tat org.elasticsearch.action.ActionRunnable.onFailure(ActionRunnable.java:77)\n\tat org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:28)\n\tat org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:33)\n\tat org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:732)\n\tat org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:834)\n"}],"type":"exception","reason":"all shards failed","caused_by":{"type":"search_phase_execution_exception","reason":"all shards failed","phase":"query","grouped":true,"failed_shards":[{"shard":0,"index":".ml-stats-000001","node":"L_Ul6u4mQGGAxtka6eX6Nw","reason":{"type":"no_shard_available_action_exception","reason":"[yamlRestTest-0][127.0.0.1:45375][indices:data/read/search[phase/query]]","index_uuid":"E_wG5w0PQB6n1bpzin7lyA","shard":"0","index":".ml-stats-000001","stack_trace":"[.ml-stats-000001/E_wG5w0PQB6n1bpzin7lyA][[.ml-stats-000001][0]] NoShardAvailableActionException[[yamlRestTest-0][127.0.0.1:45375][indices:data/read/search[phase/query]]]\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:473)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:422)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.access$000(AbstractSearchAsyncAction.java:62)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction$1.onFailure(AbstractSearchAsyncAction.java:289)\n\tat org.elasticsearch.action.search.SearchExecutionStatsCollector.onFailure(SearchExecutionStatsCollector.java:62)\n\tat org.elasticsearch.action.ActionListenerResponseHandler.handleException(ActionListenerResponseHandler.java:48)\n\tat org.elasticsearch.action.search.SearchTransportService$ConnectionCountingHandler.handleException(SearchTransportService.java:400)\n\tat org.elasticsearch.transport.TransportService$6.handleException(TransportService.java:745)\n\tat org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1290)\n\tat org.elasticsearch.transport.TransportService$DirectResponseChannel.processException(TransportService.java:1399)\n\tat org.elasticsearch.transport.TransportService$DirectResponseChannel.sendResponse(TransportService.java:1373)\n\tat org.elasticsearch.transport.TaskTransportChannel.sendResponse(TaskTransportChannel.java:50)\n\tat org.elasticsearch.transport.TransportChannel.sendErrorResponse(TransportChannel.java:45)\n\tat org.elasticsearch.action.support.ChannelActionListener.onFailure(ChannelActionListener.java:40)\n\tat org.elasticsearch.action.ActionRunnable.onFailure(ActionRunnable.java:77)\n\tat org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:28)\n\tat org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:33)\n\tat org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:732)\n\tat org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:834)\n"}}],"stack_trace":"Failed to execute phase [query], all shards failed; shardFailures {[L_Ul6u4mQGGAxtka6eX6Nw][.ml-stats-000001][0]: NoShardAvailableActionException[[yamlRestTest-0][127.0.0.1:45375][indices:data/read/search[phase/query]]]}\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseFailure(AbstractSearchAsyncAction.java:636)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.executeNextPhase(AbstractSearchAsyncAction.java:357)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseDone(AbstractSearchAsyncAction.java:669)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:440)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.access$000(AbstractSearchAsyncAction.java:62)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction$1.onFailure(AbstractSearchAsyncAction.java:289)\n\tat org.elasticsearch.action.search.SearchExecutionStatsCollector.onFailure(SearchExecutionStatsCollector.java:62)\n\tat org.elasticsearch.action.ActionListenerResponseHandler.handleException(ActionListenerResponseHandler.java:48)\n\tat org.elasticsearch.action.search.SearchTransportService$ConnectionCountingHandler.handleException(SearchTransportService.java:400)\n\tat org.elasticsearch.transport.TransportService$6.handleException(TransportService.java:745)\n\tat org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1290)\n\tat org.elasticsearch.transport.TransportService$DirectResponseChannel.processException(TransportService.java:1399)\n\tat org.elasticsearch.transport.TransportService$DirectResponseChannel.sendResponse(TransportService.java:1373)\n\tat org.elasticsearch.transport.TaskTransportChannel.sendResponse(TaskTransportChannel.java:50)\n\tat org.elasticsearch.transport.TransportChannel.sendErrorResponse(TransportChannel.java:45)\n\tat org.elasticsearch.action.support.ChannelActionListener.onFailure(ChannelActionListener.java:40)\n\tat org.elasticsearch.action.ActionRunnable.onFailure(ActionRunnable.java:77)\n\tat org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:28)\n\tat org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:33)\n\tat org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:732)\n\tat org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:834)\nCaused by: [.ml-stats-000001/E_wG5w0PQB6n1bpzin7lyA][[.ml-stats-000001][0]] NoShardAvailableActionException[[yamlRestTest-0][127.0.0.1:45375][indices:data/read/search[phase/query]]]\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:473)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:422)\n\t... 20 more\n"},"stack_trace":"ElasticsearchException[all shards failed]; nested: SearchPhaseExecutionException[all shards failed]; nested: NoShardAvailableActionException[[yamlRestTest-0][127.0.0.1:45375][indices:data/read/search[phase/query]]];\n\tat org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper.serverError(ExceptionsHelper.java:57)\n\tat org.elasticsearch.xpack.ml.action.TransportGetDataFrameAnalyticsStatsAction.lambda$searchStats$8(TransportGetDataFrameAnalyticsStatsAction.java:242)\n\tat org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:117)\n\tat org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:32)\n\tat org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:83)\n\tat org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:77)\n\tat org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:32)\n\tat org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.lambda$applyInternal$2(SecurityActionFilter.java:165)\n\tat org.elasticsearch.action.ActionListener$3.onResponse(ActionListener.java:167)\n\tat org.elasticsearch.action.search.TransportMultiSearchAction$1.finish(TransportMultiSearchAction.java:167)\n\tat org.elasticsearch.action.search.TransportMultiSearchAction$1.handleResponse(TransportMultiSearchAction.java:153)\n\tat org.elasticsearch.action.search.TransportMultiSearchAction$1.onFailure(TransportMultiSearchAction.java:146)\n\tat org.elasticsearch.action.support.TransportAction$1.onFailure(TransportAction.java:92)\n\tat org.elasticsearch.action.support.ContextPreservingActionListener.onFailure(ContextPreservingActionListener.java:39)\n\tat org.elasticsearch.action.ActionListener$3.onFailure(ActionListener.java:172)\n\tat org.elasticsearch.action.ActionListener$4.onFailure(ActionListener.java:262)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.raisePhaseFailure(AbstractSearchAsyncAction.java:660)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseFailure(AbstractSearchAsyncAction.java:636)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.executeNextPhase(AbstractSearchAsyncAction.java:357)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseDone(AbstractSearchAsyncAction.java:669)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:440)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.access$000(AbstractSearchAsyncAction.java:62)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction$1.onFailure(AbstractSearchAsyncAction.java:289)\n\tat org.elasticsearch.action.search.SearchExecutionStatsCollector.onFailure(SearchExecutionStatsCollector.java:62)\n\tat org.elasticsearch.action.ActionListenerResponseHandler.handleException(ActionListenerResponseHandler.java:48)\n\tat org.elasticsearch.action.search.SearchTransportService$ConnectionCountingHandler.handleException(SearchTransportService.java:400)\n\tat org.elasticsearch.transport.TransportService$6.handleException(TransportService.java:745)\n\tat org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1290)\n\tat org.elasticsearch.transport.TransportService$DirectResponseChannel.processException(TransportService.java:1399)\n\tat org.elasticsearch.transport.TransportService$DirectResponseChannel.sendResponse(TransportService.java:1373)\n\tat org.elasticsearch.transport.TaskTransportChannel.sendResponse(TaskTransportChannel.java:50)\n\tat org.elasticsearch.transport.TransportChannel.sendErrorResponse(TransportChannel.java:45)\n\tat org.elasticsearch.action.support.ChannelActionListener.onFailure(ChannelActionListener.java:40)\n\tat org.elasticsearch.action.ActionRunnable.onFailure(ActionRunnable.java:77)\n\tat org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:28)\n\tat org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:33)\n\tat org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:732)\n\tat org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:834)\nCaused by: Failed to execute phase [query], all shards failed; shardFailures {[L_Ul6u4mQGGAxtka6eX6Nw][.ml-stats-000001][0]: NoShardAvailableActionException[[yamlRestTest-0][127.0.0.1:45375][indices:data/read/search[phase/query]]]}\n\t... 24 more\nCaused by: [.ml-stats-000001/E_wG5w0PQB6n1bpzin7lyA][[.ml-stats-000001][0]] NoShardAvailableActionException[[yamlRestTest-0][127.0.0.1:45375][indices:data/read/search[phase/query]]]\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:473)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:422)\n\t... 20 more\n"},"status":500}]

  at __randomizedtesting.SeedInfo.seed([565C69B0D75B366D:DE08566A79A75B95]:0)
  at org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase.executeSection(ESClientYamlSuiteTestCase.java:451)
  at org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase.test(ESClientYamlSuiteTestCase.java:428)
  at jdk.internal.reflect.GeneratedMethodAccessor13.invoke(null:-1)
  at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  at java.lang.reflect.Method.invoke(Method.java:566)
  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
  at java.lang.Thread.run(Thread.java:834)

  Caused by: java.lang.AssertionError: expected [2xx] status code but api [ml.get_data_frame_analytics_stats] returned [500 Internal Server Error] [{"error":{"root_cause":[{"type":"no_shard_available_action_exception","reason":"[yamlRestTest-0][127.0.0.1:45375][indices:data/read/search[phase/query]]","index_uuid":"E_wG5w0PQB6n1bpzin7lyA","shard":"0","index":".ml-stats-000001","stack_trace":"[.ml-stats-000001/E_wG5w0PQB6n1bpzin7lyA][[.ml-stats-000001][0]] NoShardAvailableActionException[[yamlRestTest-0][127.0.0.1:45375][indices:data/read/search[phase/query]]]\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:473)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:422)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.access$000(AbstractSearchAsyncAction.java:62)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction$1.onFailure(AbstractSearchAsyncAction.java:289)\n\tat org.elasticsearch.action.search.SearchExecutionStatsCollector.onFailure(SearchExecutionStatsCollector.java:62)\n\tat org.elasticsearch.action.ActionListenerResponseHandler.handleException(ActionListenerResponseHandler.java:48)\n\tat org.elasticsearch.action.search.SearchTransportService$ConnectionCountingHandler.handleException(SearchTransportService.java:400)\n\tat org.elasticsearch.transport.TransportService$6.handleException(TransportService.java:745)\n\tat org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1290)\n\tat org.elasticsearch.transport.TransportService$DirectResponseChannel.processException(TransportService.java:1399)\n\tat org.elasticsearch.transport.TransportService$DirectResponseChannel.sendResponse(TransportService.java:1373)\n\tat org.elasticsearch.transport.TaskTransportChannel.sendResponse(TaskTransportChannel.java:50)\n\tat org.elasticsearch.transport.TransportChannel.sendErrorResponse(TransportChannel.java:45)\n\tat org.elasticsearch.action.support.ChannelActionListener.onFailure(ChannelActionListener.java:40)\n\tat org.elasticsearch.action.ActionRunnable.onFailure(ActionRunnable.java:77)\n\tat org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:28)\n\tat org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:33)\n\tat org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:732)\n\tat org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:834)\n"}],"type":"exception","reason":"all shards failed","caused_by":{"type":"search_phase_execution_exception","reason":"all shards failed","phase":"query","grouped":true,"failed_shards":[{"shard":0,"index":".ml-stats-000001","node":"L_Ul6u4mQGGAxtka6eX6Nw","reason":{"type":"no_shard_available_action_exception","reason":"[yamlRestTest-0][127.0.0.1:45375][indices:data/read/search[phase/query]]","index_uuid":"E_wG5w0PQB6n1bpzin7lyA","shard":"0","index":".ml-stats-000001","stack_trace":"[.ml-stats-000001/E_wG5w0PQB6n1bpzin7lyA][[.ml-stats-000001][0]] NoShardAvailableActionException[[yamlRestTest-0][127.0.0.1:45375][indices:data/read/search[phase/query]]]\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:473)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:422)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.access$000(AbstractSearchAsyncAction.java:62)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction$1.onFailure(AbstractSearchAsyncAction.java:289)\n\tat org.elasticsearch.action.search.SearchExecutionStatsCollector.onFailure(SearchExecutionStatsCollector.java:62)\n\tat org.elasticsearch.action.ActionListenerResponseHandler.handleException(ActionListenerResponseHandler.java:48)\n\tat org.elasticsearch.action.search.SearchTransportService$ConnectionCountingHandler.handleException(SearchTransportService.java:400)\n\tat org.elasticsearch.transport.TransportService$6.handleException(TransportService.java:745)\n\tat org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1290)\n\tat org.elasticsearch.transport.TransportService$DirectResponseChannel.processException(TransportService.java:1399)\n\tat org.elasticsearch.transport.TransportService$DirectResponseChannel.sendResponse(TransportService.java:1373)\n\tat org.elasticsearch.transport.TaskTransportChannel.sendResponse(TaskTransportChannel.java:50)\n\tat org.elasticsearch.transport.TransportChannel.sendErrorResponse(TransportChannel.java:45)\n\tat org.elasticsearch.action.support.ChannelActionListener.onFailure(ChannelActionListener.java:40)\n\tat org.elasticsearch.action.ActionRunnable.onFailure(ActionRunnable.java:77)\n\tat org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:28)\n\tat org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:33)\n\tat org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:732)\n\tat org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:834)\n"}}],"stack_trace":"Failed to execute phase [query], all shards failed; shardFailures {[L_Ul6u4mQGGAxtka6eX6Nw][.ml-stats-000001][0]: NoShardAvailableActionException[[yamlRestTest-0][127.0.0.1:45375][indices:data/read/search[phase/query]]]}\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseFailure(AbstractSearchAsyncAction.java:636)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.executeNextPhase(AbstractSearchAsyncAction.java:357)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseDone(AbstractSearchAsyncAction.java:669)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:440)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.access$000(AbstractSearchAsyncAction.java:62)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction$1.onFailure(AbstractSearchAsyncAction.java:289)\n\tat org.elasticsearch.action.search.SearchExecutionStatsCollector.onFailure(SearchExecutionStatsCollector.java:62)\n\tat org.elasticsearch.action.ActionListenerResponseHandler.handleException(ActionListenerResponseHandler.java:48)\n\tat org.elasticsearch.action.search.SearchTransportService$ConnectionCountingHandler.handleException(SearchTransportService.java:400)\n\tat org.elasticsearch.transport.TransportService$6.handleException(TransportService.java:745)\n\tat org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1290)\n\tat org.elasticsearch.transport.TransportService$DirectResponseChannel.processException(TransportService.java:1399)\n\tat org.elasticsearch.transport.TransportService$DirectResponseChannel.sendResponse(TransportService.java:1373)\n\tat org.elasticsearch.transport.TaskTransportChannel.sendResponse(TaskTransportChannel.java:50)\n\tat org.elasticsearch.transport.TransportChannel.sendErrorResponse(TransportChannel.java:45)\n\tat org.elasticsearch.action.support.ChannelActionListener.onFailure(ChannelActionListener.java:40)\n\tat org.elasticsearch.action.ActionRunnable.onFailure(ActionRunnable.java:77)\n\tat org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:28)\n\tat org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:33)\n\tat org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:732)\n\tat org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:834)\nCaused by: [.ml-stats-000001/E_wG5w0PQB6n1bpzin7lyA][[.ml-stats-000001][0]] NoShardAvailableActionException[[yamlRestTest-0][127.0.0.1:45375][indices:data/read/search[phase/query]]]\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:473)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:422)\n\t... 20 more\n"},"stack_trace":"ElasticsearchException[all shards failed]; nested: SearchPhaseExecutionException[all shards failed]; nested: NoShardAvailableActionException[[yamlRestTest-0][127.0.0.1:45375][indices:data/read/search[phase/query]]];\n\tat org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper.serverError(ExceptionsHelper.java:57)\n\tat org.elasticsearch.xpack.ml.action.TransportGetDataFrameAnalyticsStatsAction.lambda$searchStats$8(TransportGetDataFrameAnalyticsStatsAction.java:242)\n\tat org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:117)\n\tat org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:32)\n\tat org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:83)\n\tat org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:77)\n\tat org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:32)\n\tat org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.lambda$applyInternal$2(SecurityActionFilter.java:165)\n\tat org.elasticsearch.action.ActionListener$3.onResponse(ActionListener.java:167)\n\tat org.elasticsearch.action.search.TransportMultiSearchAction$1.finish(TransportMultiSearchAction.java:167)\n\tat org.elasticsearch.action.search.TransportMultiSearchAction$1.handleResponse(TransportMultiSearchAction.java:153)\n\tat org.elasticsearch.action.search.TransportMultiSearchAction$1.onFailure(TransportMultiSearchAction.java:146)\n\tat org.elasticsearch.action.support.TransportAction$1.onFailure(TransportAction.java:92)\n\tat org.elasticsearch.action.support.ContextPreservingActionListener.onFailure(ContextPreservingActionListener.java:39)\n\tat org.elasticsearch.action.ActionListener$3.onFailure(ActionListener.java:172)\n\tat org.elasticsearch.action.ActionListener$4.onFailure(ActionListener.java:262)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.raisePhaseFailure(AbstractSearchAsyncAction.java:660)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseFailure(AbstractSearchAsyncAction.java:636)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.executeNextPhase(AbstractSearchAsyncAction.java:357)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseDone(AbstractSearchAsyncAction.java:669)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:440)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.access$000(AbstractSearchAsyncAction.java:62)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction$1.onFailure(AbstractSearchAsyncAction.java:289)\n\tat org.elasticsearch.action.search.SearchExecutionStatsCollector.onFailure(SearchExecutionStatsCollector.java:62)\n\tat org.elasticsearch.action.ActionListenerResponseHandler.handleException(ActionListenerResponseHandler.java:48)\n\tat org.elasticsearch.action.search.SearchTransportService$ConnectionCountingHandler.handleException(SearchTransportService.java:400)\n\tat org.elasticsearch.transport.TransportService$6.handleException(TransportService.java:745)\n\tat org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1290)\n\tat org.elasticsearch.transport.TransportService$DirectResponseChannel.processException(TransportService.java:1399)\n\tat org.elasticsearch.transport.TransportService$DirectResponseChannel.sendResponse(TransportService.java:1373)\n\tat org.elasticsearch.transport.TaskTransportChannel.sendResponse(TaskTransportChannel.java:50)\n\tat org.elasticsearch.transport.TransportChannel.sendErrorResponse(TransportChannel.java:45)\n\tat org.elasticsearch.action.support.ChannelActionListener.onFailure(ChannelActionListener.java:40)\n\tat org.elasticsearch.action.ActionRunnable.onFailure(ActionRunnable.java:77)\n\tat org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:28)\n\tat org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:33)\n\tat org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:732)\n\tat org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:834)\nCaused by: Failed to execute phase [query], all shards failed; shardFailures {[L_Ul6u4mQGGAxtka6eX6Nw][.ml-stats-000001][0]: NoShardAvailableActionException[[yamlRestTest-0][127.0.0.1:45375][indices:data/read/search[phase/query]]]}\n\t... 24 more\nCaused by: [.ml-stats-000001/E_wG5w0PQB6n1bpzin7lyA][[.ml-stats-000001][0]] NoShardAvailableActionException[[yamlRestTest-0][127.0.0.1:45375][indices:data/read/search[phase/query]]]\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:473)\n\tat org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:422)\n\t... 20 more\n"},"status":500}]

    at org.junit.Assert.fail(Assert.java:88)
    at org.elasticsearch.test.rest.yaml.section.DoSection.execute(DoSection.java:285)
    at org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase.executeSection(ESClientYamlSuiteTestCase.java:444)
    at org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase.test(ESClientYamlSuiteTestCase.java:428)
    at jdk.internal.reflect.GeneratedMethodAccessor13.invoke(null:-1)
    at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:566)
    at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
    at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
    at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
    at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
    at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
    at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
    at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
    at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
    at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
    at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
    at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
    at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
    at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
    at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
    at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
    at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
    at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
    at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
    at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
    at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
    at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
    at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
    at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
    at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
    at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
    at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
    at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
    at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
    at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
    at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
    at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
    at java.lang.Thread.run(Thread.java:834)

[elasticmachine]

Pinging @elastic/ml-core (Team:ML)

[droberts195]

This is the dreaded "all shards failed" on an index that's just been created, like #40134 and summarised in #65846.

[droberts195]

Assigning low-risk as the test isn't muted and fails incredibly rarely.

[elasticsearchmachine]

This issue has been closed because it has been open for too long with no activity.

Any muted tests that were associated with this issue have been unmuted.

If the tests begin failing again, a new issue will be opened, and they may be muted again.