search

elastic / elasticsearch

Free and Open Source, Distributed, RESTful Search Engine
https://www.elastic.co/products/elasticsearch
Other
69.6k stars 24.63k forks source link

Improve Ingest Pipelines comparison when applying cluster state updates #97382

Open tlrx opened 1 year ago

tlrx commented 1 year ago

Today the list of processors of ingest pipelines is compared during cluster state updates in order to detect if the pipeline changed. In a cluster with a large number of ingest pipelines (1500+) with some of them having hundreds of processors (+300) parsing and comparing the list of processor can be inefficient and can slow down cluster state update applying.

This shows up in Node Hot Threads like this:

   100.3% [cpu=100.3%, other=0.0%] (501.5ms out of 500ms) cpu usage by thread 'elasticsearch[instance-0000000001][clusterApplierService#updateTask][T#1]'
     2/10 snapshots sharing following 34 elements
       org.yaml.snakeyaml@1.33/org.yaml.snakeyaml.scanner.ScannerImpl.checkToken(ScannerImpl.java:263)
       org.yaml.snakeyaml@1.33/org.yaml.snakeyaml.parser.ParserImpl$ParseBlockMappingValue.produce(ParserImpl.java:694)
       org.yaml.snakeyaml@1.33/org.yaml.snakeyaml.parser.ParserImpl.peekEvent(ParserImpl.java:185)
       org.yaml.snakeyaml@1.33/org.yaml.snakeyaml.parser.ParserImpl.getEvent(ParserImpl.java:195)
       com.fasterxml.jackson.dataformat.yaml@2.14.2/com.fasterxml.jackson.dataformat.yaml.YAMLParser.nextToken(YAMLParser.java:418)
       org.elasticsearch.xcontent.impl@8.7.1/org.elasticsearch.xcontent.provider.json.JsonXContentParser.nextToken(JsonXContentParser.java:58)
       app/org.elasticsearch.xcontent@8.7.1/org.elasticsearch.xcontent.support.AbstractXContentParser.readMapEntries(AbstractXContentParser.java:322)
       app/org.elasticsearch.xcontent@8.7.1/org.elasticsearch.xcontent.support.AbstractXContentParser.readValueUnsafe(AbstractXContentParser.java:400)
       app/org.elasticsearch.xcontent@8.7.1/org.elasticsearch.xcontent.support.AbstractXContentParser.readMapEntries(AbstractXContentParser.java:322)
       app/org.elasticsearch.xcontent@8.7.1/org.elasticsearch.xcontent.support.AbstractXContentParser.readValueUnsafe(AbstractXContentParser.java:400)
       app/org.elasticsearch.xcontent@8.7.1/org.elasticsearch.xcontent.support.AbstractXContentParser.readListUnsafe(AbstractXContentParser.java:370)
       app/org.elasticsearch.xcontent@8.7.1/org.elasticsearch.xcontent.support.AbstractXContentParser.readValueUnsafe(AbstractXContentParser.java:403)
       app/org.elasticsearch.xcontent@8.7.1/org.elasticsearch.xcontent.support.AbstractXContentParser.readMapEntries(AbstractXContentParser.java:322)
       app/org.elasticsearch.xcontent@8.7.1/org.elasticsearch.xcontent.support.AbstractXContentParser.readMapSafe(AbstractXContentParser.java:312)
       app/org.elasticsearch.xcontent@8.7.1/org.elasticsearch.xcontent.support.AbstractXContentParser.mapOrdered(AbstractXContentParser.java:268)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:271)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:178)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:141)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.ingest.PipelineConfiguration.getConfigAsMap(PipelineConfiguration.java:87)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.ingest.PipelineConfiguration.equals(PipelineConfiguration.java:153)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.ingest.IngestService.innerUpdatePipelines(IngestService.java:1056)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.ingest.IngestService.applyClusterState(IngestService.java:1040)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:537)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:524)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.cluster.service.ClusterApplierService.applyChanges(ClusterApplierService.java:496)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.cluster.service.ClusterApplierService.runTask(ClusterApplierService.java:427)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.cluster.service.ClusterApplierService$UpdateTask.run(ClusterApplierService.java:155)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:891)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:257)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:223)
       java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
       java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
       java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
       java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
     2/10 snapshots sharing following 38 elements
       org.yaml.snakeyaml@1.33/org.yaml.snakeyaml.scanner.ScannerImpl.checkToken(ScannerImpl.java:263)
       org.yaml.snakeyaml@1.33/org.yaml.snakeyaml.parser.ParserImpl$ParseBlockMappingValue.produce(ParserImpl.java:694)
       org.yaml.snakeyaml@1.33/org.yaml.snakeyaml.parser.ParserImpl.peekEvent(ParserImpl.java:185)
       org.yaml.snakeyaml@1.33/org.yaml.snakeyaml.parser.ParserImpl.getEvent(ParserImpl.java:195)
       com.fasterxml.jackson.dataformat.yaml@2.14.2/com.fasterxml.jackson.dataformat.yaml.YAMLParser.nextToken(YAMLParser.java:418)
       org.elasticsearch.xcontent.impl@8.7.1/org.elasticsearch.xcontent.provider.json.JsonXContentParser.nextToken(JsonXContentParser.java:58)
       app/org.elasticsearch.xcontent@8.7.1/org.elasticsearch.xcontent.support.AbstractXContentParser.readMapEntries(AbstractXContentParser.java:322)
       app/org.elasticsearch.xcontent@8.7.1/org.elasticsearch.xcontent.support.AbstractXContentParser.readValueUnsafe(AbstractXContentParser.java:400)
       app/org.elasticsearch.xcontent@8.7.1/org.elasticsearch.xcontent.support.AbstractXContentParser.readListUnsafe(AbstractXContentParser.java:370)
       app/org.elasticsearch.xcontent@8.7.1/org.elasticsearch.xcontent.support.AbstractXContentParser.readValueUnsafe(AbstractXContentParser.java:403)
       app/org.elasticsearch.xcontent@8.7.1/org.elasticsearch.xcontent.support.AbstractXContentParser.readMapEntries(AbstractXContentParser.java:322)
       app/org.elasticsearch.xcontent@8.7.1/org.elasticsearch.xcontent.support.AbstractXContentParser.readValueUnsafe(AbstractXContentParser.java:400)
       app/org.elasticsearch.xcontent@8.7.1/org.elasticsearch.xcontent.support.AbstractXContentParser.readMapEntries(AbstractXContentParser.java:322)
       app/org.elasticsearch.xcontent@8.7.1/org.elasticsearch.xcontent.support.AbstractXContentParser.readValueUnsafe(AbstractXContentParser.java:400)
       app/org.elasticsearch.xcontent@8.7.1/org.elasticsearch.xcontent.support.AbstractXContentParser.readListUnsafe(AbstractXContentParser.java:370)
       app/org.elasticsearch.xcontent@8.7.1/org.elasticsearch.xcontent.support.AbstractXContentParser.readValueUnsafe(AbstractXContentParser.java:403)
       app/org.elasticsearch.xcontent@8.7.1/org.elasticsearch.xcontent.support.AbstractXContentParser.readMapEntries(AbstractXContentParser.java:322)
       app/org.elasticsearch.xcontent@8.7.1/org.elasticsearch.xcontent.support.AbstractXContentParser.readMapSafe(AbstractXContentParser.java:312)
       app/org.elasticsearch.xcontent@8.7.1/org.elasticsearch.xcontent.support.AbstractXContentParser.mapOrdered(AbstractXContentParser.java:268)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:271)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:178)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:141)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.ingest.PipelineConfiguration.getConfigAsMap(PipelineConfiguration.java:87)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.ingest.PipelineConfiguration.equals(PipelineConfiguration.java:153)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.ingest.IngestService.innerUpdatePipelines(IngestService.java:1056)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.ingest.IngestService.applyClusterState(IngestService.java:1040)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:537)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:524)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.cluster.service.ClusterApplierService.applyChanges(ClusterApplierService.java:496)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.cluster.service.ClusterApplierService.runTask(ClusterApplierService.java:427)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.cluster.service.ClusterApplierService$UpdateTask.run(ClusterApplierService.java:155)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:891)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:257)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:223)
       java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
       java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
       java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
       java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
     2/10 snapshots sharing following 35 elements
       org.yaml.snakeyaml@1.33/org.yaml.snakeyaml.scanner.ScannerImpl.checkToken(ScannerImpl.java:263)
       org.yaml.snakeyaml@1.33/org.yaml.snakeyaml.parser.ParserImpl$ParseBlockMappingKey.produce(ParserImpl.java:662)
       org.yaml.snakeyaml@1.33/org.yaml.snakeyaml.parser.ParserImpl.peekEvent(ParserImpl.java:185)
       org.yaml.snakeyaml@1.33/org.yaml.snakeyaml.parser.ParserImpl.getEvent(ParserImpl.java:195)
       com.fasterxml.jackson.dataformat.yaml@2.14.2/com.fasterxml.jackson.dataformat.yaml.YAMLParser.nextToken(YAMLParser.java:418)
       com.fasterxml.jackson.core@2.14.2/com.fasterxml.jackson.core.JsonParser.nextFieldName(JsonParser.java:1065)
       org.elasticsearch.xcontent.impl@8.7.1/org.elasticsearch.xcontent.provider.json.JsonXContentParser.nextFieldName(JsonXContentParser.java:69)
       app/org.elasticsearch.xcontent@8.7.1/org.elasticsearch.xcontent.support.AbstractXContentParser.readMapEntries(AbstractXContentParser.java:324)
       app/org.elasticsearch.xcontent@8.7.1/org.elasticsearch.xcontent.support.AbstractXContentParser.readValueUnsafe(AbstractXContentParser.java:400)
       app/org.elasticsearch.xcontent@8.7.1/org.elasticsearch.xcontent.support.AbstractXContentParser.readMapEntries(AbstractXContentParser.java:322)
       app/org.elasticsearch.xcontent@8.7.1/org.elasticsearch.xcontent.support.AbstractXContentParser.readValueUnsafe(AbstractXContentParser.java:400)
       app/org.elasticsearch.xcontent@8.7.1/org.elasticsearch.xcontent.support.AbstractXContentParser.readListUnsafe(AbstractXContentParser.java:370)
       app/org.elasticsearch.xcontent@8.7.1/org.elasticsearch.xcontent.support.AbstractXContentParser.readValueUnsafe(AbstractXContentParser.java:403)
       app/org.elasticsearch.xcontent@8.7.1/org.elasticsearch.xcontent.support.AbstractXContentParser.readMapEntries(AbstractXContentParser.java:322)
       app/org.elasticsearch.xcontent@8.7.1/org.elasticsearch.xcontent.support.AbstractXContentParser.readMapSafe(AbstractXContentParser.java:312)
       app/org.elasticsearch.xcontent@8.7.1/org.elasticsearch.xcontent.support.AbstractXContentParser.mapOrdered(AbstractXContentParser.java:268)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:271)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:178)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:141)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.ingest.PipelineConfiguration.getConfigAsMap(PipelineConfiguration.java:87)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.ingest.PipelineConfiguration.equals(PipelineConfiguration.java:153)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.ingest.IngestService.innerUpdatePipelines(IngestService.java:1056)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.ingest.IngestService.applyClusterState(IngestService.java:1040)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:537)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:524)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.cluster.service.ClusterApplierService.applyChanges(ClusterApplierService.java:496)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.cluster.service.ClusterApplierService.runTask(ClusterApplierService.java:427)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.cluster.service.ClusterApplierService$UpdateTask.run(ClusterApplierService.java:155)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:891)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:257)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:223)
       java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
       java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
       java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
       java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
     3/10 snapshots sharing following 23 elements
       app/org.elasticsearch.xcontent@8.7.1/org.elasticsearch.xcontent.support.AbstractXContentParser.readValueUnsafe(AbstractXContentParser.java:403)
       app/org.elasticsearch.xcontent@8.7.1/org.elasticsearch.xcontent.support.AbstractXContentParser.readMapEntries(AbstractXContentParser.java:322)
       app/org.elasticsearch.xcontent@8.7.1/org.elasticsearch.xcontent.support.AbstractXContentParser.readMapSafe(AbstractXContentParser.java:312)
       app/org.elasticsearch.xcontent@8.7.1/org.elasticsearch.xcontent.support.AbstractXContentParser.mapOrdered(AbstractXContentParser.java:268)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:271)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:178)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:141)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.ingest.PipelineConfiguration.getConfigAsMap(PipelineConfiguration.java:87)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.ingest.PipelineConfiguration.equals(PipelineConfiguration.java:153)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.ingest.IngestService.innerUpdatePipelines(IngestService.java:1056)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.ingest.IngestService.applyClusterState(IngestService.java:1040)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:537)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:524)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.cluster.service.ClusterApplierService.applyChanges(ClusterApplierService.java:496)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.cluster.service.ClusterApplierService.runTask(ClusterApplierService.java:427)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.cluster.service.ClusterApplierService$UpdateTask.run(ClusterApplierService.java:155)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:891)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:257)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:223)
       java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
       java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
       java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
       java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
     unique snapshot
       org.yaml.snakeyaml@1.33/org.yaml.snakeyaml.resolver.Resolver.addImplicitResolver(Resolver.java:102)
       org.yaml.snakeyaml@1.33/org.yaml.snakeyaml.resolver.Resolver.addImplicitResolvers(Resolver.java:69)
       org.yaml.snakeyaml@1.33/org.yaml.snakeyaml.resolver.Resolver.<init>(Resolver.java:77)
       com.fasterxml.jackson.dataformat.yaml@2.14.2/com.fasterxml.jackson.dataformat.yaml.YAMLParser.<init>(YAMLParser.java:113)
       com.fasterxml.jackson.dataformat.yaml@2.14.2/com.fasterxml.jackson.dataformat.yaml.YAMLFactory._createParser(YAMLFactory.java:524)
       com.fasterxml.jackson.dataformat.yaml@2.14.2/com.fasterxml.jackson.dataformat.yaml.YAMLFactory.createParser(YAMLFactory.java:449)
       org.elasticsearch.xcontent.impl@8.7.1/org.elasticsearch.xcontent.provider.yaml.YamlXContentImpl.createParser(YamlXContentImpl.java:90)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:264)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:178)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:141)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.ingest.PipelineConfiguration.getConfigAsMap(PipelineConfiguration.java:87)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.ingest.PipelineConfiguration.equals(PipelineConfiguration.java:153)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.ingest.IngestService.innerUpdatePipelines(IngestService.java:1056)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.ingest.IngestService.applyClusterState(IngestService.java:1040)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:537)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:524)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.cluster.service.ClusterApplierService.applyChanges(ClusterApplierService.java:496)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.cluster.service.ClusterApplierService.runTask(ClusterApplierService.java:427)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.cluster.service.ClusterApplierService$UpdateTask.run(ClusterApplierService.java:155)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:891)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:257)
       app/org.elasticsearch.server@8.7.1/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:223)
       java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
       java.base@20.0.1/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
       java.base@20.0.1/java.lang.Thread.runWith(Thread.java:1636)
       java.base@20.0.1/java.lang.Thread.run(Thread.java:1623)
elasticsearchmachine commented 1 year ago

Pinging @elastic/es-data-management (Team:Data Management)