Closed cbharathnoor closed 5 years ago
Pinging @elastic/es-distributed
Can you please check that the azure plugin is correctly installed an all nodes (specially 10.244.3.11:9300) as it seems that the type of repository azure
is unknown from an elasticsearch node.
You should check for the presence of the azure plugin in the plugins/
directory on all nodes as well as through the Nodes Info API using https://www.elastic.co/guide/en/elasticsearch/reference/2.4/cluster-nodes-info.html
The above mentioned IP (10.244.3.11) is a master node. I have cross verified the plugins, it looks fine for me. Please find the reference screenshot attached.
I have also got the json data with the help of node info api,
{
"cluster_name" : "myesdb",
"nodes" : {
"XHoVvecFRRm6P7DR1aKyUg" : {
"name" : "Living Totem",
"transport_address" : "10.244.3.12:9300",
"host" : "10.244.3.12",
"ip" : "10.244.3.12",
"version" : "2.4.0",
"build" : "ce9f0c7",
"http_address" : "10.244.3.12:9200",
"attributes" : {
"master" : "false"
},
"settings" : {
"cloud" : {
"kubernetes" : {
"namespace" : "default",
"service" : "elasticsearch-discovery"
}
},
"cluster" : {
"name" : "myesdb"
},
"node" : {
"data" : "true",
"master" : "false"
},
"path" : {
"data" : "/data/data",
"logs" : "/data/log",
"plugins" : "/elasticsearch/plugins",
"work" : "/data/work",
"home" : "/elasticsearch"
},
"discovery" : {
"type" : "kubernetes",
"zen" : {
"minimum_master_nodes" : "1"
}
},
"name" : "Living Totem",
"client" : {
"type" : "node"
},
"http" : {
"compression" : "true",
"enabled" : "true",
"cors" : {
"allow-origin" : "*",
"enabled" : "true"
}
},
"index" : {
"number_of_shards" : "1",
"number_of_replicas" : "0"
},
"bootstrap" : {
"mlockall" : "true"
},
"config" : {
"ignore_system_properties" : "true"
},
"network" : {
"host" : "_eth0_"
}
},
"os" : {
"refresh_interval_in_millis" : 1000,
"name" : "Linux",
"arch" : "amd64",
"version" : "4.15.0-1023-azure",
"available_processors" : 2,
"allocated_processors" : 2
},
"process" : {
"refresh_interval_in_millis" : 1000,
"id" : 105,
"mlockall" : true
},
"jvm" : {
"pid" : 105,
"version" : "1.8.0_92-internal",
"vm_name" : "OpenJDK 64-Bit Server VM",
"vm_version" : "25.92-b14",
"vm_vendor" : "Oracle Corporation",
"start_time_in_millis" : 1542177903862,
"mem" : {
"heap_init_in_bytes" : 1073741824,
"heap_max_in_bytes" : 1056309248,
"non_heap_init_in_bytes" : 2555904,
"non_heap_max_in_bytes" : 0,
"direct_max_in_bytes" : 1056309248
},
"gc_collectors" : [ "ParNew", "ConcurrentMarkSweep" ],
"memory_pools" : [ "Code Cache", "Metaspace", "Compressed Class Space", "Par Eden Space", "Par Survivor Space", "CMS Old Gen" ],
"using_compressed_ordinary_object_pointers" : "true"
},
"thread_pool" : {
"force_merge" : {
"type" : "fixed",
"min" : 1,
"max" : 1,
"queue_size" : -1
},
"percolate" : {
"type" : "fixed",
"min" : 2,
"max" : 2,
"queue_size" : 1000
},
"fetch_shard_started" : {
"type" : "scaling",
"min" : 1,
"max" : 4,
"keep_alive" : "5m",
"queue_size" : -1
},
"listener" : {
"type" : "fixed",
"min" : 1,
"max" : 1,
"queue_size" : -1
},
"index" : {
"type" : "fixed",
"min" : 2,
"max" : 2,
"queue_size" : 200
},
"refresh" : {
"type" : "scaling",
"min" : 1,
"max" : 1,
"keep_alive" : "5m",
"queue_size" : -1
},
"suggest" : {
"type" : "fixed",
"min" : 2,
"max" : 2,
"queue_size" : 1000
},
"generic" : {
"type" : "cached",
"keep_alive" : "30s",
"queue_size" : -1
},
"warmer" : {
"type" : "scaling",
"min" : 1,
"max" : 1,
"keep_alive" : "5m",
"queue_size" : -1
},
"search" : {
"type" : "fixed",
"min" : 4,
"max" : 4,
"queue_size" : 1000
},
"flush" : {
"type" : "scaling",
"min" : 1,
"max" : 1,
"keep_alive" : "5m",
"queue_size" : -1
},
"fetch_shard_store" : {
"type" : "scaling",
"min" : 1,
"max" : 4,
"keep_alive" : "5m",
"queue_size" : -1
},
"management" : {
"type" : "scaling",
"min" : 1,
"max" : 5,
"keep_alive" : "5m",
"queue_size" : -1
},
"get" : {
"type" : "fixed",
"min" : 2,
"max" : 2,
"queue_size" : 1000
},
"bulk" : {
"type" : "fixed",
"min" : 2,
"max" : 2,
"queue_size" : 50
},
"snapshot" : {
"type" : "scaling",
"min" : 1,
"max" : 1,
"keep_alive" : "5m",
"queue_size" : -1
}
},
"transport" : {
"bound_address" : [ "10.244.3.12:9300", "[fe80::9472:30ff:fe0c:d33]:9300" ],
"publish_address" : "10.244.3.12:9300",
"profiles" : { }
},
"http" : {
"bound_address" : [ "10.244.3.12:9200", "[fe80::9472:30ff:fe0c:d33]:9200" ],
"publish_address" : "10.244.3.12:9200",
"max_content_length_in_bytes" : 104857600
},
"plugins" : [ {
"name" : "cloud-azure",
"version" : "2.4.0",
"description" : "The Azure Cloud plugin allows to use Azure API for the unicast discovery mechanism and add Azure storage repositories.",
"jvm" : true,
"classname" : "org.elasticsearch.plugin.cloud.azure.CloudAzurePlugin",
"isolated" : true,
"site" : false
}, {
"name" : "cloud-kubernetes",
"version" : "2.4.0_01",
"description" : "Elasticsearch Kubernetes cloud plugin",
"jvm" : true,
"classname" : "io.fabric8.elasticsearch.plugin.discovery.kubernetes.KubernetesDiscoveryPlugin",
"isolated" : true,
"site" : false
}, {
"name" : "prometheus-exporter",
"version" : "2.4.0.0",
"description" : "Prometheus Exporter",
"jvm" : true,
"classname" : "org.elasticsearch.plugin.prometheus.PrometheusExporterPlugin",
"isolated" : false,
"site" : false
} ],
"modules" : [ {
"name" : "lang-expression",
"version" : "2.4.0",
"description" : "Lucene expressions integration for Elasticsearch",
"jvm" : true,
"classname" : "org.elasticsearch.script.expression.ExpressionPlugin",
"isolated" : true,
"site" : false
}, {
"name" : "lang-groovy",
"version" : "2.4.0",
"description" : "Groovy scripting integration for Elasticsearch",
"jvm" : true,
"classname" : "org.elasticsearch.script.groovy.GroovyPlugin",
"isolated" : true,
"site" : false
}, {
"name" : "reindex",
"version" : "2.4.0",
"description" : "_reindex and _update_by_query APIs",
"jvm" : true,
"classname" : "org.elasticsearch.index.reindex.ReindexPlugin",
"isolated" : true,
"site" : false
} ]
},
"GqpgZBFCSxqh61dwXrMdhg" : {
"name" : "Hydro",
"transport_address" : "10.244.0.11:9300",
"host" : "10.244.0.11",
"ip" : "10.244.0.11",
"version" : "2.4.0",
"build" : "ce9f0c7",
"http_address" : "10.244.0.11:9200",
"attributes" : {
"data" : "false",
"master" : "false"
},
"settings" : {
"cloud" : {
"kubernetes" : {
"namespace" : "default",
"service" : "elasticsearch-discovery"
}
},
"cluster" : {
"name" : "myesdb"
},
"node" : {
"data" : "false",
"master" : "false"
},
"path" : {
"data" : "/data/data",
"logs" : "/data/log",
"plugins" : "/elasticsearch/plugins",
"work" : "/data/work",
"home" : "/elasticsearch"
},
"discovery" : {
"type" : "kubernetes",
"zen" : {
"minimum_master_nodes" : "1"
}
},
"name" : "Hydro",
"client" : {
"type" : "node"
},
"http" : {
"compression" : "true",
"enabled" : "true",
"cors" : {
"allow-origin" : "*",
"enabled" : "true"
}
},
"index" : {
"number_of_shards" : "1",
"number_of_replicas" : "0"
},
"bootstrap" : {
"mlockall" : "true"
},
"config" : {
"ignore_system_properties" : "true"
},
"network" : {
"host" : "_eth0_"
}
},
"os" : {
"refresh_interval_in_millis" : 1000,
"name" : "Linux",
"arch" : "amd64",
"version" : "4.15.0-1023-azure",
"available_processors" : 2,
"allocated_processors" : 2
},
"process" : {
"refresh_interval_in_millis" : 1000,
"id" : 105,
"mlockall" : true
},
"jvm" : {
"pid" : 105,
"version" : "1.8.0_92-internal",
"vm_name" : "OpenJDK 64-Bit Server VM",
"vm_version" : "25.92-b14",
"vm_vendor" : "Oracle Corporation",
"start_time_in_millis" : 1542177851167,
"mem" : {
"heap_init_in_bytes" : 1073741824,
"heap_max_in_bytes" : 1056309248,
"non_heap_init_in_bytes" : 2555904,
"non_heap_max_in_bytes" : 0,
"direct_max_in_bytes" : 1056309248
},
"gc_collectors" : [ "ParNew", "ConcurrentMarkSweep" ],
"memory_pools" : [ "Code Cache", "Metaspace", "Compressed Class Space", "Par Eden Space", "Par Survivor Space", "CMS Old Gen" ],
"using_compressed_ordinary_object_pointers" : "true"
},
"thread_pool" : {
"force_merge" : {
"type" : "fixed",
"min" : 1,
"max" : 1,
"queue_size" : -1
},
"percolate" : {
"type" : "fixed",
"min" : 2,
"max" : 2,
"queue_size" : 1000
},
"fetch_shard_started" : {
"type" : "scaling",
"min" : 1,
"max" : 4,
"keep_alive" : "5m",
"queue_size" : -1
},
"listener" : {
"type" : "fixed",
"min" : 1,
"max" : 1,
"queue_size" : -1
},
"index" : {
"type" : "fixed",
"min" : 2,
"max" : 2,
"queue_size" : 200
},
"refresh" : {
"type" : "scaling",
"min" : 1,
"max" : 1,
"keep_alive" : "5m",
"queue_size" : -1
},
"suggest" : {
"type" : "fixed",
"min" : 2,
"max" : 2,
"queue_size" : 1000
},
"generic" : {
"type" : "cached",
"keep_alive" : "30s",
"queue_size" : -1
},
"warmer" : {
"type" : "scaling",
"min" : 1,
"max" : 1,
"keep_alive" : "5m",
"queue_size" : -1
},
"search" : {
"type" : "fixed",
"min" : 4,
"max" : 4,
"queue_size" : 1000
},
"flush" : {
"type" : "scaling",
"min" : 1,
"max" : 1,
"keep_alive" : "5m",
"queue_size" : -1
},
"fetch_shard_store" : {
"type" : "scaling",
"min" : 1,
"max" : 4,
"keep_alive" : "5m",
"queue_size" : -1
},
"management" : {
"type" : "scaling",
"min" : 1,
"max" : 5,
"keep_alive" : "5m",
"queue_size" : -1
},
"get" : {
"type" : "fixed",
"min" : 2,
"max" : 2,
"queue_size" : 1000
},
"bulk" : {
"type" : "fixed",
"min" : 2,
"max" : 2,
"queue_size" : 50
},
"snapshot" : {
"type" : "scaling",
"min" : 1,
"max" : 1,
"keep_alive" : "5m",
"queue_size" : -1
}
},
"transport" : {
"bound_address" : [ "[fe80::2c58:63ff:fedc:dea6]:9300", "10.244.0.11:9300" ],
"publish_address" : "10.244.0.11:9300",
"profiles" : { }
},
"http" : {
"bound_address" : [ "[fe80::2c58:63ff:fedc:dea6]:9200", "10.244.0.11:9200" ],
"publish_address" : "10.244.0.11:9200",
"max_content_length_in_bytes" : 104857600
},
"plugins" : [ {
"name" : "cloud-azure",
"version" : "2.4.0",
"description" : "The Azure Cloud plugin allows to use Azure API for the unicast discovery mechanism and add Azure storage repositories.",
"jvm" : true,
"classname" : "org.elasticsearch.plugin.cloud.azure.CloudAzurePlugin",
"isolated" : true,
"site" : false
}, {
"name" : "cloud-kubernetes",
"version" : "2.4.0_01",
"description" : "Elasticsearch Kubernetes cloud plugin",
"jvm" : true,
"classname" : "io.fabric8.elasticsearch.plugin.discovery.kubernetes.KubernetesDiscoveryPlugin",
"isolated" : true,
"site" : false
}, {
"name" : "prometheus-exporter",
"version" : "2.4.0.0",
"description" : "Prometheus Exporter",
"jvm" : true,
"classname" : "org.elasticsearch.plugin.prometheus.PrometheusExporterPlugin",
"isolated" : false,
"site" : false
} ],
"modules" : [ {
"name" : "lang-expression",
"version" : "2.4.0",
"description" : "Lucene expressions integration for Elasticsearch",
"jvm" : true,
"classname" : "org.elasticsearch.script.expression.ExpressionPlugin",
"isolated" : true,
"site" : false
}, {
"name" : "lang-groovy",
"version" : "2.4.0",
"description" : "Groovy scripting integration for Elasticsearch",
"jvm" : true,
"classname" : "org.elasticsearch.script.groovy.GroovyPlugin",
"isolated" : true,
"site" : false
}, {
"name" : "reindex",
"version" : "2.4.0",
"description" : "_reindex and _update_by_query APIs",
"jvm" : true,
"classname" : "org.elasticsearch.index.reindex.ReindexPlugin",
"isolated" : true,
"site" : false
} ]
},
"s1go_31dROSVlck03YOAow" : {
"name" : "The Profile",
"transport_address" : "10.244.3.11:9300",
"host" : "10.244.3.11",
"ip" : "10.244.3.11",
"version" : "2.4.0",
"build" : "ce9f0c7",
"http_address" : "10.244.3.11:9200",
"attributes" : {
"data" : "false",
"master" : "true"
},
"settings" : {
"cloud" : {
"kubernetes" : {
"namespace" : "default",
"service" : "elasticsearch-discovery"
}
},
"cluster" : {
"name" : "myesdb"
},
"node" : {
"data" : "false",
"master" : "true"
},
"path" : {
"data" : "/data/data",
"logs" : "/data/log",
"plugins" : "/elasticsearch/plugins",
"work" : "/data/work",
"home" : "/elasticsearch"
},
"discovery" : {
"type" : "kubernetes",
"zen" : {
"minimum_master_nodes" : "1"
}
},
"name" : "The Profile",
"client" : {
"type" : "node"
},
"http" : {
"compression" : "true",
"enabled" : "true",
"cors" : {
"allow-origin" : "*",
"enabled" : "true"
}
},
"index" : {
"number_of_shards" : "1",
"number_of_replicas" : "0"
},
"bootstrap" : {
"mlockall" : "true"
},
"config" : {
"ignore_system_properties" : "true"
},
"network" : {
"host" : "_eth0_"
}
},
"os" : {
"refresh_interval_in_millis" : 1000,
"name" : "Linux",
"arch" : "amd64",
"version" : "4.15.0-1023-azure",
"available_processors" : 2,
"allocated_processors" : 2
},
"process" : {
"refresh_interval_in_millis" : 1000,
"id" : 102,
"mlockall" : true
},
"jvm" : {
"pid" : 102,
"version" : "1.8.0_92-internal",
"vm_name" : "OpenJDK 64-Bit Server VM",
"vm_version" : "25.92-b14",
"vm_vendor" : "Oracle Corporation",
"start_time_in_millis" : 1542177839355,
"mem" : {
"heap_init_in_bytes" : 1073741824,
"heap_max_in_bytes" : 1056309248,
"non_heap_init_in_bytes" : 2555904,
"non_heap_max_in_bytes" : 0,
"direct_max_in_bytes" : 1056309248
},
"gc_collectors" : [ "ParNew", "ConcurrentMarkSweep" ],
"memory_pools" : [ "Code Cache", "Metaspace", "Compressed Class Space", "Par Eden Space", "Par Survivor Space", "CMS Old Gen" ],
"using_compressed_ordinary_object_pointers" : "true"
},
"thread_pool" : {
"force_merge" : {
"type" : "fixed",
"min" : 1,
"max" : 1,
"queue_size" : -1
},
"percolate" : {
"type" : "fixed",
"min" : 2,
"max" : 2,
"queue_size" : 1000
},
"fetch_shard_started" : {
"type" : "scaling",
"min" : 1,
"max" : 4,
"keep_alive" : "5m",
"queue_size" : -1
},
"listener" : {
"type" : "fixed",
"min" : 1,
"max" : 1,
"queue_size" : -1
},
"index" : {
"type" : "fixed",
"min" : 2,
"max" : 2,
"queue_size" : 200
},
"refresh" : {
"type" : "scaling",
"min" : 1,
"max" : 1,
"keep_alive" : "5m",
"queue_size" : -1
},
"suggest" : {
"type" : "fixed",
"min" : 2,
"max" : 2,
"queue_size" : 1000
},
"generic" : {
"type" : "cached",
"keep_alive" : "30s",
"queue_size" : -1
},
"warmer" : {
"type" : "scaling",
"min" : 1,
"max" : 1,
"keep_alive" : "5m",
"queue_size" : -1
},
"search" : {
"type" : "fixed",
"min" : 4,
"max" : 4,
"queue_size" : 1000
},
"flush" : {
"type" : "scaling",
"min" : 1,
"max" : 1,
"keep_alive" : "5m",
"queue_size" : -1
},
"fetch_shard_store" : {
"type" : "scaling",
"min" : 1,
"max" : 4,
"keep_alive" : "5m",
"queue_size" : -1
},
"management" : {
"type" : "scaling",
"min" : 1,
"max" : 5,
"keep_alive" : "5m",
"queue_size" : -1
},
"get" : {
"type" : "fixed",
"min" : 2,
"max" : 2,
"queue_size" : 1000
},
"bulk" : {
"type" : "fixed",
"min" : 2,
"max" : 2,
"queue_size" : 50
},
"snapshot" : {
"type" : "scaling",
"min" : 1,
"max" : 1,
"keep_alive" : "5m",
"queue_size" : -1
}
},
"transport" : {
"bound_address" : [ "[fe80::642c:43ff:fe70:ebd2]:9300", "10.244.3.11:9300" ],
"publish_address" : "10.244.3.11:9300",
"profiles" : { }
},
"http" : {
"bound_address" : [ "[fe80::642c:43ff:fe70:ebd2]:9200", "10.244.3.11:9200" ],
"publish_address" : "10.244.3.11:9200",
"max_content_length_in_bytes" : 104857600
},
"plugins" : [ {
"name" : "cloud-azure",
"version" : "2.4.0",
"description" : "The Azure Cloud plugin allows to use Azure API for the unicast discovery mechanism and add Azure storage repositories.",
"jvm" : true,
"classname" : "org.elasticsearch.plugin.cloud.azure.CloudAzurePlugin",
"isolated" : true,
"site" : false
}, {
"name" : "cloud-kubernetes",
"version" : "2.4.0_01",
"description" : "Elasticsearch Kubernetes cloud plugin",
"jvm" : true,
"classname" : "io.fabric8.elasticsearch.plugin.discovery.kubernetes.KubernetesDiscoveryPlugin",
"isolated" : true,
"site" : false
}, {
"name" : "prometheus-exporter",
"version" : "2.4.0.0",
"description" : "Prometheus Exporter",
"jvm" : true,
"classname" : "org.elasticsearch.plugin.prometheus.PrometheusExporterPlugin",
"isolated" : false,
"site" : false
} ],
"modules" : [ {
"name" : "lang-expression",
"version" : "2.4.0",
"description" : "Lucene expressions integration for Elasticsearch",
"jvm" : true,
"classname" : "org.elasticsearch.script.expression.ExpressionPlugin",
"isolated" : true,
"site" : false
}, {
"name" : "lang-groovy",
"version" : "2.4.0",
"description" : "Groovy scripting integration for Elasticsearch",
"jvm" : true,
"classname" : "org.elasticsearch.script.groovy.GroovyPlugin",
"isolated" : true,
"site" : false
}, {
"name" : "reindex",
"version" : "2.4.0",
"description" : "_reindex and _update_by_query APIs",
"jvm" : true,
"classname" : "org.elasticsearch.index.reindex.ReindexPlugin",
"isolated" : true,
"site" : false
} ]
}
}
}
Kindly let us know, if you need any additional info. Thanks !!
Please note that 2.4.0 is past post-end-of-life support.
@jasontedor Thanks for the update.
Elasticsearch version (
bin/elasticsearch --version
): 2.4.0Plugins installed: [{"name":"The Profile","component":"cloud-azure","version":"2.4.0","type":"j","url":null},{"name":"The Profile","component":"cloud-kubernetes","version":"2.4.0_01","type":"j","url":null},{"name":"The Profile","component":"prometheus-exporter","version":"2.4.0.0","type":"j","url":null},{"name":"Living Totem","component":"cloud-azure","version":"2.4.0","type":"j","url":null},{"name":"Living Totem","component":"cloud-kubernetes","version":"2.4.0_01","type":"j","url":null},{"name":"Living Totem","component":"prometheus-exporter","version":"2.4.0.0","type":"j","url":null},{"name":"Hydro","component":"cloud-azure","version":"2.4.0","type":"j","url":null},{"name":"Hydro","component":"cloud-kubernetes","version":"2.4.0_01","type":"j","url":null},{"name":"Hydro","component":"prometheus-exporter","version":"2.4.0.0","type":"j","url":null}]
JVM version (
java -version
): openjdk version "1.8.0_181"OS version (
uname -a
if on a Unix-like system): Azure Linux VM - 16.04.1-UbuntuDescription of the problem including expected versus actual behavior: We are not able to create ElasticSearch repository even after installing cloud-azure plugin.
Steps to reproduce:
Note : We have tried adding storage account and key details as part of elasticsearch.yml as mentioned below, Scenario 1 cloud.azure.storage.account & cloud.azure.storage.key Scenario 2 cloud.azure.storage.my_account.account & cloud.azure.storage.my_account.key Scenario 3 cloud.azure.storage.default.account & cloud.azure.storage.default.key
Storage account and key details is updated accordingly & verified.
Error Log: Please find the ES error logs mentioned below,
RemoteTransportException[[The Profile][10.244.3.11:9300][cluster:admin/repository/put]]; nested: RepositoryException[[es-indices] failed to create repository]; nested: IllegalArgumentException[Unknown [repository] type [azure]]; Caused by: RepositoryException[[es-indices] failed to create repository]; nested: IllegalArgumentException[Unknown [repository] type [azure]]; at org.elasticsearch.repositories.RepositoriesService.createRepositoryHolder(RepositoriesService.java:411) at org.elasticsearch.repositories.RepositoriesService.registerRepository(RepositoriesService.java:368) at org.elasticsearch.repositories.RepositoriesService.access$100(RepositoriesService.java:55) at org.elasticsearch.repositories.RepositoriesService$1.execute(RepositoriesService.java:110) at org.elasticsearch.cluster.ClusterStateUpdateTask.execute(ClusterStateUpdateTask.java:45) at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:468) at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:772) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231) at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745)
Request you to consider as a priority ticket and please let us know if we are missing out anything here @dadoonet
Thanks.