Closed tarunjangra closed 4 years ago
cebe
commented
6 years ago It is simply ignoring nodes where http:publish_address key is not available in the response.
if there is no http:publish_address we do not know how to connect to the node, can you show how the response looks like?
xjoyzaza
commented
6 years ago IN vendor yiisoft yii2-elasticsearch change the ` foreach ($nodes as $key => &$node) {
// Make sure that nodes have an 'http_address' property, which is not the case if you're using AWS
// Elasticsearch service (at least as of Oct., 2015). - TO BE VERIFIED
// Temporary workaround - simply ignore all invalid nodes
if (!isset($node['http']['publish_address'])) {
unset($nodes[$key]);
}
$node['http_address'] = $node['http']['publish_address'];
//Protocol is not a standard ES node property, so we add it manually
$node['protocol'] = $this->defaultProtocol;
}` to ` foreach ($nodes as $key => &$node) {
$node['http_address'] = $host;
//Protocol is not a standard ES node property, so we add it manually
$node['protocol'] = $this->defaultProtocol;
}`@xjoyzaza can you show me what is in $nodes at that point? That would help, me to fix this in the extension itself so you don't need that workaround.
xjoyzaza
commented
6 years ago @xjoyzaza can you show me what is in $nodes at that point? That would help, me to fix this in the extension itself so you don't need that workaround.
there is no $node["http"] res in aws response ...
cebe
commented
5 years ago that is clear, but what do we have? Is it empty? could you show an example reponse?
tarunjangra
commented
5 years ago @cebe Following is the amazon's response for api call: curl -XGET https://end-point.es.amazonaws.com/_nodes?pretty=true
You can see, "$node['http']['publish_address']" is missing in following response. Hope this will help.
{
"_nodes" : {
"total" : 1,
"successful" : 1,
"failed" : 0
},
"cluster_name" : "228928909316:ginotprod",
"nodes" : {
"7JlgsfQ1TBWQgVz-x_jh9Q" : {
"name" : "7JlgsfQ",
"version" : "6.2.3",
"build_hash" : "bbbda2a",
"total_indexing_buffer" : 106502553,
"roles" : [ "master", "data", "ingest" ],
"os" : {
"refresh_interval_in_millis" : 1000,
"available_processors" : 1,
"allocated_processors" : 1
},
"process" : {
"refresh_interval_in_millis" : 1000,
"id" : 5868,
"mlockall" : false
},
"jvm" : {
"pid" : 5868,
"start_time_in_millis" : 1551341005381,
"mem" : {
"heap_init_in_bytes" : 1073741824,
"heap_max_in_bytes" : 1065025536,
"non_heap_init_in_bytes" : 2555904,
"non_heap_max_in_bytes" : 0,
"direct_max_in_bytes" : 1065025536
},
"using_compressed_ordinary_object_pointers" : "true"
},
"thread_pool" : {
"force_merge" : {
"type" : "fixed",
"min" : 1,
"max" : 1,
"queue_size" : -1
},
"fetch_shard_started" : {
"type" : "scaling",
"min" : 1,
"max" : 2,
"keep_alive" : "5m",
"queue_size" : -1
},
"listener" : {
"type" : "fixed",
"min" : 1,
"max" : 1,
"queue_size" : -1
},
"index" : {
"type" : "fixed",
"min" : 1,
"max" : 1,
"queue_size" : 200
},
"refresh" : {
"type" : "scaling",
"min" : 1,
"max" : 1,
"keep_alive" : "5m",
"queue_size" : -1
},
"generic" : {
"type" : "scaling",
"min" : 4,
"max" : 128,
"keep_alive" : "30s",
"queue_size" : -1
},
"warmer" : {
"type" : "scaling",
"min" : 1,
"max" : 1,
"keep_alive" : "5m",
"queue_size" : -1
},
"search" : {
"type" : "fixed_auto_queue_size",
"min" : 2,
"max" : 2,
"queue_size" : 1000
},
"flush" : {
"type" : "scaling",
"min" : 1,
"max" : 1,
"keep_alive" : "5m",
"queue_size" : -1
},
"fetch_shard_store" : {
"type" : "scaling",
"min" : 1,
"max" : 2,
"keep_alive" : "5m",
"queue_size" : -1
},
"management" : {
"type" : "scaling",
"min" : 1,
"max" : 5,
"keep_alive" : "5m",
"queue_size" : -1
},
"get" : {
"type" : "fixed",
"min" : 1,
"max" : 1,
"queue_size" : 1000
},
"bulk" : {
"type" : "fixed",
"min" : 1,
"max" : 1,
"queue_size" : 200
},
"snapshot" : {
"type" : "scaling",
"min" : 1,
"max" : 1,
"keep_alive" : "5m",
"queue_size" : -1
}
},
"modules" : [ {
"name" : "aggs-matrix-stats",
"version" : "6.2.3",
"description" : "Adds aggregations whose input are a list of numeric fields and output includes a matrix.",
"classname" : "org.elasticsearch.search.aggregations.matrix.MatrixAggregationPlugin",
"extended_plugins" : [ ],
"has_native_controller" : false,
"requires_keystore" : false
}, {
"name" : "analysis-common",
"version" : "6.2.3",
"description" : "Adds \"built in\" analyzers to Elasticsearch.",
"classname" : "org.elasticsearch.analysis.common.CommonAnalysisPlugin",
"extended_plugins" : [ ],
"has_native_controller" : false,
"requires_keystore" : false
}, {
"name" : "ingest-common",
"version" : "6.2.3",
"description" : "Module for ingest processors that do not require additional security permissions or have large dependencies and resources",
"classname" : "org.elasticsearch.ingest.common.IngestCommonPlugin",
"extended_plugins" : [ ],
"has_native_controller" : false,
"requires_keystore" : false
}, {
"name" : "lang-expression",
"version" : "6.2.3",
"description" : "Lucene expressions integration for Elasticsearch",
"classname" : "org.elasticsearch.script.expression.ExpressionPlugin",
"extended_plugins" : [ ],
"has_native_controller" : false,
"requires_keystore" : false
}, {
"name" : "lang-mustache",
"version" : "6.2.3",
"description" : "Mustache scripting integration for Elasticsearch",
"classname" : "org.elasticsearch.script.mustache.MustachePlugin",
"extended_plugins" : [ ],
"has_native_controller" : false,
"requires_keystore" : false
}, {
"name" : "lang-painless",
"version" : "6.2.3",
"description" : "An easy, safe and fast scripting language for Elasticsearch",
"classname" : "org.elasticsearch.painless.PainlessPlugin",
"extended_plugins" : [ ],
"has_native_controller" : false,
"requires_keystore" : false
}, {
"name" : "mapper-extras",
"version" : "6.2.3",
"description" : "Adds advanced field mappers",
"classname" : "org.elasticsearch.index.mapper.MapperExtrasPlugin",
"extended_plugins" : [ ],
"has_native_controller" : false,
"requires_keystore" : false
}, {
"name" : "parent-join",
"version" : "6.2.3",
"description" : "This module adds the support parent-child queries and aggregations",
"classname" : "org.elasticsearch.join.ParentJoinPlugin",
"extended_plugins" : [ ],
"has_native_controller" : false,
"requires_keystore" : false
}, {
"name" : "percolator",
"version" : "6.2.3",
"description" : "Percolator module adds capability to index queries and query these queries by specifying documents",
"classname" : "org.elasticsearch.percolator.PercolatorPlugin",
"extended_plugins" : [ ],
"has_native_controller" : false,
"requires_keystore" : false
}, {
"name" : "rank-eval",
"version" : "6.2.3",
"description" : "The Rank Eval module adds APIs to evaluate ranking quality.",
"classname" : "org.elasticsearch.index.rankeval.RankEvalPlugin",
"extended_plugins" : [ ],
"has_native_controller" : false,
"requires_keystore" : false
}, {
"name" : "reindex",
"version" : "6.2.3",
"description" : "The Reindex module adds APIs to reindex from one index to another or update documents in place.",
"classname" : "org.elasticsearch.index.reindex.ReindexPlugin",
"extended_plugins" : [ ],
"has_native_controller" : false,
"requires_keystore" : false
}, {
"name" : "repository-url",
"version" : "6.2.3",
"description" : "Module for URL repository",
"classname" : "org.elasticsearch.plugin.repository.url.URLRepositoryPlugin",
"extended_plugins" : [ ],
"has_native_controller" : false,
"requires_keystore" : false
}, {
"name" : "transport-netty4",
"version" : "6.2.3",
"description" : "Netty 4 based transport implementation",
"classname" : "org.elasticsearch.transport.Netty4Plugin",
"extended_plugins" : [ ],
"has_native_controller" : false,
"requires_keystore" : false
} ],
"ingest" : {
"processors" : [ {
"type" : "append"
}, {
"type" : "attachment"
}, {
"type" : "convert"
}, {
"type" : "date"
}, {
"type" : "date_index_name"
}, {
"type" : "dot_expander"
}, {
"type" : "fail"
}, {
"type" : "foreach"
}, {
"type" : "grok"
}, {
"type" : "gsub"
}, {
"type" : "join"
}, {
"type" : "json"
}, {
"type" : "kv"
}, {
"type" : "lowercase"
}, {
"type" : "remove"
}, {
"type" : "rename"
}, {
"type" : "script"
}, {
"type" : "set"
}, {
"type" : "sort"
}, {
"type" : "split"
}, {
"type" : "trim"
}, {
"type" : "uppercase"
}, {
"type" : "urldecode"
}, {
"type" : "user_agent"
} ]
}
}
}
}```
tkthundr
commented
5 years ago Setting autodetectCluster to false will basically use the node specified in the config. This works for me
'elasticSearch' => [
'class' => 'yii\elasticsearch\Connection',
'defaultProtocol'=>'https',
'autodetectCluster'=>false,
'nodes' => [
['http_address' => '<your-node>.us-east-1.es.amazonaws.com'],
],
],
beowulfenator
commented
4 years ago Node autodetection issue adressed in 2.0.4:
- Enh #28: AWS Elasticsearch service compatibility (andrey-bahrachev)
If autodetection does not work, the recommended solution is to disable autodetection and list nodes manually.
I'm installed with 2.1 yii2-elasticsearch version because es i'm trying to work with is 6.2. And it is not allowing me to connect with AWS elasticsearch service. Where as my custom elasticsearch installation works fine.
What steps will reproduce the problem?
here is my component configuration.
'elasticsearch' => [ 'class' => 'yii\elasticsearch\Connection', 'nodes' => [ [ 'http_address' => 'end-point.es.amazonaws.com', 'protocol' => 'https' ], ], ],Producing following error `Executing...PHP Notice 'yii\base\ErrorException' with message 'Undefined index: http'in /var/www/html/staging/vendor/yiisoft/yii2-elasticsearch/Connection.php:190
Stack trace:
0 /var/www/html/staging/vendor/yiisoft/yii2-elasticsearch/Connection.php(190): yii\base\ErrorHandler->handleError(8, 'Undefined index...', '/var/www/html/s...', 190, Array)
1 /var/www/html/staging/vendor/yiisoft/yii2-elasticsearch/Connection.php(155): yii\elasticsearch\Connection->populateNodes()
2 /var/www/html/staging/vendor/yiisoft/yii2-elasticsearch/Connection.php(259): yii\elasticsearch\Connection->open()
3 /var/www/html/staging/common/models/es/BaseModel.php(129): yii\elasticsearch\Connection->createCommand()
4 /var/www/html/staging/common/models/es/BaseModel.php(134): common\models\es\BaseModel::deleteIndex()
5 /var/www/html/staging/console/controllers/EsController.php(114): common\models\es\BaseModel::resetIndex()
6 /var/www/html/staging/console/controllers/EsController.php(206): console\controllers\EsController->reindexInBulk(Array, '100')
7 [internal function]: console\controllers\EsController->actionReindex()
8 /var/www/html/staging/vendor/yiisoft/yii2/base/InlineAction.php(57): call_user_func_array(Array, Array)
9 /var/www/html/staging/vendor/yiisoft/yii2/base/Controller.php(157): yii\base\InlineAction->runWithParams(Array)
10 /var/www/html/staging/vendor/yiisoft/yii2/console/Controller.php(148): yii\base\Controller->runAction('reindex', Array)
11 /var/www/html/staging/vendor/yiisoft/yii2/base/Module.php(528): yii\console\Controller->runAction('reindex', Array)
12 /var/www/html/staging/vendor/yiisoft/yii2/console/Application.php(180): yii\base\Module->runAction('es/reindex', Array)
13 /var/www/html/staging/vendor/yiisoft/yii2/console/Application.php(147): yii\console\Application->runAction('es/reindex', Array)
14 /var/www/html/staging/vendor/yiisoft/yii2/base/Application.php(386): yii\console\Application->handleRequest(Object(yii\console\Request))
15 /var/www/html/staging/yii(31): yii\base\Application->run()
16 {main}
`
What's expected?
It should be connect. but as per connection.php. It is simply ignoring nodes where http:publish_address key is not available in the response.
Additional info