nsano-rururu
commented
3 years ago ES8 support may need to be considered in addition to doc_type.
In ES7.X this include_type_name default has been changed to false, but elastalert explicitly specifies include_type_name = true to use the type, but it will be removed in ES8. Will be.
Also, the specification of dateOptionalTime is abolished and becomesdate_optional_time.
elasticsearch>=7.0.0,<8.0.0
ProcessController: /home/node/.local/lib/python3.8/site-packages/elasticsearch/connection/base.py:193: ElasticsearchDeprecationWarning: Camel case format name dateOptionalTime is deprecated and will be removed in a future version. Use snake case name date_optional_time instead.
warnings.warn(message, category=ElasticsearchDeprecationWarning)
/home/node/.local/lib/python3.8/site-packages/elasticsearch/connection/base.py:193: ElasticsearchDeprecationWarning: [types removal] Using include_type_name in put mapping requests is deprecated. The parameter will be removed in the next major version.
warnings.warn(message, category=ElasticsearchDeprecationWarning)The following are expected changes that need to be addressed.
___init___.py
add
def is_atleasteight(self):
"""
Returns True when the Elasticsearch server version >= 8
"""
return int(self.es_version.split(".")[0]) >= 8elastalert/create_index.py
before
if is_atleastseven(esversion):
# TODO remove doc_type completely when elasicsearch client allows doc_type=None
# doc_type is a deprecated feature and will be completely removed in Elasicsearch 8
es_client.indices.put_mapping(index=ea_index, doc_type='_doc',
body=es_index_mappings['elastalert'], include_type_name=True)
es_client.indices.put_mapping(index=ea_index + '_status', doc_type='_doc',
body=es_index_mappings['elastalert_status'], include_type_name=True)
es_client.indices.put_mapping(index=ea_index + '_silence', doc_type='_doc',
body=es_index_mappings['silence'], include_type_name=True)
es_client.indices.put_mapping(index=ea_index + '_error', doc_type='_doc',
body=es_index_mappings['elastalert_error'], include_type_name=True)
es_client.indices.put_mapping(index=ea_index + '_past', doc_type='_doc',
body=es_index_mappings['past_elastalert'], include_type_name=True)after
if is_atleasteight(esversion):
es_client.indices.put_mapping(index=ea_index,
body=es_index_mappings['elastalert'])
es_client.indices.put_mapping(index=ea_index + '_status'
body=es_index_mappings['elastalert_status'])
es_client.indices.put_mapping(index=ea_index + '_silence',
body=es_index_mappings['silence'])
es_client.indices.put_mapping(index=ea_index + '_error',
body=es_index_mappings['elastalert_error'])
es_client.indices.put_mapping(index=ea_index + '_past',
body=es_index_mappings['past_elastalert'])
elif is_atleastseven(esversion):
es_client.indices.put_mapping(index=ea_index, doc_type='_doc',
body=es_index_mappings['elastalert'])
es_client.indices.put_mapping(index=ea_index + '_status', doc_type='_doc',
body=es_index_mappings['elastalert_status'])
es_client.indices.put_mapping(index=ea_index + '_silence', doc_type='_doc',
body=es_index_mappings['silence'])
es_client.indices.put_mapping(index=ea_index + '_error', doc_type='_doc',
body=es_index_mappings['elastalert_error'])
es_client.indices.put_mapping(index=ea_index + '_past', doc_type='_doc',
body=es_index_mappings['past_elastalert'])elastalert/loaders.py
It is necessary to prevent the following checks from being performed on elasticsearch 8 or later versions
if rule.get('use_count_query') or rule.get('use_terms_query'):
if 'doc_type' not in rule:
raise EAException('doc_type must be specified.')elastalert/elastalert.py
before
# Record doc_type for use in get_top_counts
if 'doc_type' not in rule and len(hits):
rule['doc_type'] = hits[0]['_type']after
# Record doc_type for use in get_top_counts
if not self.thread_data.current_es.is_atleasteight():
if 'doc_type' not in rule and len(hits):
rule['doc_type'] = hits[0]['_type']before
if not rule['five']:
res = self.thread_data.current_es.deprecated_search(
index=index,
doc_type=rule['doc_type'],
body=query,
search_type='count',
ignore_unavailable=True
)
else:
res = self.thread_data.current_es.deprecated_search(index=index, doc_type=rule['doc_type'],
body=query, size=0, ignore_unavailable=True)after
if not rule['five']:
if self.thread_data.current_es.is_atleasteight():
res = self.thread_data.current_es.deprecated_search(
index=index,
body=query,
search_type='count',
ignore_unavailable=True
)
else:
res = self.thread_data.current_es.deprecated_search(
index=index,
doc_type=rule['doc_type'],
body=query,
search_type='count',
ignore_unavailable=True
)
else:
res = self.thread_data.current_es.deprecated_search(index=index, doc_type=rule['doc_type'],
body=query, size=0, ignore_unavailable=True)before
if not rule['five']:
res = self.thread_data.current_es.deprecated_search(
index=index,
doc_type=rule.get('doc_type'),
body=query,
search_type='count',
ignore_unavailable=True
)
else:
res = self.thread_data.current_es.deprecated_search(index=index, doc_type=rule.get('doc_type'),
body=query, size=0, ignore_unavailable=True)after
if not rule['five']:
if self.thread_data.current_es.is_atleasteight():
res = self.thread_data.current_es.deprecated_search(
index=index,
body=query,
search_type='count',
ignore_unavailable=True
)
else:
res = self.thread_data.current_es.deprecated_search(
index=index,
doc_type=rule.get('doc_type'),
body=query,
search_type='count',
ignore_unavailable=True
)
else:
res = self.thread_data.current_es.deprecated_search(index=index, doc_type=rule.get('doc_type'),
body=query, size=0, ignore_unavailable=True)elastalert/test_rule.py
before
res = es_client.count(index=index, doc_type=doc_type, body=count_query, ignore_unavailable=True)after
if es_client.is_atleasteight():
res = es_client.count(index=index, body=count_query, ignore_unavailable=True)
else:
res = es_client.count(index=index, doc_type=doc_type, body=count_query, ignore_unavailable=True)elastalert/tests/conftest.py
mock_es_client
add
self.is_atleasteight = mock.Mock(return_value=False)mock_es_sixsix_client
add
self.is_atleasteight = mock.Mock(return_value=False)elastalert/es_mappings/8/elastalert.json
add file
dateOptionalTime→date_optional_time
{
"numeric_detection": true,
"date_detection": false,
"dynamic_templates": [
{
"strings_as_keyword": {
"mapping": {
"ignore_above": 1024,
"type": "keyword"
},
"match_mapping_type": "string"
}
}
],
"properties": {
"rule_name": {
"type": "keyword"
},
"@timestamp": {
"type": "date",
"format": "date_optional_time"
},
"alert_time": {
"type": "date",
"format": "date_optional_time"
},
"match_time": {
"type": "date",
"format": "date_optional_time"
},
"match_body": {
"enabled": "false",
"type": "object"
},
"aggregate_id": {
"type": "keyword"
}
}
}add Directory
elastalert/es_mappings/8
elastalert/es_mappings/8/elastalert_error.json ・・・add file
dateOptionalTime→date_optional_time
{
"properties": {
"data": {
"type": "object",
"enabled": "false"
},
"@timestamp": {
"type": "date",
"format": "date_optional_time"
}
}
}elastalert/es_mappings/8/elastalert_status.json・・・add file
dateOptionalTime→date_optional_time
{
"properties": {
"rule_name": {
"type": "keyword"
},
"@timestamp": {
"type": "date",
"format": "date_optional_time"
}
}
}elastalert/es_mappings/8/past_elastalert.json・・・add file
dateOptionalTime→date_optional_time
{
"properties": {
"rule_name": {
"type": "keyword"
},
"match_body": {
"type": "object",
"enabled": "false"
},
"@timestamp": {
"type": "date",
"format": "date_optional_time"
},
"aggregate_id": {
"type": "keyword"
}
}
}elastalert/es_mappings/8/silence.json・・・add file
dateOptionalTime→date_optional_time
{
"properties": {
"rule_name": {
"type": "keyword"
},
"until": {
"type": "date",
"format": "date_optional_time"
},
"@timestamp": {
"type": "date",
"format": "date_optional_time"
}
}
}elastalert/create_index.py
before
es_index_mappings = read_es_index_mappings() if is_atleastsix(esversion) else read_es_index_mappings(5)after
es_index_mappings = read_es_index_mappings() is_atleasteight(esversion) elif is_atleastsix(esversion) else read_es_index_mappings(5)tests/create_index_test.py
add test_read_es_8_index_mappings
def test_read_es_8_index_mappings():
mappings = elastalert.create_index.read_es_index_mappings(8)
assert len(mappings) == len(es_mappings)
print((json.dumps(mappings, indent=2)))
ferozsalam
jertel
sethmlarson
LaZyDK
konstantin-921
Elasticsearch v8 no longer supports doc_type. There is likely going to be some effort need to update this project to deal with this.