jowg-amazon
commented
2 months ago Hi @lsoumille, this issue occurs if there is a detector that has threat intel enabled on an older domain (2.11 - 2.15) and is now upgraded to 2.16. We are working on a code fix but in the meantime to unblock the creation of source configs you can manually do a reindex for the .opensearch-sap--job index. I have pasted the steps you can run below in the dev console to create a temp index, reindex, then update the mapping for the original index and reindex your data back. With this new mapping you should be able to create new source configs. Let us know if you have additional questions.
// 1. Check to ensure index exists
GET _cat/indices/.opensearch-sap--job?expand_wildcards=all
// 2. Check to see if current mapping contains "dynamic" : "strict" as a field
GET .opensearch-sap--job/_mapping
// 3. Save docs in system index locally if needed
POST .opensearch-sap--job/_search
{
"query": {
"match_all": {}
}
}
// 4. Create a new temp index to reindex data to
PUT /.opensearch-sap--job-new
// 5. Reindex from old to new index
POST /_reindex
{
"source": {
"index": ".opensearch-sap--job"
},
"dest": {
"index": ".opensearch-sap--job-new"
}
}
// 6. Ensure all documents were reindexed correctly to the new index
POST .opensearch-sap--job-new/_search
{
"query": {
"match_all": {}
}
}
// 7. Delete the old index
DELETE /.opensearch-sap--job
// 8. Create a new index with name ".opensearch-sap--job" with the correct mappings (two hyphens are needed between sap and job)
PUT /.opensearch-sap--job
{
"mappings": {
"_meta" : {
"schema_version": 2
},
"properties": {
"source_config": {
"properties": {
"version": {
"type": "long"
},
"name": {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword"
}
}
},
"format": {
"type": "keyword"
},
"type": {
"type": "keyword"
},
"description": {
"type": "text"
},
"created_by_user": {
"properties": {
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"backend_roles": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"roles": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"custom_attribute_names": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
}
}
},
"created_at": {
"type": "date",
"format": "strict_date_time||epoch_millis"
},
"source" : {
"properties": {
"s3": {
"properties": {
"bucket_name": {
"type": "keyword"
},
"object_key": {
"type": "keyword"
},
"region": {
"type": "keyword"
},
"role_arn": {
"type": "keyword"
}
}
},
"ioc_upload": {
"properties": {
"file_name": {
"type": "keyword"
},
"iocs": {
"type" : "text"
}
}
}
}
},
"enabled_time": {
"type": "date",
"format": "strict_date_time||epoch_millis"
},
"last_update_time": {
"type": "date",
"format": "strict_date_time||epoch_millis"
},
"schedule": {
"properties": {
"interval": {
"properties": {
"period": {
"type": "integer"
},
"start_time": {
"type": "date",
"format": "strict_date_time||epoch_millis"
},
"unit": {
"type": "keyword"
}
}
}
}
},
"state": {
"type": "keyword"
},
"refresh_type": {
"type": "keyword"
},
"enabled_for_scan": {
"type": "boolean"
},
"last_refreshed_time": {
"type": "date",
"format": "strict_date_time||epoch_millis"
},
"last_refreshed_user": {
"properties": {
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"backend_roles": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"roles": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"custom_attribute_names": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
}
}
},
"enabled": {
"type": "boolean"
},
"ioc_store_config": {
"properties": {
"default": {
"properties": {
"ioc_to_index_details": {
"properties": {
"ioc_type": {
"type": "keyword"
},
"index_pattern": {
"type": "keyword"
},
"active_index": {
"type": "keyword"
}
}
}
}
}
}
},
"ioc_types": {
"type": "text",
"fields" : {
"keyword" : {
"type" : "keyword"
}
}
}
}
},
"schema_version": {
"type": "integer"
},
"enabled_time": {
"type": "long"
},
"indices": {
"type": "text"
},
"last_update_time": {
"type": "long"
},
"name": {
"type": "text"
},
"schedule": {
"properties": {
"interval": {
"properties": {
"period": {
"type": "long"
},
"start_time": {
"type": "long"
},
"unit": {
"type": "text"
}
}
}
}
},
"state": {
"type": "text"
},
"update_enabled": {
"type": "boolean"
},
"update_stats": {
"properties": {
"last_failed_at_in_epoch_millis": {
"type": "long"
},
"last_processing_time_in_millis": {
"type": "long"
},
"last_skipped_at_in_epoch_millis": {
"type": "long"
},
"last_succeeded_at_in_epoch_millis": {
"type": "long"
}
}
}
}
}
}
// 9. reindex from temp index to original index
POST /_reindex
{
"source": {
"index": ".opensearch-sap--job-new"
},
"dest": {
"index": ".opensearch-sap--job"
}
}
// 10. Double check the mapping of the original index and ensure the version is 2
GET /.opensearch-sap--job/_mapping
// 11. Ensure all documents were reindex to the original index
POST .opensearch-sap--job/_search
{
"query": {
"match_all": {}
}
}
// 12. Delete the temp index
DELETE /.opensearch-sap--job-new
lsoumille
What is the bug? We upgraded recently our OpenSearch setup from 2.15 to 2.16. We tried to add a local custom threat intelligence source through the UI, but we are facing an issue. The source file is the default one provided by default https://opensearch.org/docs/latest/assets/examples/all-ioc-type-examples.json
The error message is:
[strict_dynamic_mapping_exception] mapping set to strict, dynamic introduction of [source_config] within [_doc] is not allowedHow can one reproduce the bug? Steps to reproduce the behavior:
What is the expected behavior? A new log souce should have been created.
What is your host/environment?
Do you have any screenshots?
Do you have any additional context? Add any other context about the problem.