[FR] [DAC] Update default KQL parsing behavior to normalize keywords for custom rule directories.

[eric-forte-elastic]

Related Issue:

https://github.com/elastic/detection-rules/issues/3624

Related PRs:

https://github.com/elastic/detection-rules/pull/3574

Summary

This PR updates the default KQL parsing behavior when a custom rule directory is set. Now when the NORMALIZE_KQL_KEYWORDS environment variable is set, the normalization flag is set to True

Testing

Make a custom KQL rule and modify the keyword to be upper case and test with view rule.

Example KQL Rule

``` [metadata] creation_date = "2023/05/22" maturity = "production" updated_date = "2024/05/21" [transform] [[transform.osquery]] label = "Osquery - Retrieve DNS Cache" query = "SELECT * FROM dns_cache" [[transform.osquery]] label = "Osquery - Retrieve All Services" query = "SELECT description, display_name, name, path, pid, service_type, start_type, status, user_account FROM services" [[transform.osquery]] label = "Osquery - Retrieve Services Running on User Accounts" query = """ SELECT description, display_name, name, path, pid, service_type, start_type, status, user_account FROM services WHERE NOT (user_account LIKE '%LocalSystem' OR user_account LIKE '%LocalService' OR user_account LIKE '%NetworkService' OR user_account == null) """ [[transform.osquery]] label = "Osquery - Retrieve Service Unsigned Executables with Virustotal Link" query = """ SELECT concat('https://www.virustotal.com/gui/file/', sha1) AS VtLink, name, description, start_type, status, pid, services.path FROM services JOIN authenticode ON services.path = authenticode.path OR services.module_path = authenticode.path JOIN hash ON services.path = hash.path WHERE authenticode.result != 'trusted' """ [rule] author = ["Elastic"] description = """ This rule is triggered when a hash indicator from the Threat Intel Filebeat module or integrations has a match against an event that contains file hashes, such as antivirus alerts, process creation, library load, and file operation events. """ from = "now-65m" index = ["auditbeat-*", "endgame-*", "filebeat-*", "logs-*", "winlogbeat-*"] interval = "1h" language = "kuery" license = "Elastic License v2" name = "Test Threat Intel Hash Indicator Match" note = """## Triage and Analysis fing the incident response data, update logging and audit policies to improve the mean time to detect (MTTD) and the mean time to respond (MTTR). """ references = [ "https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-module-threatintel.html", "https://www.elastic.co/guide/en/security/master/es-threat-intel-integrations.html", "https://www.elastic.co/security/tip", ] risk_score = 99 rule_id = "f62a0edd-4cf3-46fe-b5f2-58fd23db991e" setup = """## Setup This rule needs threat intelligence indicators to work. Threat intelligence indicators can be collected using an [Elastic Agent integration](https://www.elastic.co/guide/en/security/current/es-threat-intel-integrations.html#agent-ti-integration), the [Threat Intel module](https://www.elastic.co/guide/en/security/current/es-threat-intel-integrations.html#ti-mod-integration), or a [custom integration](https://www.elastic.co/guide/en/security/current/es-threat-intel-integrations.html#custom-ti-integration). More information can be found [here](https://www.elastic.co/guide/en/security/current/es-threat-intel-integrations.html). """ severity = "critical" tags = ["OS: Windows", "Data Source: Elastic Endgame", "Rule Type: Indicator Match"] threat_index = ["filebeat-*", "logs-ti_*"] threat_indicator_path = "threat.indicator" threat_language = "kuery" threat_query = """ @timestamp >= "now-30d/d" and event.module:(threatintel or ti_*) and (threat.indicator.file.hash.*:* or threat.indicator.file.pe.imphash:*) and not labels.is_ioc_transform_source:"true" """ timeline_id = "495ad7a7-316e-4544-8a0f-9c098daee76e" timeline_title = "Generic Threat Match Timeline" timestamp_override = "event.ingested" type = "threat_match" query = ''' file.hash.*:* OR process.hash.*:* or dll.hash.*:* ''' [[rule.threat_filters]] [rule.threat_filters."$state"] store = "appState" [rule.threat_filters.meta] disabled = false key = "event.category" negate = false type = "phrase" [rule.threat_filters.meta.params] query = "threat" [rule.threat_filters.query.match_phrase] "event.category" = "threat" [[rule.threat_filters]] [rule.threat_filters."$state"] store = "appState" [rule.threat_filters.meta] disabled = false key = "event.kind" negate = false type = "phrase" [rule.threat_filters.meta.params] query = "enrichment" [rule.threat_filters.query.match_phrase] "event.kind" = "enrichment" [[rule.threat_filters]] [rule.threat_filters."$state"] store = "appState" [rule.threat_filters.meta] disabled = false key = "event.type" negate = false type = "phrase" [rule.threat_filters.meta.params] query = "indicator" [rule.threat_filters.query.match_phrase] "event.type" = "indicator" [[rule.threat_mapping]] [[rule.threat_mapping.entries]] field = "file.hash.md5" type = "mapping" value = "threat.indicator.file.hash.md5" [[rule.threat_mapping]] [[rule.threat_mapping.entries]] field = "file.hash.sha1" type = "mapping" value = "threat.indicator.file.hash.sha1" [[rule.threat_mapping]] [[rule.threat_mapping.entries]] field = "file.hash.sha256" type = "mapping" value = "threat.indicator.file.hash.sha256" [[rule.threat_mapping]] [[rule.threat_mapping.entries]] field = "dll.hash.md5" type = "mapping" value = "threat.indicator.file.hash.md5" [[rule.threat_mapping]] [[rule.threat_mapping.entries]] field = "dll.hash.sha1" type = "mapping" value = "threat.indicator.file.hash.sha1" [[rule.threat_mapping]] [[rule.threat_mapping.entries]] field = "dll.hash.sha256" type = "mapping" value = "threat.indicator.file.hash.sha256" [[rule.threat_mapping]] [[rule.threat_mapping.entries]] field = "process.hash.md5" type = "mapping" value = "threat.indicator.file.hash.md5" [[rule.threat_mapping]] [[rule.threat_mapping.entries]] field = "process.hash.sha1" type = "mapping" value = "threat.indicator.file.hash.sha1" [[rule.threat_mapping]] [[rule.threat_mapping.entries]] field = "process.hash.sha256" type = "mapping" value = "threat.indicator.file.hash.sha256" ```

Output


[brokensound77]

These should not just default to that behavior simply based on the presence of the custom rules envvar, as that may not be the desired behavior. Please update to expose a new config parameter where these can be set and pushed down as needed

[eric-forte-elastic]

These should not just default to that behavior simply based on the presence of the custom rules envvar, as that may not be the desired behavior. Please update to expose a new config parameter where these can be set and pushed down as needed

Sure, happy to update it to be a parameter instead. I was basing the initial approach off of your description in the issue.

I will look at allowing normalization on KQL queries for custom rules by default.

And I understood that to mean that you wanted a default based on the presence of custom rules.

[eric-forte-elastic]

Updated to now have a separate environment variable NORMALIZE_KQL_KEYWORDS to set the desire to normalize the keywords. Given that this is no longer DAC specific we may want to think about merging this directly to main.

Another considered approach was to put the value in the config file; however, this would require fairly significant restructuring of the detection rules logic flow (specifically how we use utils.py) to support this. Furthermore, this would override a well debated design decision we made in the initial POC. As such, I do not think it is a desirable method.

[eric-forte-elastic]

Upon further consideration, while the original POC decision for the config setup has merit, the enhanced usability for the end user of being able to specify the variable in the config, warrants modifying this structure to support it. Updating shortly.

[brokensound77]

the enhanced usability for the end user of being able to specify the variable in the config, warrants modifying this structure to support it

I concur and think it requires it. We should strive to make the use of environmental variables optional (with configs), where possible to minimize complexity of calls

[eric-forte-elastic]

the enhanced usability for the end user of being able to specify the variable in the config, warrants modifying this structure to support it

I concur and think it requires it. We should strive to make the use of environmental variables optional (with configs), where possible to minimize complexity of calls

Updated to use the config. As a note to my prior comment, the restructuring was already done in another PR so this PR should be small. :rocket:

[eric-forte-elastic]

As a note, this will require an accompanying docs PR since we are updating the config.