[azure_frontdoor] waf ingest pipeline does not parse correctly to ECS Fields

[bvader]

The logs-azure_frontdoor.waf-1.0.0 does not appear to parse messages from Azure Frontdoor WAF correctly

These are several sample messages from a customer It seems that this pipeline needs to be gone through in detail The following fields appear to be missing / I took a quick pass at fixing

client.ip
source.ip
url.domain
url.original
http.response.status_code
http.request.method

The DNS fields etc are not properly parsed set to ECS Fields There may be other ECS fields as well that I am not familiar with

{
    "category": "FrontDoorHealthProbeLog",
    "operationName": "Microsoft.Cdn/Profiles/FrontDoorHealthProbeLog/Write",
    "properties": {
        "DNSLatencyMicroseconds": "n/a",
        "POP": "CYS",
        "connectionLatencyMilliseconds": "n/a",
        "healthProbeId": "08008F0A7C024D8388C9999E156C9877",
        "httpStatusCode": "404",
        "httpVerb": "HEAD",
        "originIP": "8.8.8.8",
        "originName": "redacted.net",
        "probeURL": "http://redacted.net:80/",
        "result": "OriginError",
        "totalLatencyMilliseconds": "44"
    },
    "resourceId": "/SUBSCRIPTIONS/57F1136F-4C47-4F9F-A10B-C3D50D5D73CD/RESOURCEGROUPS/redacted/PROVIDERS/MICROSOFT.CDN/PROFILES/redacted",
    "time": "2023-07-10T01:39:59.7478609Z"
}

{
    "category": "FrontDoorHealthProbeLog",
    "operationName": "Microsoft.Cdn/Profiles/FrontDoorHealthProbeLog/Write",
    "properties": {
        "DNSLatencyMicroseconds": "n/a",
        "POP": "CYS",
        "connectionLatencyMilliseconds": "n/a",
        "healthProbeId": "08008F0A7C024D8388C9999E156C9877",
        "httpStatusCode": "404",
        "httpVerb": "HEAD",
        "originIP": "8.8.8.8",
        "originName": "redacted.net",
        "probeURL": "http://redacted.net:80/",
        "result": "OriginError",
        "totalLatencyMilliseconds": "44"
    },
    "resourceId": "/SUBSCRIPTIONS/57F1136F-4C47-4F9F-A10B-C3D50D5D73CD/RESOURCEGROUPS/redacted/PROVIDERS/MICROSOFT.CDN/PROFILES/redacted",
    "time": "2023-07-10T01:39:59.7478609Z"
}

Here is a sample of my temp fix, this is not intended to be prod code Note: I removed the remove of event.original since there is another issue on that

PUT _ingest/pipeline/logs-azure_frontdoor.waf-1.0.0
{
  "description": "Pipeline for processing azure frontdoor waf logs",
  "processors": [
    {
      "set": {
        "field": "ecs.version",
        "value": "8.8.0"
      }
    },
    {
      "set": {
        "field": "event.category",
        "value": [
          "network"
        ]
      }
    },
    {
      "set": {
        "field": "event.type",
        "value": [
          "connection"
        ]
      }
    },
    {
      "rename": {
        "field": "message",
        "target_field": "event.original",
        "ignore_missing": true
      }
    },
    {
      "set": {
        "field": "cloud.provider",
        "value": "azure"
      }
    },
    {
      "drop": {
        "description": "Drop if  inavlid json",
        "if": """ctx.event?.original != null && ctx.event.original.contains("records")"""
      }
    },
    {
      "json": {
        "field": "event.original",
        "target_field": "azure.frontdoor.waf"
      }
    },
    {
      "rename": {
        "field": "azure.frontdoor.waf.resourceId",
        "target_field": "azure.frontdoor.resource_id",
        "ignore_missing": true
      }
    },
    {
      "rename": {
        "field": "azure.frontdoor.waf.operationName",
        "target_field": "azure.frontdoor.operation_name",
        "ignore_missing": true
      }
    },
    {
      "rename": {
        "field": "azure.frontdoor.waf.properties.trackingReference",
        "target_field": "azure.frontdoor.tracking_reference",
        "ignore_missing": true
      }
    },
    {
      "rename": {
        "field": "azure.frontdoor.waf.category",
        "target_field": "azure.frontdoor.category",
        "ignore_missing": true
      }
    },
    {
      "rename": {
        "field": "azure.frontdoor.waf.properties.originIP",
        "target_field": "client.ip",
        "ignore_missing": true
      }
    },
    {
      "set": {
        "field": "source.ip",
        "value": "{{{client.ip}}}"
      }
    },
    {
      "rename": {
        "field": "azure.frontdoor.waf.properties.originPort",
        "target_field": "client.port",
        "ignore_missing": true
      }
    },
    {
      "set": {
        "field": "source.port",
        "value": "{{{client.port}}}"
      }
    },
    {
      "rename": {
        "field": "azure.frontdoor.waf.properties.action",
        "target_field": "event.action",
        "ignore_missing": true
      }
    },
    {
      "rename": {
        "field": "azure.frontdoor.waf.properties.socketIP",
        "target_field": "client.address",
        "ignore_missing": true
      }
    },
    {
      "rename": {
        "field": "azure.frontdoor.waf.properties.ruleName",
        "target_field": "rule.name",
        "ignore_missing": true
      }
    },
    {
      "rename": {
        "field": "azure.frontdoor.waf.properties.details",
        "target_field": "azure.frontdoor.waf.details",
        "ignore_missing": true
      }
    },
    {
      "rename": {
        "field": "azure.frontdoor.waf.properties.originName",
        "target_field": "url.domain",
        "ignore_missing": true
      }
    },
    {
      "set": {
        "field": "dns.question.registered_domain",
        "value": "{{{url.domain}}}"
      }
    },
    {
      "set": {
        "field": "dns.question.name",
        "value": "{{{url.domain}}}"
      }
    },
    {
      "set": {
        "field": "http.response.status_code",
        "value": "{{{azure.frontdoor.waf.properties.httpStatusCode}}}"
      }
    },
    {
      "set": {
        "field": "http.request.method",
        "value": "{{{azure.frontdoor.waf.properties.httpVerb}}}"
      }
    },
    {
      "rename": {
        "field": "azure.frontdoor.waf.properties.policy",
        "target_field": "azure.frontdoor.waf.policy",
        "ignore_missing": true
      }
    },
    {
      "rename": {
        "field": "azure.frontdoor.waf.properties.policyMode",
        "target_field": "azure.frontdoor.waf.policy_mode",
        "ignore_missing": true
      }
    },
    {
      "rename": {
        "field": "azure.frontdoor.waf.properties.probeURL",
        "target_field": "url.original",
        "ignore_missing": true
      }
    },
    {
      "convert": {
        "field": "client.port",
        "type": "long",
        "ignore_missing": true
      }
    },
    {
      "date": {
        "field": "azure.frontdoor.waf.time",
        "target_field": "@timestamp",
        "formats": [
          "ISO8601"
        ]
      }
    },
    {
      "remove": {
        "field": [
          "azure.frontdoor.waf.time",
          "azure.frontdoor.waf.properties",
          "azure.frontdoor.waf.details.matches"
        ],
        "ignore_missing": true
      }
    },
    {
      "geoip": {
        "field": "client.ip",
        "target_field": "source.geo",
        "ignore_missing": true
      }
    },
    {
      "geoip": {
        "database_file": "GeoLite2-ASN.mmdb",
        "field": "client.ip",
        "target_field": "source.as",
        "properties": [
          "asn",
          "organization_name"
        ],
        "ignore_missing": true
      }
    },
    {
      "rename": {
        "field": "source.as.asn",
        "target_field": "source.as.number",
        "ignore_missing": true
      }
    },
    {
      "rename": {
        "field": "source.as.organization_name",
        "target_field": "source.as.organization.name",
        "ignore_missing": true
      }
    },
    {
      "pipeline": {
        "name": "logs-azure_frontdoor.waf@custom",
        "ignore_missing_pipeline": true
      }
    }
  ],
  "on_failure": [
    {
      "set": {
        "field": "event.kind",
        "value": "pipeline_error"
      }
    },
    {
      "append": {
        "field": "error.message",
        "value": "{{{ _ingest.on_failure_message }}}"
      }
    }
  ],
  "_meta": {
    "managed_by": "fleet",
    "managed": true,
    "package": {
      "name": "azure_frontdoor"
    }
  }
}

[botelastic[bot]]

Hi! We just realized that we haven't looked into this issue in a while. We're sorry! We're labeling this issue as Stale to make it hit our filters and make sure we get back to it as soon as possible. In the meantime, it'd be extremely helpful if you could take a look at it as well and confirm its relevance. A simple comment with a nice emoji will be enough :+1. Thank you for your contribution!

[elasticmachine]

Pinging @elastic/security-service-integrations (Team:Security-Service Integrations)

[efd6]

According to the MS docs the IP and port are held in clientIP and clientPort. I also do not see originName. Where did you get the logs that do not match the documentation?