panw.panos integration field type conflicts

[colin-stubbs]

Some GlobalProtect logs are received from PanOS devices differently, in particular GlobalProtect logs,

Example indexed document where:

panw.panos.response_time is a string, when it should be a long panw.panos.sequence_number is numeric, when it should be a keyword

{"_index":".ds-logs-panw.panos-customer-2022.09.14-000013","_id":"1","_version":1,"_score":0,"_source":{"agent":{"name":"dc1-customer","id":"d5b0c0d1-6a24-4ac1-a494-14e2be0e7245","type":"filebeat","ephemeral_id":"20f8e27a-96d3-417b-984d-18b8978d52e9","version":"8.3.2"},"log":{"source":{"address":"192.168.1.1:34976"},"syslog":{"severity":{"code":6,"name":"Informational"},"hostname":"FIREWALL.IP-FW01.FIREWALL","priority":14,"facility":{"code":1,"name":"user-level"}}},"elastic_agent":{"id":"d5b0c0d1-6a24-4ac1-a494-14e2be0e7245","version":"8.3.2","snapshot":false},"source":{"nat":{"ip":"203.0.113.1"},"geo":{"continent_name":"Oceania","region_iso_code":"AU-NSW","city_name":"Sydney","country_iso_code":"AU","country_name":"Australia","name":"AU","region_name":"New South Wales","location":{"lon":151.2006,"lat":-33.8715}},"as":{"number":10143,"organization":{"name":"Exetel Pty Ltd"}},"ip":"10.0.0.1","user":{"name":"admin.user"}},"message":"vsys1,gateway-config-release,configuration,,,admin.user,AU,customer-BLAH,203.0.113.1,0.0.0.0,10.0.0.1,0.0.0.0,83fc2cdc-3299-4d6d-880f-872beff30e26,BLAH,5.2.11,Windows,\"Microsoft Windows 11 Enterprise , 64-bit\",1,,,\"\",success,,0,,0,vpn1-FIREWALLcloud-GW,503,0x0,0,0,0,0,,FIREWALL.IP-FW01,1","panw":{"panos":{"attempted_gateways":"0","serial_number":"BLAH","repeat_count":1,"action_flags":"0x0","source":{"nat":{"ip":"203.0.113.1"}},"type":"GLOBALPROTECT","priority":"0","virtual_sys":"vsys1","sequence_number":503,"selection_type":"0","stage":"configuration","client_ver":"5.2.11","sub_type":"0","error_code":"0","response_time":"0"}},"tags":["panw-panos","forwarded"],"network":{"type":"ipv4"},"input":{"type":"udp"},"observer":{"product":"PAN-OS","hostname":"vpn1-FIREWALLcloud-GW","vendor":"Palo Alto Networks","serial_number":"007000001111","type":"firewall"},"@timestamp":"2022-09-14T16:40:43.000+10:00","ecs":{"version":"8.3.0"},"related":{"hosts":["vpn1-FIREWALLcloud-GW","customer-BLAH"],"ip":["10.0.0.1","203.0.113.1"],"user":["admin.user"]},"data_stream":{"namespace":"customer","type":"logs","dataset":"panw.panos"},"host":{"os":{"family":"Windows","full":"Microsoft Windows 11 Enterprise , 64-bit"},"ip":"10.0.0.1","name":"customer-BLAH","id":"83fc2cdc-3299-4d6d-880f-872beff30e26"},"event":{"duration":0,"agent_id_status":"verified","ingested":"2022-09-14T06:40:44Z","code":"gateway-config-release","timezone":"+10:00","created":"2022-09-14T16:40:43.000+10:00","kind":"event","category":["network"],"dataset":"panw.panos","outcome":"success"}},"fields":{"event.category":["network"],"elastic_agent.version":["8.3.2"],"host.os.full":["Microsoft Windows 11 Enterprise , 64-bit"],"panw.panos.client_ver":["5.2.11"],"panw.panos.action_flags":["0x0"],"panw.panos.repeat_count":[1],"observer.vendor":["Palo Alto Networks"],"log.syslog.facility.name":["user-level"],"source.user.name.text":["admin.user"],"source.geo.region_name":["New South Wales"],"log.syslog.severity.name":["Informational"],"source.ip":["10.0.0.1"],"agent.name":["dc1-customer"],"panw.panos.type":["GLOBALPROTECT"],"host.name":["customer-BLAH"],"source.geo.region_iso_code":["AU-NSW"],"event.agent_id_status":["verified"],"event.kind":["event"],"event.outcome":["success"],"source.geo.city_name":["Sydney"],"panw.panos.response_time":["0"],"log.syslog.severity.code":[6],"panw.panos.serial_number":["BLAH"],"input.type":["udp"],"data_stream.type":["logs"],"panw.panos.priority":["0"],"tags":["panw-panos","forwarded"],"observer.serial_number":["007000001111"],"related.user":["admin.user"],"event.code":["gateway-config-release"],"agent.id":["d5b0c0d1-6a24-4ac1-a494-14e2be0e7245"],"ecs.version":["8.3.0"],"panw.panos.sequence_number":[503],"observer.type":["firewall"],"log.source.address":["192.168.1.1:34976"],"event.created":["2022-09-14T06:40:43.000Z"],"agent.version":["8.3.2"],"related.hosts":["vpn1-FIREWALLcloud-GW","customer-BLAH"],"panw.panos.error_code":[0],"source.user.name":["admin.user"],"host.os.family":["Windows"],"source.as.number":[10143],"log.syslog.hostname":["FIREWALL.IP-FW01.FIREWALL"],"panw.panos.source.nat.ip":["203.0.113.1"],"source.geo.location":[{"coordinates":[151.2006,-33.8715],"type":"Point"}],"source.nat.ip":["203.0.113.1"],"host.ip":["10.0.0.1"],"agent.type":["filebeat"],"event.module":["panw"],"related.ip":["10.0.0.1","203.0.113.1"],"panw.panos.sub_type":["0"],"source.geo.country_iso_code":["AU"],"observer.product":["PAN-OS"],"elastic_agent.snapshot":[false],"panw.panos.virtual_sys":["vsys1"],"log.syslog.priority":[14],"host.id":["83fc2cdc-3299-4d6d-880f-872beff30e26"],"panw.panos.selection_type":["0"],"event.timezone":["+10:00"],"network.type":["ipv4"],"source.as.organization.name.text":["Exetel Pty Ltd"],"elastic_agent.id":["d5b0c0d1-6a24-4ac1-a494-14e2be0e7245"],"data_stream.namespace":["customer"],"panw.panos.attempted_gateways":["0"],"source.as.organization.name":["Exetel Pty Ltd"],"source.geo.continent_name":["Oceania"],"panw.panos.stage":["configuration"],"message":["vsys1,gateway-config-release,configuration,,,admin.user,AU,customer-BLAH,203.0.113.1,0.0.0.0,10.0.0.1,0.0.0.0,83fc2cdc-3299-4d6d-880f-872beff30e26,BLAH,5.2.11,Windows,\"Microsoft Windows 11 Enterprise , 64-bit\",1,,,\"\",success,,0,,0,vpn1-FIREWALLcloud-GW,503,0x0,0,0,0,0,,FIREWALL.IP-FW01,1"],"observer.hostname":["vpn1-FIREWALLcloud-GW"],"event.duration":[0],"event.ingested":["2022-09-14T06:40:44.000Z"],"@timestamp":["2022-09-14T06:40:43.000Z"],"data_stream.dataset":["panw.panos"],"source.geo.name":["AU"],"agent.ephemeral_id":["20f8e27a-96d3-417b-984d-18b8978d52e9"],"source.geo.country_name":["Australia"],"event.dataset":["panw.panos"],"log.syslog.facility.code":[1]}}

We have fixed this with the following custom ingest pipeline,

PUT _ingest/pipeline/logs-panw.panos@custom
{
  "version": 1,
  "processors": [
    {
      "convert": {
        "field": "panw.panos.response_time",
        "type": "long",
        "ignore_missing": true,
        "description": "Fix instances where response_time is a string/keyword"
      }
    },
    {
      "convert": {
        "field": "panw.panos.sequence_number",
        "type": "string",
        "ignore_missing": true,
        "description": "Fix instances where sequence_number is not a string/keyword"
      }
    }
  ]
}

Can this please be corrected in the next panw.panos integration release?

[elasticmachine]

Pinging @elastic/security-external-integrations (Team:Security-External Integrations)

[efd6]

@colin-stubbs Can you provide an event.original for this issue?

[efd6]

As of v3.1.2, these fields are being mapped as specified above AFAICS. Can you confirm?

Running a reconstructed message based on the document above through the system tests gives this

{
  "@timestamp": "2022-09-14T06:40:43.000Z",
  "agent": {
    "ephemeral_id": "c9057a1e-3f46-4e5b-bfe0-d36449d1b68d",
    "id": "552540f0-70a3-42b2-a02e-a94b71754dbd",
    "name": "docker-fleet-agent",
    "type": "filebeat",
    "version": "8.4.1"
  },
  "data_stream": {
    "dataset": "panw.panos",
    "namespace": "ep",
    "type": "logs"
  },
  "ecs": {
    "version": "8.4.0"
  },
  "elastic_agent": {
    "id": "552540f0-70a3-42b2-a02e-a94b71754dbd",
    "snapshot": false,
    "version": "8.4.1"
  },
  "event": {
    "agent_id_status": "verified",
    "category": [
      "network"
    ],
    "code": "gateway-config-release",
    "created": "2021-11-09T21:45:36.000Z",
    "dataset": "panw.panos",
    "duration": 0,
    "ingested": "2022-11-04T06:46:04Z",
    "kind": "event",
    "original": "Nov 30 16:09:08 931201168,2021-11-09T21:45:36.000000Z,BLAH,GLOBALPROTECT,globalprotect,9.1,2022-09-14T16:40:43.000+10:00,vsys1,gateway-config-release,configuration,,,admin.user,AU,customer-BLAH,203.0.113.1,0.0.0.0,10.0.0.1,0.0.0.0,83fc2cdc-3299-4d6d-880f-872beff30e26,BLAH,5.2.11,Windows,\"Microsoft Windows 11 Enterprise , 64-bit\",1,,,\"\",success,,0,,0,vpn1-FIREWALLcloud-GW,503,0x0,0,0,0,0,,FIREWALL.IP-FW01,1",
    "outcome": "success",
    "timezone": "+00:00"
  },
  "host": {
    "id": "83fc2cdc-3299-4d6d-880f-872beff30e26",
    "ip": "10.0.0.1",
    "name": "customer-BLAH",
    "os": {
      "family": "Windows",
      "full": "Microsoft Windows 11 Enterprise , 64-bit"
    }
  },
  "input": {
    "type": "log"
  },
  "log": {
    "file": {
      "path": "/tmp/service_logs/panw-panos-globalprotect.log"
    },
    "offset": 0
  },
  "message": "vsys1,gateway-config-release,configuration,,,admin.user,AU,customer-BLAH,203.0.113.1,0.0.0.0,10.0.0.1,0.0.0.0,83fc2cdc-3299-4d6d-880f-872beff30e26,BLAH,5.2.11,Windows,\"Microsoft Windows 11 Enterprise , 64-bit\",1,,,\"\",success,,0,,0,vpn1-FIREWALLcloud-GW,503,0x0,0,0,0,0,,FIREWALL.IP-FW01,1",
  "network": {
    "type": "ipv4"
  },
  "observer": {
    "product": "PAN-OS",
    "serial_number": "BLAH",
    "type": "firewall",
    "vendor": "Palo Alto Networks"
  },
  "panw": {
    "panos": {
      "action_flags": "0x0",
      "client_ver": "5.2.11",
      "device_group_hierarchy1": "1",
      "error_code": 0,
      "gateway": "FIREWALL.IP-FW01",
      "portal": "vpn1-FIREWALLcloud-GW",
      "priority": "0",
      "repeat_count": 1,
      "response_time": 0,
      "selection_type": "0",
      "sequence_number": "503",
      "serial_number": "BLAH",
      "stage": "configuration",
      "sub_type": "globalprotect",
      "type": "GLOBALPROTECT",
      "virtual_sys": "vsys1"
    }
  },
  "related": {
    "hosts": [
      "customer-BLAH"
    ],
    "ip": [
      "10.0.0.1",
      "203.0.113.1"
    ],
    "user": [
      "admin.user"
    ]
  },
  "source": {
    "geo": {
      "name": "AU"
    },
    "ip": "10.0.0.1",
    "nat": {
      "ip": "203.0.113.1"
    },
    "user": {
      "name": "admin.user"
    }
  },
  "tags": [
    "preserve_original_event",
    "panw-panos",
    "forwarded"
  ]
}

[botelastic[bot]]

Hi! We just realized that we haven't looked into this issue in a while. We're sorry! We're labeling this issue as Stale to make it hit our filters and make sure we get back to it as soon as possible. In the meantime, it'd be extremely helpful if you could take a look at it as well and confirm its relevance. A simple comment with a nice emoji will be enough :+1. Thank you for your contribution!