search

elastic / integrations

Elastic Integrations
https://www.elastic.co/integrations
Other
196 stars 424 forks source link

F5 BIG-IP - url decode user_agent fields #11211

Closed willemri closed 1 day ago

willemri commented 5 days ago

Integration Name

F5 BIG-IP [f5_bigip]

Dataset Name

f5_bigip.log

Integration Version

1.19.1

Agent Version

8.14.3

Agent Output Type

elasticsearch

Elasticsearch Version

8.14.3

OS Version and Architecture

RedHat 9 x64

Software/API Version

No response

Error Message

the user_agent.original is still url encoded, which makes it difficult to search on.

Can the field be url decoded? user_agent.original: SonyEricssonK750i%2FR1CA%20Browser%2FSEMC-Browser%2F4.2%20Profile%2FMIDP-2.0%20Configuration%2FCLDC-1.1

Event Original

{
  "_index": ".ds-logs-f5_bigip.log-default-2024.09.11-000006",
  "_id": "hZm7HpIB03V3Uvav4wzx",
  "_version": 1,
  "_score": 0,
  "_source": {
    "agent": {
      "name": "ingest",
      "id": "12aed476-12de-43e9-b19c-4e7c251567f0",
      "type": "filebeat",
      "ephemeral_id": "f6de896e-de0f-49d7-bcf8-192674885a5c",
      "version": "8.14.3"
    },
    "f5_bigip": {
      "log": {
        "errdefs": {
          "msgno": "01490506:5:"
        },
        "hostname": "f5hostname.test.internal",
        "access": {
          "profile": "/Common/testuri"
        },
        "partition": "Common",
        "session": {
          "id": [
            "e8ca19a9"
          ]
        },
        "telemetry": {
          "event": {
            "category": "APM"
          },
          "timestamp": "2024-09-23T11:54:35.866Z"
        },
        "user": {
          "agent": "SonyEricssonK750i%2FR1CA%20Browser%2FSEMC-Browser%2F4.2%20Profile%2FMIDP-2.0%20Configuration%2FCLDC-1.1"
        },
        "partition_name": "Common",
        "tenant": "Common"
      }
    },
    "elastic_agent": {
      "id": "12aed476-12de-43e9-b19c-4e7c251567f0",
      "version": "8.14.3",
      "snapshot": false
    },
    "tags": [
      "preserve_original_event",
      "preserve_duplicate_custom_fields",
      "forwarded",
      "f5_bigip-log"
    ],
    "input": {
      "type": "http_endpoint"
    },
    "observer": {
      "product": "Application Performance Monitoring",
      "vendor": "F5"
    },
    "@timestamp": "2024-09-23T11:54:35.866Z",
    "ecs": {
      "version": "8.11.0"
    },
    "related": {
      "hosts": [
        "f5hostname.test.internal"
      ]
    },
    "data_stream": {
      "namespace": "default",
      "type": "logs",
      "dataset": "f5_bigip.log"
    },
    "host": {
      "name": "f5hostname.test.internal"
    },
    "event": {
      "agent_id_status": "verified",
      "ingested": "2024-09-23T11:54:42Z",
      "original": "{\"Access_Profile\":\"/Common/testuri\",\"Partition\":\"Common\",\"Session_ID\":\"e8ca19a9\",\"User_Agent\":\"SonyEricssonK750i%2FR1CA%20Browser%2FSEMC-Browser%2F4.2%20Profile%2FMIDP-2.0%20Configuration%2FCLDC-1.1\",\"errdefs_msgno\":\"01490506:5:\",\"f5telemetry_timestamp\":\"2024-09-23T11:54:35.866Z\",\"hostname\":\"f5hostname.test.internal\",\"originalRawData\":\"hostname=\\\"f5hostname.test.internal\\\",errdefs_msgno=\\\"01490506:5:\\\",partition_name=\\\"Common\\\",session_id=\\\"e8ca19a9\\\",Access_Profile=\\\"/Common/testuri\\\",Partition=\\\"Common\\\",Session_ID=\\\"e8ca19a9\\\",User_Agent=\\\"SonyEricssonK750i%2FR1CA%20Browser%2FSEMC-Browser%2F4.2%20Profile%2FMIDP-2.0%20Configuration%2FCLDC-1.1\\\"\",\"partition_name\":\"Common\",\"session_id\":\"e8ca19a9\",\"telemetryEventCategory\":\"APM\",\"tenant\":\"Common\"}",
      "kind": "event",
      "category": [
        "network"
      ],
      "type": [
        "info"
      ],
      "dataset": "f5_bigip.log"
    },
    "user_agent": {
      "original": "SonyEricssonK750i%2FR1CA%20Browser%2FSEMC-Browser%2F4.2%20Profile%2FMIDP-2.0%20Configuration%2FCLDC-1.1",
      "name": "Other",
      "device": {
        "name": "Ericsson K750i"
      }
    }
  },
  "fields": {
    "f5_bigip.log.errdefs.msgno": [
      "01490506:5:"
    ],
    "elastic_agent.version": [
      "8.14.3"
    ],
    "event.category": [
      "network"
    ],
    "host.name.text": [
      "f5hostname.test.internal"
    ],
    "user_agent.original.text": [
      "SonyEricssonK750i%2FR1CA%20Browser%2FSEMC-Browser%2F4.2%20Profile%2FMIDP-2.0%20Configuration%2FCLDC-1.1"
    ],
    "f5_bigip.log.partition_name": [
      "Common"
    ],
    "f5_bigip.log.telemetry.event.category": [
      "APM"
    ],
    "observer.vendor": [
      "F5"
    ],
    "agent.type": [
      "filebeat"
    ],
    "f5_bigip.log.session.id": [
      "e8ca19a9"
    ],
    "event.module": [
      "f5_bigip"
    ],
    "agent.name.text": [
      "ingest"
    ],
    "agent.name": [
      "ingest"
    ],
    "observer.product": [
      "Application Performance Monitoring"
    ],
    "elastic_agent.snapshot": [
      false
    ],
    "f5_bigip.log.partition": [
      "Common"
    ],
    "host.name": [
      "f5hostname.test.internal"
    ],
    "event.agent_id_status": [
      "verified"
    ],
    "event.kind": [
      "event"
    ],
    "user_agent.original": [
      "SonyEricssonK750i%2FR1CA%20Browser%2FSEMC-Browser%2F4.2%20Profile%2FMIDP-2.0%20Configuration%2FCLDC-1.1"
    ],
    "event.original": [
      "{\"Access_Profile\":\"/Common/testuri\",\"Partition\":\"Common\",\"Session_ID\":\"e8ca19a9\",\"User_Agent\":\"SonyEricssonK750i%2FR1CA%20Browser%2FSEMC-Browser%2F4.2%20Profile%2FMIDP-2.0%20Configuration%2FCLDC-1.1\",\"errdefs_msgno\":\"01490506:5:\",\"f5telemetry_timestamp\":\"2024-09-23T11:54:35.866Z\",\"hostname\":\"f5hostname.test.internal\",\"originalRawData\":\"hostname=\\\"f5hostname.test.internal\\\",errdefs_msgno=\\\"01490506:5:\\\",partition_name=\\\"Common\\\",session_id=\\\"e8ca19a9\\\",Access_Profile=\\\"/Common/testuri\\\",Partition=\\\"Common\\\",Session_ID=\\\"e8ca19a9\\\",User_Agent=\\\"SonyEricssonK750i%2FR1CA%20Browser%2FSEMC-Browser%2F4.2%20Profile%2FMIDP-2.0%20Configuration%2FCLDC-1.1\\\"\",\"partition_name\":\"Common\",\"session_id\":\"e8ca19a9\",\"telemetryEventCategory\":\"APM\",\"tenant\":\"Common\"}"
    ],
    "elastic_agent.id": [
      "12aed476-12de-43e9-b19c-4e7c251567f0"
    ],
    "data_stream.namespace": [
      "default"
    ],
    "input.type": [
      "http_endpoint"
    ],
    "user_agent.name": [
      "Other"
    ],
    "data_stream.type": [
      "logs"
    ],
    "user_agent.device.name.text": [
      "Ericsson K750i"
    ],
    "f5_bigip.log.user.agent": [
      "SonyEricssonK750i%2FR1CA%20Browser%2FSEMC-Browser%2F4.2%20Profile%2FMIDP-2.0%20Configuration%2FCLDC-1.1"
    ],
    "tags": [
      "preserve_original_event",
      "preserve_duplicate_custom_fields",
      "forwarded",
      "f5_bigip-log"
    ],
    "f5_bigip.log.access.profile": [
      "/Common/testuri"
    ],
    "event.ingested": [
      "2024-09-23T11:54:42.000Z"
    ],
    "@timestamp": [
      "2024-09-23T11:54:35.866Z"
    ],
    "agent.id": [
      "12aed476-12de-43e9-b19c-4e7c251567f0"
    ],
    "user_agent.name.text": [
      "Other"
    ],
    "ecs.version": [
      "8.11.0"
    ],
    "f5_bigip.log.telemetry.timestamp": [
      "2024-09-23T11:54:35.866Z"
    ],
    "data_stream.dataset": [
      "f5_bigip.log"
    ],
    "event.type": [
      "info"
    ],
    "agent.ephemeral_id": [
      "f6de896e-de0f-49d7-bcf8-192674885a5c"
    ],
    "agent.version": [
      "8.14.3"
    ],
    "related.hosts": [
      "f5hostname.test.internal"
    ],
    "user_agent.device.name": [
      "Ericsson K750i"
    ],
    "f5_bigip.log.hostname": [
      "f5hostname.test.internal"
    ],
    "event.dataset": [
      "f5_bigip.log"
    ],
    "f5_bigip.log.tenant": [
      "Common"
    ]
  }
}

What did you do?

{
  "urldecode": { "field": "[user_agent][original]"  }
  "urldecode": { "field": "[f5_bigip][log][user][agent]"  }
}

What did you see?

{
    "user_agent": {
      "original": "SonyEricssonK750i%2FR1CA%20Browser%2FSEMC-Browser%2F4.2%20Profile%2FMIDP-2.0%20Configuration%2FCLDC-1.1",
      "name": "Other",
      "device": {
        "name": "Ericsson K750i"
      }
    }
  },
    "user_agent.original.text": [
      "SonyEricssonK750i%2FR1CA%20Browser%2FSEMC-Browser%2F4.2%20Profile%2FMIDP-2.0%20Configuration%2FCLDC-1.1"
    ],
    "user_agent.original": [
      "SonyEricssonK750i%2FR1CA%20Browser%2FSEMC-Browser%2F4.2%20Profile%2FMIDP-2.0%20Configuration%2FCLDC-1.1"
    ],

What did you expect to see?

{
    "user_agent": {
      "original": "SonyEricssonK750i/R1CA Browser/SEMC-Browser/4.2 Profile/MIDP-2.0 Configuration/CLDC-1.1",
      "name": "Other",
      "device": {
        "name": "Ericsson K750i"
      }
    }
  },
    "user_agent.original.text": [
      "SonyEricssonK750i/R1CA Browser/SEMC-Browser/4.2 Profile/MIDP-2.0 Configuration/CLDC-1.1"
    ],
    "user_agent.original": [
      "SonyEricssonK750i/R1CA Browser/SEMC-Browser/4.2 Profile/MIDP-2.0 Configuration/CLDC-1.1"
    ],

Anything else?

No response

elasticmachine commented 5 days ago

Pinging @elastic/security-service-integrations (Team:Security-Service Integrations)