search

elastic / integrations

Elastic Integrations
https://www.elastic.co/integrations
Other
194 stars 420 forks source link

[AWS]: Request Parameters and Response Elements Too Long to Search or Filter #10818

Open terrancedejesus opened 3 weeks ago

terrancedejesus commented 3 weeks ago

Integration Name

AWS [aws]

Dataset Name

aws.cloudtrail

Integration Version

2.23.0

Agent Version

8.14.3

Agent Output Type

elasticsearch

Elasticsearch Version

8.14.1

OS Version and Architecture

Ubuntu 22.04

Software/API Version

No response

Error Message

Ignored Value: The value in this field is too long and cannot be searched or filtered.

Event Original

{
  "_index": ".ds-logs-aws.cloudtrail-default-2024.07.30-000002",
  "_id": "3180f6e90d-000000025993",
  "_version": 1,
  "_score": 0,
  "_ignored": [
    "aws.cloudtrail.response_elements"
  ],
  "_source": {
    "agent": {
      "name": "ip-172-31-33-37",
      "id": "6c631dc5-5349-45b3-8cd1-483990f30255",
      "ephemeral_id": "b9c4fff6-a880-4c33-8ed4-3818631b989e",
      "type": "filebeat",
      "version": "8.14.3"
    },
    "log": {
      "file": {
        "path": "https://<SANITIZED_BUCKET_NAME>.s3.us-west-2.amazonaws.com/AWSLogs/<SANITIZED_ACCOUNT_ID>/CloudTrail/us-east-1/2024/08/20/<SANITIZED_ACCOUNT_ID>_CloudTrail_us-east-1_20240820T0300Z_0UDy81vfIWWF9Ln5.json.gz"
      },
      "offset": 25993
    },
    "elastic_agent": {
      "id": "6c631dc5-5349-45b3-8cd1-483990f30255",
      "version": "8.14.3",
      "snapshot": false
    },
    "source": {
      "geo": {
        "continent_name": "North America",
        "region_iso_code": "US-OH",
        "city_name": "Massillon",
        "country_iso_code": "US",
        "country_name": "United States",
        "region_name": "Ohio",
        "location": {
          "lon": -81.4971,
          "lat": 40.8133
        }
      },
      "as": {
        "number": 12097,
        "organization": {
          "name": "MASSCOM"
        }
      },
      "address": "<SANITIZED_IP_ADDRESS>",
      "ip": "<SANITIZED_IP_ADDRESS>"
    },
    "tags": [
      "forwarded",
      "aws-cloudtrail"
    ],
    "cloud": {
      "region": "us-east-1",
      "account": {
        "id": "<SANITIZED_ACCOUNT_ID>"
      }
    },
    "input": {
      "type": "aws-s3"
    },
    "@timestamp": "2024-08-20T02:59:33.000Z",
    "ecs": {
      "version": "8.11.0"
    },
    "related": {
      "user": [
        "stratus"
      ]
    },
    "data_stream": {
      "namespace": "default",
      "type": "logs",
      "dataset": "aws.cloudtrail"
    },
    "tls": {
      "cipher": "TLS_AES_128_GCM_SHA256",
      "client": {
        "server_name": "sts.us-east-1.amazonaws.com"
      },
      "version": "1.3",
      "version_protocol": "tls"
    },
    "event": {
      "agent_id_status": "verified",
      "ingested": "2024-08-20T02:59:54Z",
      "provider": "sts.amazonaws.com",
      "created": "2024-08-20T02:59:47.685Z",
      "kind": "event",
      "action": "GetFederationToken",
      "id": "c6af9006-1233-4535-bc52-40ac6b7b5a7a",
      "type": [
        "info"
      ],
      "dataset": "aws.cloudtrail",
      "outcome": "success"
    },
    "aws": {
      "s3": {
        "bucket": {
          "name": "<SANITIZED_BUCKET_NAME>",
          "arn": "arn:aws:s3:::<SANITIZED_BUCKET_NAME>"
        },
        "object": {
          "key": "AWSLogs/<SANITIZED_ACCOUNT_ID>/CloudTrail/us-east-1/2024/08/20/<SANITIZED_ACCOUNT_ID>_CloudTrail_us-east-1_20240820T0300Z_0UDy81vfIWWF9Ln5.json.gz"
        }
      },
      "cloudtrail": {
        "event_version": "1.08",
        "flattened": {
          "request_parameters": {
            "name": "consoler",
            "policyArns": [
              {
                "arn": "arn:aws:iam::aws:policy/AdministratorAccess"
              }
            ]
          },
          "response_elements": {
            "federatedUser": {
              "arn": "arn:aws:sts::<SANITIZED_ACCOUNT_ID>:federated-user/consoler",
              "federatedUserId": "<SANITIZED_ACCOUNT_ID>:consoler"
            },
            "credentials": {
              "accessKeyId": "<SANITIZED_ACCESS_KEY_ID>",
              "sessionToken": "<SANITIZED_SESSION_TOKEN>",
              "expiration": "Aug 20, 2024, 2:59:33 PM"
            },
            "packedPolicySize": 7
          }
        },
        "event_type": "AwsApiCall",
        "read_only": false,
        "user_identity": {
          "access_key_id": "<SANITIZED_ACCESS_KEY_ID>",
          "type": "IAMUser",
          "arn": "arn:aws:iam::<SANITIZED_ACCOUNT_ID>:user/stratus"
        },
        "recipient_account_id": "<SANITIZED_ACCOUNT_ID>",
        "event_category": "Management",
        "request_parameters": "{name=consoler, policyArns=[{arn=arn:aws:iam::aws:policy/AdministratorAccess}]}",
        "request_id": "00026031-2409-4418-b4ec-f1341e9638d1",
        "response_elements": "{federatedUser={arn=arn:aws:sts::<SANITIZED_ACCOUNT_ID>:federated-user/consoler, federatedUserId=<SANITIZED_ACCOUNT_ID>:consoler}, credentials={accessKeyId=<SANITIZED_ACCESS_KEY_ID>, sessionToken=<SANITIZED_SESSION_TOKEN>, expiration=Aug 20, 2024, 2:59:33 PM}, packedPolicySize=7}",
        "management_event": true
      }
    },
    "user": {
      "name": "stratus",
      "id": "<SANITIZED_USER_ID>"
    },
    "user_agent": {
      "original": "aws-cli/2.11.15 Python/3.11.3 Darwin/23.6.0 exe/x86_64 prompt/off command/sts.get-federation-token",
      "name": "aws-cli",
      "device": {
        "name": "Other"
      },
      "version": "2.11.15"
    }
  },
  "fields": {
    "aws.cloudtrail.request_parameters.text": [
      "{name=consoler, policyArns=[{arn=arn:aws:iam::aws:policy/AdministratorAccess}]}"
    ],
    "elastic_agent.version": [
      "8.14.3"
    ],
    "tls.version_protocol": [
      "tls"
    ],
    "user_agent.original.text": [
      "aws-cli/2.11.15 Python/3.11.3 Darwin/23.6.0 exe/x86_64 prompt/off command/sts.get-federation-token"
    ],
    "aws.cloudtrail.flattened.response_elements": [
      {
        "federatedUser": {
          "arn": "arn:aws:sts::<SANITIZED_ACCOUNT_ID>:federated-user/consoler",
          "federatedUserId": "<SANITIZED_ACCOUNT_ID>:consoler"
        },
        "credentials": {
          "accessKeyId": "<SANITIZED_ACCESS_KEY_ID>",
          "sessionToken": "<SANITIZED_SESSION_TOKEN>",
          "expiration": "Aug 20, 2024, 2:59:33 PM"
        },
        "packedPolicySize": 7
      }
    ],
    "agent.name.text": [
      "ip-172-31-33-37"
    ],
    "source.geo.region_name": [
      "Ohio"
    ],
    "source.ip": [
      "<SANITIZED_IP_ADDRESS>"
    ],
    "agent.name": [
      "ip-172-31-33-37"
    ],
    "user_agent.version": [
      "2.11.15"
    ],
    "event.agent_id_status": [
      "verified"
    ],
    "source.geo.region_iso_code": [
      "US-OH"
    ],
    "aws.cloudtrail.management_event": [
      "true"
    ],
    "event.kind": [
      "event"
    ],
    "aws.cloudtrail.user_identity.arn": [
      "arn:aws:iam::<SANITIZED_ACCOUNT_ID>:user/stratus"
    ],
    "event.outcome": [
      "success"
    ],
    "source.geo.city_name": [
      "Massillon"
    ],
    "tls.version": [
      "1.3"
    ],
    "user_agent.original": [
      "aws-cli/2.11.15 Python/3.11.3 Darwin/23.6.0 exe/x86_64 prompt/off command/sts.get-federation-token"
    ],
    "cloud.region": [
      "us-east-1"
    ],
    "user.id": [
      "<SANITIZED_USER_ID>"
    ],
    "input.type": [
      "aws-s3"
    ],
    "log.offset": [
      25993
    ],
    "user_agent.name": [
      "aws-cli"
    ],
    "data_stream.type": [
      "logs"
    ],
    "related.user": [
      "stratus"
    ],
    "tags": [
      "forwarded",
      "aws-cloudtrail"
    ],
    "event.provider": [
      "sts.amazonaws.com"
    ],
    "agent.id": [
      "6c631dc5-5349-45b3-8cd1-483990f30255"
    ],
    "ecs.version": [
      "8.11.0"
    ],
    "event.created": [
      "2024-08-20T02:59:47.685Z"
    ],
    "aws.cloudtrail.event_version": [
      "1.08"
    ],
    "agent.version": [
      "8.14.3"
    ],
    "source.as.number": [
      12097
    ],
    "aws.cloudtrail.read_only": [
      false
    ],
    "aws.cloudtrail.event_category": [
      "Management"
    ],
    "aws.cloudtrail.user_identity.type": [
      "IAMUser"
    ],
    "aws.s3.bucket.arn": [
      "arn:aws:s3:::<SANITIZED_BUCKET_NAME>"
    ],
    "aws.cloudtrail.recipient_account_id": [
      "<SANITIZED_ACCOUNT_ID>"
    ],
    "aws.cloudtrail.request_id": [
      "00026031-2409-4418-b4ec-f1341e9638d1"
    ],
    "tls.cipher": [
      "TLS_AES_128_GCM_SHA256"
    ],
    "user.name": [
      "stratus"
    ],
    "source.geo.location": [
      {
        "coordinates": [
          -81.4971,
          40.8133
        ],
        "type": "Point"
      }
    ],
    "source.address": [
      "<SANITIZED_IP_ADDRESS>"
    ],
    "aws.cloudtrail.flattened.request_parameters": [
      {
        "name": "consoler",
        "policyArns": [
          {
            "arn": "arn:aws:iam::aws:policy/AdministratorAccess"
          }
        ]
      }
    ],
    "agent.type": [
      "filebeat"
    ],
    "event.module": [
      "aws"
    ],
    "source.geo.country_iso_code": [
      "US"
    ],
    "aws.cloudtrail.response_elements.text": [
      "{federatedUser={arn=arn:aws:sts::<SANITIZED_ACCOUNT_ID>:federated-user/consoler, federatedUserId=<SANITIZED_ACCOUNT_ID>:consoler}, credentials={accessKeyId=<SANITIZED_ACCESS_KEY_ID>, sessionToken=<SANITIZED_SESSION_TOKEN>, expiration=Aug 20, 2024, 2:59:33 PM}, packedPolicySize=7}"
    ],
    "elastic_agent.snapshot": [
      false
    ],
    "aws.cloudtrail.event_type": [
      "AwsApiCall"
    ],
    "aws.s3.bucket.name": [
      "<SANITIZED_BUCKET_NAME>"
    ],
    "source.as.organization.name.text": [
      "MASSCOM"
    ],
    "elastic_agent.id": [
      "6c631dc5-5349-45b3-8cd1-483990f30255"
    ],
    "data_stream.namespace": [
      "default"
    ],
    "source.as.organization.name": [
      "MASSCOM"
    ],
    "source.geo.continent_name": [
      "North America"
    ],
    "tls.client.server_name": [
      "sts.us-east-1.amazonaws.com"
    ],
    "event.action": [
      "GetFederationToken"
    ],
    "event.ingested": [
      "2024-08-20T02:59:54.000Z"
    ],
    "@timestamp": [
      "2024-08-20T02:59:33.000Z"
    ],
    "cloud.account.id": [
      "<SANITIZED_ACCOUNT_ID>"
    ],
    "aws.cloudtrail.user_identity.access_key_id": [
      "<SANITIZED_ACCESS_KEY_ID>"
    ],
    "data_stream.dataset": [
      "aws.cloudtrail"
    ],
    "event.type": [
      "info"
    ],
    "log.file.path": [
      "https://<SANITIZED_BUCKET_NAME>.s3.us-west-2.amazonaws.com/AWSLogs/<SANITIZED_ACCOUNT_ID>/CloudTrail/us-east-1/2024/08/20/<SANITIZED_ACCOUNT_ID>_CloudTrail_us-east-1_20240820T0300Z_0UDy81vfIWWF9Ln5.json.gz"
    ],
    "agent.ephemeral_id": [
      "b9c4fff6-a880-4c33-8ed4-3818631b989e"
    ],
    "aws.cloudtrail.request_parameters": [
      "{name=consoler, policyArns=[{arn=arn:aws:iam::aws:policy/AdministratorAccess}]}"
    ],
    "event.id": [
      "c6af9006-1233-4535-bc52-40ac6b7b5a7a"
    ],
    "source.geo.country_name": [
      "United States"
    ],
    "user_agent.device.name": [
      "Other"
    ],
    "aws.s3.object.key": [
      "AWSLogs/<SANITIZED_ACCOUNT_ID>/CloudTrail/us-east-1/2024/08/20/<SANITIZED_ACCOUNT_ID>_CloudTrail_us-east-1_20240820T0300Z_0UDy81vfIWWF9Ln5.json.gz"
    ],
    "event.dataset": [
      "aws.cloudtrail"
    ],
    "user.name.text": [
      "stratus"
    ]
  },
  "ignored_field_values": {
    "aws.cloudtrail.response_elements": [
      "{federatedUser={arn=arn:aws:sts::<SANITIZED_ACCOUNT_ID>:federated-user/consoler, federatedUserId=<SANITIZED_ACCOUNT_ID>:consoler}, credentials={accessKeyId=<SANITIZED_ACCESS_KEY_ID>, sessionToken=<SANITIZED_SESSION_TOKEN>, expiration=Aug 20, 2024, 2:59:33 PM}, packedPolicySize=7}"
    ]
  }
}

What did you do?

AWS integration with Collect CloudTrail logs from S3 where SQS queue URL is used.

What did you see?

Screenshot 2024-08-19 at 11 06 55 PM

What did you expect to see?

I expected to be able to search and filter on this field as well as aws.cloudtrail.request_parameters. I am unable to use, for example, ES|QL's DISSECT command because of this.

Anything else?

If I understand correctly, the field size accepted character count just needs increased. Please note that aws.cloudtrail.request_parameters and aws.cloudtrail.response_elements can be very large fields.

terrancedejesus commented 3 weeks ago

@andrewkroh - Please let me know if someone would just prefer access to my stack with this data available. Happy to share.