search

zmoog / public-notes

Apache License 2.0
0 stars 1 forks source link

How to test an ingest pipeline #10

Closed zmoog closed 1 year ago

zmoog commented 1 year ago

I fixed a small bug in a PR and want to run a final test.

I want to test the following:

zmoog commented 1 year ago

I expect version 1.5.9 to fail.

POST _ingest/pipeline/logs-azure.application_gateway-1.5.9/_simulate
{
  "docs": [
    {
      "_source": {
        "tags": {
          "preserve_original_event": true
        },
        "event": {
          "original": "{\"resourceId\":\"/SUBSCRIPTIONS/23103928-B2CF-472A-8CDB-0146E2849129/RESOURCEGROUPS/PEERINGTEST/PROVIDERS/MICROSOFT.NETWORK/APPLICATIONGATEWAYS/Application-Gateway-Name\",\"operationName\":\"ApplicationGatewayAccess\",\"timestamp\":\"2017-04-26T19:27:38Z\",\"category\":\"ApplicationGatewayAccessLog\",\"properties\":{\"instanceId\":\"ApplicationGatewayRole_IN_0\",\"clientIP\":\"67.43.156.7\",\"clientPort\":46886,\"httpMethod\":\"GET\",\"requestUri\":\"/phpmyadmin/scripts/setup.php\",\"requestQuery\":\"X-AzureApplicationGateway-CACHE-HIT=0&SERVER-ROUTED=10.4.0.4&X-AzureApplicationGateway-LOG-ID=874f1f0f-6807-41c9-b7bc-f3cfa74aa0b1&SERVER-STATUS=404\",\"userAgent\":\"-\",\"httpStatus\":404,\"httpVersion\":\"HTTP/1.0\",\"receivedBytes\":65,\"sentBytes\":553,\"timeTaken\":205,\"sslEnabled\":\"off\",\"host\":\"www.contoso.com\",\"originalHost\":\"www.contoso.com\"}}"

        },
        "message": "{\"resourceId\":\"/SUBSCRIPTIONS/23103928-B2CF-472A-8CDB-0146E2849129/RESOURCEGROUPS/PEERINGTEST/PROVIDERS/MICROSOFT.NETWORK/APPLICATIONGATEWAYS/Application-Gateway-Name\",\"operationName\":\"ApplicationGatewayAccess\",\"timestamp\":\"2017-04-26T19:27:38Z\",\"category\":\"ApplicationGatewayAccessLog\",\"properties\":{\"instanceId\":\"ApplicationGatewayRole_IN_0\",\"clientIP\":\"67.43.156.7\",\"clientPort\":46886,\"httpMethod\":\"GET\",\"requestUri\":\"/phpmyadmin/scripts/setup.php\",\"requestQuery\":\"X-AzureApplicationGateway-CACHE-HIT=0&SERVER-ROUTED=10.4.0.4&X-AzureApplicationGateway-LOG-ID=874f1f0f-6807-41c9-b7bc-f3cfa74aa0b1&SERVER-STATUS=404\",\"userAgent\":\"-\",\"httpStatus\":404,\"httpVersion\":\"HTTP/1.0\",\"receivedBytes\":65,\"sentBytes\":553,\"timeTaken\":205,\"sslEnabled\":\"off\",\"host\":\"www.contoso.com\",\"originalHost\":\"www.contoso.com\"}}"

      }
    }
  ]
}

And it fails with the expected error "field [event.original] already exists rename":

{
  "docs": [
    {
      "doc": {
        "_index": "_index",
        "_id": "_id",
        "_version": "-3",
        "_source": {
          "event": {
            "original": """{"resourceId":"/SUBSCRIPTIONS/23103928-B2CF-472A-8CDB-0146E2849129/RESOURCEGROUPS/PEERINGTEST/PROVIDERS/MICROSOFT.NETWORK/APPLICATIONGATEWAYS/Application-Gateway-Name","operationName":"ApplicationGatewayAccess","timestamp":"2017-04-26T19:27:38Z","category":"ApplicationGatewayAccessLog","properties":{"instanceId":"ApplicationGatewayRole_IN_0","clientIP":"67.43.156.7","clientPort":46886,"httpMethod":"GET","requestUri":"/phpmyadmin/scripts/setup.php","requestQuery":"X-AzureApplicationGateway-CACHE-HIT=0&SERVER-ROUTED=10.4.0.4&X-AzureApplicationGateway-LOG-ID=874f1f0f-6807-41c9-b7bc-f3cfa74aa0b1&SERVER-STATUS=404","userAgent":"-","httpStatus":404,"httpVersion":"HTTP/1.0","receivedBytes":65,"sentBytes":553,"timeTaken":205,"sslEnabled":"off","host":"www.contoso.com","originalHost":"www.contoso.com"}}""",
            "kind": "event",
            "category": [
              "network"
            ],
            "type": [
              "connection"
            ]
          },
          "ecs": {
            "version": "8.5.0"
          },
          "error": {
            "message": [
              "field [event.original] already exists rename"
            ]
          },
          "tags": {
            "preserve_original_event": true
          }
        },
        "_ingest": {
          "timestamp": "2023-02-24T12:24:00.816687068Z"
        }
      }
    }
  ]
}
zmoog commented 1 year ago

I expect version 1.5.10 to succeed.

POST _ingest/pipeline/logs-azure.application_gateway-1.5.10/_simulate
{
  "docs": [
    {
      "_source": {
        "event": {
          "original": "{\"resourceId\":\"/SUBSCRIPTIONS/23103928-B2CF-472A-8CDB-0146E2849129/RESOURCEGROUPS/PEERINGTEST/PROVIDERS/MICROSOFT.NETWORK/APPLICATIONGATEWAYS/Application-Gateway-Name\",\"operationName\":\"ApplicationGatewayAccess\",\"timestamp\":\"2017-04-26T19:27:38Z\",\"category\":\"ApplicationGatewayAccessLog\",\"properties\":{\"instanceId\":\"ApplicationGatewayRole_IN_0\",\"clientIP\":\"67.43.156.7\",\"clientPort\":46886,\"httpMethod\":\"GET\",\"requestUri\":\"/phpmyadmin/scripts/setup.php\",\"requestQuery\":\"X-AzureApplicationGateway-CACHE-HIT=0&SERVER-ROUTED=10.4.0.4&X-AzureApplicationGateway-LOG-ID=874f1f0f-6807-41c9-b7bc-f3cfa74aa0b1&SERVER-STATUS=404\",\"userAgent\":\"-\",\"httpStatus\":404,\"httpVersion\":\"HTTP/1.0\",\"receivedBytes\":65,\"sentBytes\":553,\"timeTaken\":205,\"sslEnabled\":\"off\",\"host\":\"www.contoso.com\",\"originalHost\":\"www.contoso.com\"}}"

        },
        "message": "{\"resourceId\":\"/SUBSCRIPTIONS/23103928-B2CF-472A-8CDB-0146E2849129/RESOURCEGROUPS/PEERINGTEST/PROVIDERS/MICROSOFT.NETWORK/APPLICATIONGATEWAYS/Application-Gateway-Name\",\"operationName\":\"ApplicationGatewayAccess\",\"timestamp\":\"2017-04-26T19:27:38Z\",\"category\":\"ApplicationGatewayAccessLog\",\"properties\":{\"instanceId\":\"ApplicationGatewayRole_IN_0\",\"clientIP\":\"67.43.156.7\",\"clientPort\":46886,\"httpMethod\":\"GET\",\"requestUri\":\"/phpmyadmin/scripts/setup.php\",\"requestQuery\":\"X-AzureApplicationGateway-CACHE-HIT=0&SERVER-ROUTED=10.4.0.4&X-AzureApplicationGateway-LOG-ID=874f1f0f-6807-41c9-b7bc-f3cfa74aa0b1&SERVER-STATUS=404\",\"userAgent\":\"-\",\"httpStatus\":404,\"httpVersion\":\"HTTP/1.0\",\"receivedBytes\":65,\"sentBytes\":553,\"timeTaken\":205,\"sslEnabled\":\"off\",\"host\":\"www.contoso.com\",\"originalHost\":\"www.contoso.com\"}}"

      }
    }
  ]
}
{
  "docs": [
    {
      "doc": {
        "_index": "_index",
        "_id": "_id",
        "_version": "-3",
        "_source": {
          "cloud": {
            "provider": "azure",
            "account": {
              "id": "23103928-B2CF-472A-8CDB-0146E2849129"
            }
          },
          "observer": {
            "product": "Web Application Firewall",
            "vendor": "Azure",
            "name": "Application-Gateway-Name",
            "type": "firewall"
          },
          "@timestamp": "2017-04-26T19:27:38.000Z",
          "ecs": {
            "version": "8.5.0"
          },
          "related": {
            "hosts": [
              "www.contoso.com"
            ],
            "ip": [
              "67.43.156.7"
            ]
          },
          "destination": {
            "address": "www.contoso.com",
            "bytes": 553,
            "domain": "www.contoso.com"
          },
          "http": {
            "request": {
              "method": "GET"
            },
            "response": {
              "status_code": 404
            },
            "version": "1.0"
          },
          "source": {
            "geo": {
              "continent_name": "Asia",
              "country_iso_code": "BT",
              "country_name": "Bhutan",
              "location": {
                "lon": 90.5,
                "lat": 27.5
              }
            },
            "as": {
              "number": 35908
            },
            "address": "67.43.156.7",
            "port": 46886,
            "bytes": 65,
            "ip": "67.43.156.7"
          },
          "event": {
            "kind": "event",
            "category": [
              "network"
            ],
            "type": [
              "connection"
            ]
          },
          "url": {
            "path": "/phpmyadmin/scripts/setup.php",
            "query": "X-AzureApplicationGateway-CACHE-HIT=0&SERVER-ROUTED=10.4.0.4&X-AzureApplicationGateway-LOG-ID=874f1f0f-6807-41c9-b7bc-f3cfa74aa0b1&SERVER-STATUS=404",
            "domain": "www.contoso.com"
          },
          "azure": {
            "subscription_id": "23103928-B2CF-472A-8CDB-0146E2849129",
            "resource": {
              "provider": "MICROSOFT.NETWORK/APPLICATIONGATEWAYS",
              "name": "Application-Gateway-Name",
              "id": "/SUBSCRIPTIONS/23103928-B2CF-472A-8CDB-0146E2849129/RESOURCEGROUPS/PEERINGTEST/PROVIDERS/MICROSOFT.NETWORK/APPLICATIONGATEWAYS/Application-Gateway-Name",
              "group": "PEERINGTEST"
            },
            "application_gateway": {
              "operation_name": "ApplicationGatewayAccess",
              "instance_id": "ApplicationGatewayRole_IN_0"
            }
          },
          "network": {
            "protocol": "http",
            "bytes": 618
          }
        },
        "_ingest": {
          "timestamp": "2023-02-24T12:20:55.380622722Z"
        }
      }
    }
  ]
}