search

openappsec / openappsec

open-appsec is a machine learning security engine that preemptively and automatically prevents threats against Web Application & APIs. This repo include the main code and logic.
https://openappsec.io
Apache License 2.0
797 stars 58 forks source link

OpenAppSec NGinx Ingress is not detecting any traffic #45

Closed kzgrzendek closed 12 months ago

kzgrzendek commented 1 year ago

Hi,

First of all, I'd like to thank the team for providing what seems an awesome tool, and a suitable solution for all those who wants an opensource kube-native and prod ready WAF solution.

Problem description

By following the howto videos and the tutorial section of the docs, I just can't get to have the NGinx Ingress installation to work.

Environment

I'm on a managed K8S cluster (provided by OVH), which runs the 1.25.9-2 version of Kubernetes.

I'm deploying an application called REDCap on that cluster, and am just trying to test the WAF by testing SQL injection (by hand like in the howto video, or via Zed Attack Proxy).

I ensured that my application was reachable as normal with the vanilla NGinx Ingress Controller, then installed OpenAppSec in default detect-learn mode using the recommended installation script, with the "Ingress duplication" method, and redirected my DNS entry towards the OpenAppSec Ingress IP.

FYI, the Ingress does the TLS termination with certificates, set as recommended by Kubernetes/NGinx. Nothing fancy, those are official signed certificates by a recognized CA, and the application is accessible via HTTPS without issues, just as it should be.

Issue description

I can access my application by its URL, but by looking at the logs as recommended in the documentation, it seems that the open-appsec container only detects local calls to the Ingress Controller made by its probe, and not the calls made to my application : 

❯ kubectl -n appsec logs -f deployments/open-appsec-open-appsec-k8s-nginx-ingress-controller -c open-appsec
{
    "eventTime": "2023-08-10T07:00:13.079",
    "eventName": "Web Request",
    "eventSeverity": "Info",
    "eventPriority": "Low",
    "eventType": "Event Driven",
    "eventLevel": "Log",
    "eventLogLevel": "info",
    "eventAudience": "Security",
    "eventAudienceTeam": "",
    "eventFrequency": 0,
    "eventTags": [
        "Threat Prevention",
        "Web Application & API Protection"
    ],
    "eventSource": {
        "agentId": "9ea71274-32b4-4815-a23c-240ac97d2df4",
        "eventTraceId": "",
        "eventSpanId": "",
        "issuingEngineVersion": "1.0.0-open-source",
        "serviceName": "HTTP Transaction Handler",
        "serviceId": "1",
        "k8sClusterId": "871275fe-a843-486b-b739-b3e6ef09f51a",
        "assetId": "Any",
        "assetName": "Any"
    },
    "eventData": {
        "logIndex": 7154,
        "eventReferenceId": "408a0a5d-7db7-4eab-9b7b-66bcdad5ba02",
        "assetId": "Any",
        "assetName": "Any",
        "sourceIP": "127.0.0.1",
        "httpSourceId": "127.0.0.1",
        "sourcePort": 44202,
        "httpHostName": "127.0.0.1:10246",
        "httpMethod": "GET",
        "httpUriPath": "/is-dynamic-lb-initialized",
        "httpUriQuery": "",
        "ruleId": "Any",
        "securityAction": "Detect",
        "waapOverride": "None",
        "practiceType": "Threat Prevention",
        "practiceSubType": "Web Application",
        "ruleName": "Any",
        "practiceId": "fd1bea85-70e4-4d91-a58e-c675c3eb1a63",
        "practiceName": "open-appsec-best-practice-policy/appsec-best-practice",
        "waapIncidentType": "",
        "matchedSample": "",
        "matchedLocation": "",
        "matchedParameter": "",
        "waapFoundIndicators": "",
        "matchedIndicators": "",
        "learnedIndicators": "",
        "waapUserReputationScore": 298,
        "waapUserReputation": "Low",
        "waapUriFalsePositiveScore": 0,
        "waapKeywordsScore": 0,
        "waapFinalScore": 0,
        "waapCalculatedThreatLevel": 0
    }
}
{
    "eventTime": "2023-08-10T07:00:13.084",
    "eventName": "Web Request",
    "eventSeverity": "Info",
    "eventPriority": "Low",
    "eventType": "Event Driven",
    "eventLevel": "Log",
    "eventLogLevel": "info",
    "eventAudience": "Security",
    "eventAudienceTeam": "",
    "eventFrequency": 0,
    "eventTags": [
        "Threat Prevention",
        "Web Application & API Protection"
    ],
    "eventSource": {
        "agentId": "9ea71274-32b4-4815-a23c-240ac97d2df4",
        "eventTraceId": "",
        "eventSpanId": "",
        "issuingEngineVersion": "1.0.0-open-source",
        "serviceName": "HTTP Transaction Handler",
        "serviceId": "1",
        "k8sClusterId": "871275fe-a843-486b-b739-b3e6ef09f51a",
        "assetId": "Any",
        "assetName": "Any"
    },
    "eventData": {
        "logIndex": 7155,
        "eventReferenceId": "0df12102-9df2-4ee2-856c-6918cc1a4ae8",
        "assetId": "Any",
        "assetName": "Any",
        "sourceIP": "127.0.0.1",
        "httpSourceId": "127.0.0.1",
        "sourcePort": 44210,
        "httpHostName": "127.0.0.1:10246",
        "httpMethod": "GET",
        "httpUriPath": "/is-dynamic-lb-initialized",
        "httpUriQuery": "",
        "ruleId": "Any",
        "securityAction": "Detect",
        "waapOverride": "None",
        "practiceType": "Threat Prevention",
        "practiceSubType": "Web Application",
        "ruleName": "Any",
        "practiceId": "fd1bea85-70e4-4d91-a58e-c675c3eb1a63",
        "practiceName": "open-appsec-best-practice-policy/appsec-best-practice",
        "waapIncidentType": "",
        "matchedSample": "",
        "matchedLocation": "",
        "matchedParameter": "",
        "waapFoundIndicators": "",
        "matchedIndicators": "",
        "learnedIndicators": "",
        "waapUserReputationScore": 298,
        "waapUserReputation": "Low",
        "waapUriFalsePositiveScore": 0,
        "waapKeywordsScore": 0,
        "waapFinalScore": 0,
        "waapCalculatedThreatLevel": 0
    }
}

[and it goes on and on like this]

But, when I look at the logs of the NGinx controller container (the one in the appsec namespace), I can see that the calls to my application are logged as expected : 

❯ kubectl -n appsec logs -f deployments/open-appsec-open-appsec-k8s-nginx-ingress-controller -c controller

10.2.0.0 - - [15/Aug/2023:15:41:13 +0000] "GET / HTTP/2.0" 200 8646 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/116.0" 332 0.132 [[REDACTED]-externe-qual-[REDACTED]-ext-qual-httpd-svc-80] [] 10.2.3.198:80 8666 0.132 200 7812e1225ea8aa7ec004c7c12f398e66
|2023-08-15T15:41:13.263: is_ngx_cp_attachment_disabled@ngx_http_cp_attachment_module.c:243 [uid 5 | pid 319] | Reconfiguring the local NGINX attachment state
10.2.0.0 - - [15/Aug/2023:15:41:13 +0000] "GET /[REDACTED]_v13.1.13/Resources/webpack/css/bundle.css?1692112200 HTTP/2.0" 304 0 "https://[REDACTED]/" "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/116.0" 217 0.026 [[REDACTED]-externe-qual-[REDACTED]-ext-qual-httpd-svc-80] [] 10.2.3.198:80 0 0.026 304 bad0a742b21817b312416d7ac79f9ea9
10.2.0.0 - - [15/Aug/2023:15:41:13 +0000] "GET /[REDACTED]_v13.1.13/Resources/webpack/css/fontawesome/css/all.min.css?1692112200 HTTP/2.0" 304 0 "https://[REDACTED]/" "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/116.0" 89 0.065 [[REDACTED]-externe-qual-[REDACTED]-ext-qual-httpd-svc-80] [] 10.2.3.198:80 0 0.065 304 3f260df420bc2d2fa5ce8322c14f3711
10.2.0.0 - - [15/Aug/2023:15:41:13 +0000] "GET /[REDACTED]_v13.1.13/Resources/css/messenger.css?1692112199 HTTP/2.0" 304 0 "https://[REDACTED]/" "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/116.0" 98 0.065 [[REDACTED]-externe-qual-[REDACTED]-ext-qual-httpd-svc-80] [] 10.2.3.198:80 0 0.065 304 e7203b47148cbb45a78c33adc781dc08

Finally, I can see that the annotation and Ingress class on the Ingres resource have been configured correctly by the install script : 

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:

...

  annotations:
    meta.helm.sh/release-name: [REDACTED]
    meta.helm.sh/release-namespace: [REDACTED]
    nginx.ingress.kubernetes.io/affinity: cookie
    nginx.ingress.kubernetes.io/client-body-timeout: "3600"
    nginx.ingress.kubernetes.io/client-header-timeout: "3600"
    nginx.ingress.kubernetes.io/client_max_body_size: 5000m
    nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
    nginx.ingress.kubernetes.io/proxy-body-size: 5000m
    nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
    nginx.ingress.kubernetes.io/session-cookie-expires: "86400"
    nginx.ingress.kubernetes.io/session-cookie-max-age: "86400"
    openappsec.io/policy: open-appsec-best-practice-policy

...

spec:
  ingressClassName: appsec-nginx
...

Analysis

I have the impression that the solution has been correctly setup by the script, and the traffic is going to the right Ingress Controller, but for some reasons, the traffic just cannot be analyzed by the open-appsec controller.

Any thoughts?

Thanks! :)

MaxShapiro commented 1 year ago

Hi Kévin!

First of all thanks for the detailed description! It sure makes things easier to debug.

It seems that you indeed configured the application in a good way. To help us further understand what went wrong I would appreciate it if you could provide some additional information.

  1. Did you set a specific URL as the host in the ingress configuration, or is it set for any host (*)

  2. Could you please provide the policy file generated by the application? You can find it located under /etc/cp/conf/policy.json in the openappsec container, in the Ingress Controller pod (feel free to censure any sensitive information such as URLs etc)

Thanks in advance!

kzgrzendek commented 1 year ago

You're most welcome!

1) The Ingress resource is matching a specific host (to match the certificate I'm using). That seems to be OK since my application is reachable.

2) Here is the Policy resource : 

❯ kubectl get policies.openappsec.io
NAME                               AGE
open-appsec-best-practice-policy   105m

~
❯ kubectl get policies.openappsec.io open-appsec-best-practice-policy -o yaml
apiVersion: openappsec.io/v1beta1
kind: Policy
metadata:
  annotations:
    meta.helm.sh/release-name: open-appsec-helm
    meta.helm.sh/release-namespace: default
  creationTimestamp: "2023-08-15T15:18:57Z"
  generation: 1
  labels:
    app.kubernetes.io/managed-by: Helm
  name: open-appsec-best-practice-policy
  resourceVersion: "1517110099"
  uid: 593d1758-bfdc-464e-a525-cd2deb12e9d0
spec:
  default:
    custom-response: 403-forbidden
    exceptions: []
    mode: detect-learn
    practices:
    - appsec-best-practice
    source-identifiers: ""
    triggers:
    - appsec-log-trigger
    trusted-sources: ""

And here is the corresponding policy file inside the open-appsec container : 

❯ kubectl -n appsec exec -it deployments/open-appsec-open-appsec-k8s-nginx-ingress-controller -c open-appsec -- sh
/ # cat /etc/cp/conf/policy.json
{
    "waap": {
        "WAAP": {
            "WebAPISecurity": [],
            "WebApplicationSecurity": [
                {
                    "context": "practiceId(354c2586-f223-4cd3-9548-3316c4d79419)",
                    "webAttackMitigation": true,
                    "webAttackMitigationSeverity": "high",
                    "webAttackMitigationAction": "balanced",
                    "webAttackMitigationMode": "Prevent",
                    "practiceAdvancedConfig": {
                        "httpHeaderMaxSize": 102400,
                        "httpIllegalMethodsAllowed": 0,
                        "httpRequestBodyMaxSize": 1000000,
                        "jsonMaxObjectDepth": 40,
                        "urlMaxSize": 32768
                    },
                    "csrfProtection": "Disabled",
                    "openRedirect": "Disabled",
                    "errorDisclosure": "Disabled",
                    "practiceId": "354c2586-f223-4cd3-9548-3316c4d79419",
                    "practiceName": "open-appsec-best-practice-policy/appsec-best-practice",
                    "assetId": "Any",
                    "assetName": "Any",
                    "ruleId": "Any",
                    "ruleName": "Any",
                    "schemaValidation": false,
                    "schemaValidation_v2": "Disabled",
                    "oas": [],
                    "triggers": [
                        {
                            "$triggerType": "log",
                            "id": "6bed0747-7096-4b55-879d-2c3b85e52029",
                            "name": "open-appsec-best-practice-policy/appsec-log-trigger",
                            "log": {
                                "context": "triggerId(6bed0747-7096-4b55-879d-2c3b85e52029)",
                                "triggerName": "open-appsec-best-practice-policy/appsec-log-trigger",
                                "triggerType": "log",
                                "verbosity": "Standard",
                                "acAllow": false,
                                "acDrop": false,
                                "complianceViolations": false,
                                "complianceWarnings": false,
                                "extendloggingMinSeverity": "high",
                                "extendlogging": true,
                                "logToAgent": true,
                                "logToCef": false,
                                "logToCloud": false,
                                "logToK8sService": true,
                                "logToSyslog": false,
                                "responseBody": false,
                                "responseCode": false,
                                "tpDetect": true,
                                "tpPrevent": true,
                                "webBody": false,
                                "webHeaders": false,
                                "webRequests": false,
                                "webUrlPath": true,
                                "webUrlQuery": true,
                                "urlForSyslog": ":514",
                                "urlForCef": ":0",
                                "formatLoggingOutput": true
                            }
                        }
                    ],
                    "applicationUrls": "",
                    "overrides": [],
                    "trustedSources": [
                        {
                            "id": "3878bbf4-7629-4c89-af76-6e537c54500f",
                            "name": "",
                            "numOfSources": 0,
                            "sourcesIdentifiers": [],
                            "parameterType": "TrustedSource"
                        }
                    ],
                    "waapParameters": [],
                    "botProtection": false,
                    "antiBot": {
                        "injected": [],
                        "validated": []
                    },
                    "botProtection_v2": "Detect"
                }
            ]
        }
    },
    "triggers": {
        "rulebase": {
            "log": [
                {
                    "context": "triggerId(6bed0747-7096-4b55-879d-2c3b85e52029)",
                    "triggerName": "open-appsec-best-practice-policy/appsec-log-trigger",
                    "triggerType": "log",
                    "verbosity": "Standard",
                    "acAllow": false,
                    "acDrop": false,
                    "complianceViolations": false,
                    "complianceWarnings": false,
                    "extendloggingMinSeverity": "high",
                    "extendlogging": true,
                    "logToAgent": true,
                    "logToCef": false,
                    "logToCloud": false,
                    "logToK8sService": true,
                    "logToSyslog": false,
                    "responseBody": false,
                    "responseCode": false,
                    "tpDetect": true,
                    "tpPrevent": true,
                    "webBody": false,
                    "webHeaders": false,
                    "webRequests": false,
                    "webUrlPath": true,
                    "webUrlQuery": true,
                    "urlForSyslog": ":514",
                    "urlForCef": ":0",
                    "formatLoggingOutput": true
                }
            ],
            "webUserResponse": [
                {
                    "context": "triggerId(c78cc22e-fc81-4aa0-99a6-bb4aac0bed0e)",
                    "triggerName": "open-appsec-best-practice-policy/403-forbidden",
                    "details level": "response-code-only",
                    "response body": "",
                    "response code": 403,
                    "response title": ""
                }
            ]
        }
    },
    "rules": {
        "rulebase": {
            "rulesConfig": [
                {
                    "assetId": "[REDACTED]",
                    "assetName": "[REDACTED]",
                    "ruleId": "[REDACTED]",
                    "ruleName": "[REDACTED]",
                    "context": "Any(All(Any(EqualHost([REDACTED])),EqualListeningPort(80)),All(Any(EqualHost([REDACTED])),EqualListeningPort(443)))",
                    "priority": 1,
                    "isCleanup": false,
                    "parameters": [
                        {
                            "parameterId": "",
                            "parameterName": "",
                            "parameterType": "Exception"
                        }
                    ],
                    "practices": [
                        {
                            "practiceId": "97570537-908d-43ec-8e12-11ff228caafd",
                            "practiceName": "open-appsec-best-practice-policy/appsec-best-practice",
                            "practiceType": "WebApplication"
                        }
                    ],
                    "triggers": [
                        {
                            "triggerId": "6bed0747-7096-4b55-879d-2c3b85e52029",
                            "triggerName": "open-appsec-best-practice-policy/appsec-log-trigger",
                            "triggerType": "log"
                        },
                        {
                            "triggerId": "c78cc22e-fc81-4aa0-99a6-bb4aac0bed0e",
                            "triggerName": "open-appsec-best-practice-policy/403-forbidden",
                            "triggerType": "WebUserResponse"
                        }
                    ],
                    "zoneId": "",
                    "zoneName": ""
                },
                {
                    "assetId": "Any",
                    "assetName": "Any",
                    "ruleId": "Any",
                    "ruleName": "Any",
                    "context": "All()",
                    "priority": 1,
                    "isCleanup": false,
                    "parameters": [
                        {
                            "parameterId": "",
                            "parameterName": "",
                            "parameterType": "Exception"
                        }
                    ],
                    "practices": [
                        {
                            "practiceId": "354c2586-f223-4cd3-9548-3316c4d79419",
                            "practiceName": "open-appsec-best-practice-policy/appsec-best-practice",
                            "practiceType": "WebApplication"
                        }
                    ],
                    "triggers": [
                        {
                            "triggerId": "6bed0747-7096-4b55-879d-2c3b85e52029",
                            "triggerName": "open-appsec-best-practice-policy/appsec-log-trigger",
                            "triggerType": "log"
                        },
                        {
                            "triggerId": "c78cc22e-fc81-4aa0-99a6-bb4aac0bed0e",
                            "triggerName": "open-appsec-best-practice-policy/403-forbidden",
                            "triggerType": "WebUserResponse"
                        }
                    ],
                    "zoneId": "",
                    "zoneName": ""
                }
            ],
            "usersIdentifiers": []
        }
    },
    "exceptions": {
        "rulebase": {
            "exception": [
                {
                    "context": "Any()",
                    "exceptions": []
                }
            ]
        }
    },
    "version": "Tue Aug 15 17:10:16 UTC 2023"
}

I can confirm that the hosts I redacted are matching the one figuring in the Ingress resource.

MaxShapiro commented 1 year ago

Hi again!

Thanks for providing the information, that indeed helped. We've identified the issue and are working on a solution, we'll update you once it's fixed.

kzgrzendek commented 1 year ago

Well done, and thanks for your reactivity!

If you have by any chance a rough estimation (days, weeks, months?), and/or a workaround in the meantime, I'd happily have it :)

If not, good luck, and I'll follow the progress here ;)

Thanks again

MaxShapiro commented 1 year ago

This shouldn't take more than a couple of days

kzgrzendek commented 1 year ago

Awesome!

MaxShapiro commented 1 year ago

Hi!

Just wanted to update you that we've found the issue and the fix will be available in the next release.

kzgrzendek commented 1 year ago

Hello :)

Thank you for the update!

Do you have any rough idea of when the next release will be published?

WrightNed commented 1 year ago

The next release is scheduled for tomorrow.

kzgrzendek commented 12 months ago

Hi again :)

I just tried the last version released, and I'm happy to confirm that the blocking pattern is now working as expected!

Here is an example with an SQL injection from the OWASP Top 10 inside my application's URL :

{
    "eventTime": "2023-08-26T20:38:00.048",
    "eventName": "Web Request",
    "eventSeverity": "Critical",
    "eventPriority": "High",
    "eventType": "Event Driven",
    "eventLevel": "Log",
    "eventLogLevel": "info",
    "eventAudience": "Security",
    "eventAudienceTeam": "",
    "eventFrequency": 0,
    "eventTags": [
        "Threat Prevention",
        "Web Application & API Protection"
    ],
    "eventSource": {
        "agentId": "de336d3b-99eb-4a02-9811-4328db976666",
        "eventTraceId": "",
        "eventSpanId": "",
        "issuingEngineVersion": "1.0.1-open-source",
        "serviceName": "HTTP Transaction Handler",
        "serviceId": "1",
        "k8sClusterId": "df2865d8-bcaf-42ef-a191-a379f703666a",
        "assetId": "[REDCATED]/",
        "assetName": "[REDCATED]"
    },
    "eventData": {
        "logIndex": 86,
        "eventReferenceId": "491b4db0-921f-4797-93f8-0e8651b040a1",
        "assetId": "[REDCATED]",
        "assetName": "[REDCATED]",
        "eventConfidence": "Very High",
        "sourceIP": "[REDCATED]",
        "httpSourceId": "[REDCATED]",
        "sourcePort": [REDCATED],
        "httpHostName": "[REDCATED]",
        "httpMethod": "GET",
        "httpUriPath": "/",
        "httpUriQuery": "id=%27%20UNION%20SELECT%20SLEEP(10);--",
        "ruleId": "Any",
        "securityAction": "Prevent",
        "waapOverride": "None",
        "practiceType": "Threat Prevention",
        "practiceSubType": "Web Application",
        "ruleName": "Any",
        "practiceId": "5fbca4cf-d1b0-4143-afec-00026151ea19",
        "practiceName": "open-appsec-best-practice-policy/appsec-best-practice",
        "waapIncidentType": "Remote Code Execution, SQL Injection",
        "matchedSample": "' union select sleep(10);--",
        "matchedLocation": "url parameter",
        "matchedParameter": "id",
        "waapFoundIndicators": "[', --, ;, ;--, probing, regex_postfix_0, regex_prefix_0, regex_sqli_23, regex_sqli_24, select, sleep(, sqli_fast_reg_1, sqli_fast_reg_5, union]",
        "matchedIndicators": "[', --, ;, ;--, probing, regex_postfix_0, regex_prefix_0, regex_sqli_23, regex_sqli_24, select, sleep(, sqli_fast_reg_1, sqli_fast_reg_5, union]",
        "learnedIndicators": "",
        "waapUserReputationScore": 154,
        "waapUserReputation": "Low",
        "waapUriFalsePositiveScore": 864,
        "waapKeywordsScore": 966,
        "waapFinalScore": 1000,
        "waapCalculatedThreatLevel": 4
    }
}

I can now close this Issue if that's okay.

Thanks again to everyone involved for their reactivity, that's a huge plus for the adoption of the product.

Keep up the great work!