🔔 Implement Alert change streams + add SNS destination for webhooks

[shaeqahmed]

116

Add a feature that allows users to consume Matano alerts in realtime using webhooks for security automation use cases. To make this possible, we need to expose an SNS topic where we publish alerts, and that a user can create HTTP subscriptions on top of.

Note: Alerts in matano are deduplicated and track a series of rule matches over time, so this integration will actually more of an "alert change stream", where every time an alert is updated with new rule matches, the user is notified of (a) the latest state of the alert, (b) a contextual summary of the incoming rule matches that caused the alert to be republished and (c) a contextual diff that highlights what is actually new and never before seen.

For use cases like submitting indicators to scanner like VirusTotal, the context_diff feature would allow users to only submit the IOC (e.g. file hash) once if an alert ends up deduplicating many consecutive rule matches.

Alert SNS webhook payload structure

{
  "updated_alert": {
    "id": "61d3ac78-cedd-434e-b407-ae0b330e1d4a",
    "creation_time": "2023-03-15T06:21:13.687059Z",
    "title": "AWS Root Credentials Change",
    "severity": "medium",
    "severity_icon_url": "https://gist.githubusercontent.com/shaeqahmed/6c38fc5f0c3adb7e1a3fe6c5f78bbc4f/raw/9a12ff8d23592b31f224f9e27503e77b843b075c/apple-sev-medium-icon.png",
    "runbook": "Check the AWS root account activity",
    "false_positives": [],
    "destinations": [
      "slack_my_team",
      "jira_main"
    ],
    "context": {
      "cloud": {
        "account": {
          "id": [
            "2029292292992"
          ]
        },
        "region": [
          "us-west-2"
        ]
      },
      "event": {
        "action": [
          "RunInstances"
        ],
        "outcome": [
          "failure"
        ],
        "provider": [
          "ec2.amazonaws.com"
        ],
        "type": [
          "info"
        ]
      },
      "matano": {
        "table": [
          "aws_cloudtrail"
        ]
      },
      "related": {
        "user": [
          "backup",
          "John Doe"
        ]
      },
      "source": {
        "address": [
          "5.205.62.253"
        ],
        "ip": [
          "5.205.62.253"
        ]
      },
      "user": {
        "id": [
          "AIDA929292929292"
        ],
        "name": [
          "backup"
        ]
      }
    },
    "tables": [
      "aws_cloudtrail"
    ],
    "match_count": 2,
    "update_count": 1,
    "destination_to_alert_info": {
      "slack_my_team": "{\"ts\":\"1678861298.161909\"}"
    }
  },
  "incoming_rule_matches_context": { 
    "cloud": {
      "account": {
        "id": [
          "2929922929292"
        ]
      },
      "region": [
        "us-west-2"
      ]
    },
    "event": {
      "action": [
        "RunInstances"
      ],
      "outcome": [
        "failure"
      ],
      "provider": [
        "ec2.amazonaws.com"
      ],
      "type": [
        "info"
      ]
    },
    "matano": {
      "table": [
        "aws_cloudtrail"
      ]
    },
    "related": {
      "user": [
        "John Doe"
      ]
    },
    "source": {
      "address": [
        "5.205.62.253"
      ],
      "ip": [
        "5.205.62.253"
      ]
    },
    "user": {
      "id": [
        "AIDA929292929292"
      ],
      "name": [
        "John Doe"
      ]
    }
  },
  "context_diff": { // what's new
    "user": {
      "name": [
        "John Doe"
      ]
    },
    "related": {
      "user": [
        "John Doe"
      ]
    }
   } 
}